diff --git "a/hypernet/full_repos_results/cr_test/results.json" "b/hypernet/full_repos_results/cr_test/results.json" new file mode 100644--- /dev/null +++ "b/hypernet/full_repos_results/cr_test/results.json" @@ -0,0 +1,60969 @@ +{ + "exact_match_pct": 58.018372703412076, + "exact_match_count": 4421, + "n": 7620, + "code_bleu": 0.0, + "edit_similarity": 0.7920622486791775, + "entries": [ + { + "repo": "Chen-zexi/vllm-cli", + "expected": "2", + "got": "2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "0", + "got": "0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "1", + "got": "1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "4", + "got": "3", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "3", + "got": "3", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "[]", + "got": "[]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "{}", + "got": "{}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "10", + "got": "10", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"1\"", + "got": "\"1\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"7\"", + "got": "\"7\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "200", + "got": "200", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "[0]", + "got": "[0]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"0\"", + "got": "\"0\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "0.9", + "got": "0.9", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "0.7", + "got": "0.7", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "100", + "got": "100", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "500", + "got": "500", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "404", + "got": "404", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "None", + "got": "None", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "9000", + "got": "9000", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "True", + "got": "True", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "8000", + "got": "8000", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "8080", + "got": "8080", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "part", + "got": "part in warning", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 0.42105263157894735 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "4096", + "got": "\"2048\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "2048", + "got": "2048", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "8001", + "got": "8001", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "8002", + "got": "8002", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "data", + "got": "data", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "value", + "got": "value", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"2,3\"", + "got": "\"2,3\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"123\"", + "got": "\"123\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "False", + "got": "False", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"3,4\"", + "got": "\"3,4\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "12345", + "got": "12345", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "18080", + "got": "18080", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"0,1\"", + "got": "\"0,1\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "ports", + "got": "ports", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"info\"", + "got": "\"info\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"stop\"", + "got": "\"stop\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"gpt2\"", + "got": "\"gpt2\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"INFO\"", + "got": "\"INFO\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"True\"", + "got": "\"True\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "models", + "got": "models", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"gguf\"", + "got": "\"gguf\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "active", + "got": "active", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "config", + "got": "config", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "[0, 1]", + "got": "[0, 1]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"test\"", + "got": "\"test\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"list\"", + "got": "\"list\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"serve\"", + "got": "\"serve\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"MyOrg\"", + "got": "\"MyOrg\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"local\"", + "got": "\"local\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"DEBUG\"", + "got": "\"DEBUG\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "success", + "got": "success", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "logs[0]", + "got": "logs[0]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "logs[1]", + "got": "logs[1]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"error\"", + "got": "\"error\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"models\"", + "got": "\"models\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"status\"", + "got": "\"status\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "is_valid", + "got": "is_valid", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "manifest", + "got": "manifest", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "log_text", + "got": "log_text", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "profiles", + "got": "profiles", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"model1\"", + "got": "\"test-model\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "args.lora", + "got": "args.lora", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "errors[0]", + "got": "errors[0]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"profile\"", + "got": "\"profile\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"healthy\"", + "got": "\"healthy\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "call_args", + "got": "call_args", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"0,1,2,3\"", + "got": "\"0,1,2,3\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"0.0.0.0\"", + "got": "\"0.0.0.0\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"running\"", + "got": "\"healthy\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"unknown\"", + "got": "\"failed\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.23529411764705882 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"success\"", + "got": "\"success\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"vllm-cli\"", + "got": "\"vllm_cli\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "model_data", + "got": "model_data", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "os.environ", + "got": "os.environ", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "new_models", + "got": "new_models", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "is_healthy", + "got": "is_healthy", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "chunks[-1]", + "got": "chunks", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.75 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"shortcuts\"", + "got": "\"shortcuts\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"new-model\"", + "got": "\"new-model\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "directories", + "got": "directories", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"profile_B\"", + "got": "\"profile_B\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"profile_C\"", + "got": "\"profile_C\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "warning_msg", + "got": "warning_msg", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"new_value\"", + "got": "\"new_value\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "test_models", + "got": "test_models", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"qwen3:30b\"", + "got": "\"qwen3:30b\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "model_names", + "got": "model_names", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "unsupported", + "got": "UNSUPPORTED_ARCHITECTURES", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "command_str", + "got": "command_str", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "mock_server", + "got": "mock_server", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"localhost\"", + "got": "\"localhost\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"port_8004\"", + "got": "\"8004\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7058823529411765 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"test-model\"", + "got": "\"test-model\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "expected_dir", + "got": "expected_dir", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"test_value\"", + "got": "\"test_value\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "not is_valid", + "got": "not is_valid", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "original_env", + "got": "original_env", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "ollama_model", + "got": "ollama_model", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "profile_data", + "got": "profile_data", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "updated_data", + "got": "updated_data", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "all_profiles", + "got": "all_profiles", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "all(results)", + "got": "all(results)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "mock_process", + "got": "mock_process", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "float(\"inf\")", + "got": "float(\"inf\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"model-name\"", + "got": "\"model-name\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "server_config", + "got": "server_config", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "mock_profiles", + "got": "mock_profiles", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "user_profiles", + "got": "user_profiles", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"config-name\"", + "got": "\"config-name\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"actual-name\"", + "got": "\"actual-name\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "subprocess_env", + "got": "subprocess_env", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"parent_value\"", + "got": "\"parent_value\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "initial_models", + "got": "initial_models", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"ollama_model\"", + "got": "\"ollama_model\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "{\"port\": 8080}", + "got": "{\"port\": 8080}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "not is_healthy", + "got": "not is_healthy", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"profile_value\"", + "got": "\"profile_value\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "temp_config_dir", + "got": "temp_config_dir", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "registry.models", + "got": "registry.models", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "router.backends", + "got": "router.backends", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "mock_scan.called", + "got": "not any(m[\"name\"] == \"old_model\" for m in models)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.18461538461538463 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"high_throughput\"", + "got": "\"high_throughput\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "notification_text", + "got": "notification_text", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "model_config.port", + "got": "8001", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"subprocess_value\"", + "got": "\"subprocess_value\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"ollama/phi3:mini\"", + "got": "\"phi3:mini\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7586206896551724 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"ollama/qwen3:30b\"", + "got": "\"qwen3:30b\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7586206896551724 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "[\"gpt\", \"chatgpt\"]", + "got": "[\"gpt\", \"chatgpt\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "ModelState.RUNNING", + "got": "ModelState.ACTIVE", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6857142857142857 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "selected_model_path", + "got": "selected_model_path", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"hf_test_token_123\"", + "got": "\"hf_test_token_123\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"registration-fail\"", + "got": "\"registration-fail\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "str(exc_info.value)", + "got": "str(exc_info.value)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"actual-model-name\"", + "got": "\"actual-model-name\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "ModelState.SLEEPING", + "got": "ModelState.SLEEPING", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"Connection failed\"", + "got": "\"Connection failed\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "manager.vllm_servers", + "got": "manager.vllm_servers", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "expected_dir.exists()", + "got": "expected_dir.exists()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "initial_stats[\"hits\"]", + "got": "0", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "manager.user_profiles", + "got": "manager.user_profiles", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "summary1[\"available\"]", + "got": "summary1[\"available\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "len(mock_vllm_servers)", + "got": "len(mock_vllm_servers)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "server.registry.models", + "got": "server.registry.models", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"meta-llama/Llama-2-7b\"", + "got": "\"meta-llama/Llama-2-7b\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "process._active_servers", + "got": "process._active_servers", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"http://localhost:8001\"", + "got": "\"http://localhost:8001\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"http://localhost:9999\"", + "got": "\"http://localhost:9999\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "isinstance(config, dict)", + "got": "config", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "isinstance(models, list)", + "got": "isinstance(models, list)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"http://127.0.0.1:29999\"", + "got": "\"http://127.0.0.1:29999\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "summary1[\"total_models\"]", + "got": "2", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"http://localhost:18001\"", + "got": "backend_url", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.11428571428571428 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "isinstance(errors, list)", + "got": "len(errors) == 2", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.45 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "RegistrationStatus.ERROR", + "got": "RegistrationStatus.ERROR", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "{\"description\": \"Standard\"}", + "got": "{\"description\": \"Standard\"}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "f\"http://127.0.0.1:{port1}\"", + "got": "port1", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3125 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "f\"http://127.0.0.1:{port2}\"", + "got": "f\"http://127.0.0.1:{port2}\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "proxy_server.router.backends", + "got": "proxy_server.router.get_backend(\"model1\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.782608695652174 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "isinstance(fresh_models, list)", + "got": "any(m[\"name\"] == \"test\" for m in fresh_models)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.42105263157894735 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "{\"model1\", \"model2\", \"model3\"}", + "got": "{\"model1\", \"model2\", \"model3\"}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "config_manager.config_dir.exists()", + "got": "not (temp_config_dir / \"config.yaml\").exists()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.525 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"--seed 42 --enable-prefix-caching\"", + "got": "\"--seed 42 --enable-prefix-caching\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "isinstance(manifest[\"models\"], list)", + "got": "all(\"path\" in m for m in manifest[\"models\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.55 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "isinstance(manager.cache, ModelCache)", + "got": "manager.models", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.47058823529411764 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "manager.start_model(config.models[0])", + "got": "manager.start_model(\"sleepy-model\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7222222222222222 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "temp_config_dir / \"user_profiles.json\"", + "got": "temp_config_dir / \"user_profiles.json\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "data[\"choices\"][0][\"message\"][\"content\"]", + "got": "data[\"model\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.37735849056603776 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "any(m[\"name\"] == \"qwen3:30b\" for m in models)", + "got": "any(model[\"name\"] == \"qwen3:30b\" for model in models)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9183673469387755 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "any(m[\"name\"] == \"new_model\" for m in result)", + "got": "any(m[\"name\"] == \"new_model\" for m in result)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "any(m[\"name\"] == \"bert-base\" for m in models)", + "got": "any(m[\"name\"] == \"bert-base\" for m in models)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "any(m[\"name\"] == \"mistral:7b\" for m in models)", + "got": "any(m[\"name\"] == \"mistral:7b\" for m in models)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "any(m[\"name\"] == \"old_model:v1\" for m in models)", + "got": "any(m[\"name\"] == \"old_model:v1\" for m in models)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "any(m[\"name\"] == \"new_model:v2\" for m in models)", + "got": "any(m[\"name\"] == \"new_model:v2\" for m in models)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "result is None or isinstance(result, (str, dict))", + "got": "result is None", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "any(\"llama3:8b\" in m[\"name\"] for m in ollama_models)", + "got": "ollama_models[0].get(\"name\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.35 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"API_KEY=\" in log_text or \"API_KEY\" in log_text", + "got": "\"\" in log_text", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "any(str(test_model_dir) in path for path in model_paths)", + "got": "any(\"test-gemma-custom\" in p for p in model_paths)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6792452830188679 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"llama3:8b\" in model_names or \"llama3:8b\" in str(models)", + "got": "all(name in model_names for name in set(model_names))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7155963302752294 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "any(m[\"name\"] == \"old_model:v1\" for m in initial_models)", + "got": "any(m[\"name\"] == \"old_model:v1\" for m in initial_models)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "any(\"vllm\" in str(c) or \"python\" in str(c) for c in cmd)", + "got": "\"vllm\" in cmd", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.37681159420289856 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "any(m[\"name\"] == \"new_model:v2\" for m in refreshed_models)", + "got": "any(m[\"name\"] == \"new_model:v2\" for m in refreshed_models)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "[ \"custom\", \"test\", ]", + "got": "[\"custom\", \"test\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9230769230769231 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "ollama_model_config[\"model\"] in cmd or any( ollama_model_config[\"model\"] in str(c) for c in cmd )", + "got": "any(\"/ollama\" in str(c) for c in cmd)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4626865671641791 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "Exception)", + "got": "ValueError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "SystemExit)", + "got": "SystemExit)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "ValidationError)", + "got": "ValidationError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "KeyError, match=\"Model 'nonexistent' not found\")", + "got": "KeyError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3157894736842105 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"custom\")", + "got": "\"custom\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"/proxy/models/8001\")", + "got": "\"/v1/models\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6470588235294118 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"/proxy/refresh_models\")", + "got": "\"/refresh\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6285714285714286 + }, + { + "repo": "IDSIA/sacred", + "expected": "3", + "got": "len(expected_sources)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "2", + "got": "2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "5", + "got": "4", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "9", + "got": "10", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "1", + "got": "5", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "7", + "got": "7", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "4", + "got": "3", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "0", + "got": "0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "i", + "got": "i", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "d", + "got": "d", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "t", + "got": "t", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "8", + "got": "8", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "m", + "got": "m", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "st", + "got": "st", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "10", + "got": "10", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "30", + "got": "30", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "12", + "got": "12", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "\"\"", + "got": "\"\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "{}", + "got": "{}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "23", + "got": "23", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "21", + "got": "21", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "42", + "got": "42", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "20", + "got": "20", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "22", + "got": "22", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "32", + "got": "32", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "[]", + "got": "[]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "18", + "got": "18", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "17", + "got": "17", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "T2", + "got": "T2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "foo", + "got": "foo", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "obj", + "got": "obj", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "res", + "got": "res", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "nr2", + "got": "nr2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "2.0", + "got": "2.0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "100", + "got": "100", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "\"b\"", + "got": "\"b\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "\"t\"", + "got": "\"t\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "cfg", + "got": "cfg", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "116", + "got": "84", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "[0]", + "got": "[0]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "[2]", + "got": "[2]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "[1]", + "got": "(1,)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "IDSIA/sacred", + "expected": "\"1\"", + "got": "\"test_exp\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "IDSIA/sacred", + "expected": "\"2\"", + "got": "\"2\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "None", + "got": "None", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "deps", + "got": "deps", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "[\"\"]", + "got": "[]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "IDSIA/sacred", + "expected": "\"me\"", + "got": "\"me\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "\"10\"", + "got": "\"10\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "True", + "got": "True", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "info", + "got": "info", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "data", + "got": "data", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "body", + "got": "'body'", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "IDSIA/sacred", + "expected": "l[0]", + "got": "l.revelation()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.1111111111111111 + }, + { + "repo": "IDSIA/sacred", + "expected": "l[2]", + "got": "l[2]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "name", + "got": "name", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "args", + "got": "args", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "obs2", + "got": "obs2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "_id2", + "got": "_id2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "1337", + "got": "1337", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "23.5", + "got": "23.5", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "plain", + "got": "plain", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "\"foo\"", + "got": "\"foo\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "False", + "got": "True", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "IDSIA/sacred", + "expected": "\"bar\"", + "got": "\"bar\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "[foo]", + "got": "{\"foo\"}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "IDSIA/sacred", + "expected": "\"b.d\"", + "got": "\"b.d\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "\"baz\"", + "got": "\"baz\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "seed2", + "got": "seed2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "value", + "got": "value", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "cfg()", + "got": "cfg", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.75 + }, + { + "repo": "IDSIA/sacred", + "expected": "{\"d\"}", + "got": "{\"d\"}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "102.0", + "got": "102.0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "set()", + "got": "set()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "{\"a\"}", + "got": "{\"a\", \"b\"}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "IDSIA/sacred", + "expected": "[\"b\"]", + "got": "[\"b\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "[\"a\"]", + "got": "[\"a\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "tuple", + "got": "tuple", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "kw_wc", + "got": "kw_wc", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "(1, 2)", + "got": "(1, 2)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "pytest", + "got": "pytest", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "cf.run", + "got": "cf.run", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "\"tada\"", + "got": "\"tada\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "[1234]", + "got": "[1234]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "[1, 2]", + "got": "[1, 2, 3, 4]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "IDSIA/sacred", + "expected": "fbdict", + "got": "fbdict", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "vararg", + "got": "vararg", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "kwargs", + "got": "kwargs", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "[2, 4]", + "got": "[2, 4]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "\"runs\"", + "got": "\"runs\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "\"1234\"", + "got": "\"1234\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "target", + "got": "target", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "\"hello\"", + "got": "\"hello\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "__doc__", + "got": "\"tickle\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.13333333333333333 + }, + { + "repo": "IDSIA/sacred", + "expected": "\"foo42\"", + "got": "\"foo42\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "(1, 15)", + "got": "(2, 25)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7142857142857143 + }, + { + "repo": "IDSIA/sacred", + "expected": "(1, 16)", + "got": "(2, 25)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "IDSIA/sacred", + "expected": "(2, 25)", + "got": "(2, 2)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9230769230769231 + }, + { + "repo": "IDSIA/sacred", + "expected": "a.dtype", + "got": "opt.np.float32", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.09523809523809523 + }, + { + "repo": "IDSIA/sacred", + "expected": "a.shape", + "got": "a.shape", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "\"7.1.2\"", + "got": "\"0.0.0\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "IDSIA/sacred", + "expected": "\"6.2.3\"", + "got": "\"6.2.3\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "metrics", + "got": "metrics", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "obs_two", + "got": "obs_two", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "expected", + "got": "expected", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "lines[1]", + "got": "cfg_text", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.125 + }, + { + "repo": "IDSIA/sacred", + "expected": "lines[2]", + "got": "lines[2]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "lines[3]", + "got": "lines[3]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "is_local", + "got": "is_local", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "run.info", + "got": "run.info", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "greeting", + "got": "\"greeting\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "IDSIA/sacred", + "expected": "\"tickle\"", + "got": "\"tickle\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "commands", + "got": "commands", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "{\"d\": 3}", + "got": "{\"a\": 1, \"b\": 1, \"c\": 2, \"d\": 3}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "IDSIA/sacred", + "expected": "{\"a\": 7}", + "got": "{\"a\": 7}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "cfg[\"c\"]", + "got": "cfg[\"c\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "cfg[\"a\"]", + "got": "cfg[\"a\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "copied_l", + "got": "copied_l", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "v_copied", + "got": "v_copied", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "k_copied", + "got": "k_copied", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "pos_args", + "got": "pos_args", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "{\"c\": 3}", + "got": "{\"a\": 6, \"b\": 6}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "IDSIA/sacred", + "expected": "\"FAILED\"", + "got": "\"FAILED\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "\"QUEUED\"", + "got": "\"QUEUED\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "sql_obs2", + "got": "sql_obs2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "f.closed", + "got": "not custom_fh.closed", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "IDSIA/sacred", + "expected": "help_text", + "got": "help_text", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "\"testmod\"", + "got": "\"testmod\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "{\"a\": 10}", + "got": "{\"b\": 30}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7777777777777778 + }, + { + "repo": "IDSIA/sacred", + "expected": "\"Unknown\"", + "got": "\"unknown\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "IDSIA/sacred", + "expected": "(0, 0, 0)", + "got": "(0, 0, 0)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "(2, 2, 2)", + "got": "(2, 2, 0)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "IDSIA/sacred", + "expected": "{B, C, D}", + "got": "[A, B, C]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "IDSIA/sacred", + "expected": "cf.logger", + "got": "cf.logger", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "cf.config", + "got": "cf.config", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "{\"i\": 10}", + "got": "{\"i\": 10}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "[2, 3, 1]", + "got": "[2, 3, 1]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "[1, 2, 3]", + "got": "[1, 2]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "IDSIA/sacred", + "expected": "[3, 1, 2]", + "got": "[1, 2, 3]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "IDSIA/sacred", + "expected": "\"RUNNING\"", + "got": "\"COMPLETED\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2 + }, + { + "repo": "IDSIA/sacred", + "expected": "\"metrics\"", + "got": "\"metrics\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "(\"args\",)", + "got": "(\"args\", \"kwargs\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "IDSIA/sacred", + "expected": "[\"about\"]", + "got": "[\"about\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "\"1.2.3rc4\"", + "got": "\"1.2.3rc4\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "get_answer", + "got": "\"get_answer\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "IDSIA/sacred", + "expected": "m.commands", + "got": "m.commands", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "has_pytest", + "got": "has_pytest", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "cfg()[\"a\"]", + "got": "cfg[\"a\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "IDSIA/sacred", + "expected": "{\"answer\"}", + "got": "{\"a\"}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "IDSIA/sacred", + "expected": "{\"a\", \"b\"}", + "got": "{'a', 'b'}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6 + }, + { + "repo": "IDSIA/sacred", + "expected": "[\"a\", \"b\"]", + "got": "[\"a\", \"b\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "fail_trace", + "got": "fail_trace", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "observer_3", + "got": "observer_3", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "observer_2", + "got": "observer_2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "event_name", + "got": "event_name", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "5 * 10 + 6", + "got": "(5 * 10 + 6) * 3", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7692307692307693 + }, + { + "repo": "IDSIA/sacred", + "expected": "ex.commands", + "got": "sys.argv", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.10526315789473684 + }, + { + "repo": "IDSIA/sacred", + "expected": "executed[0]", + "got": "executed[0]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "ing.sources", + "got": "ing.sources", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "new_version", + "got": "current_version", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6923076923076923 + }, + { + "repo": "IDSIA/sacred", + "expected": "old_version", + "got": "old_version", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "\"COMPLETED\"", + "got": "\"COMPLETED\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "saved_config", + "got": "saved_config", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "\"progress 9\"", + "got": "\"progress 0\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9166666666666666 + }, + { + "repo": "IDSIA/sacred", + "expected": "ing.commands", + "got": "ing.commands", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "transmogrify", + "got": "transmogrify", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "\"0123456789\"", + "got": "\"0123456789\\n\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9230769230769231 + }, + { + "repo": "IDSIA/sacred", + "expected": "(nr1, seed1)", + "got": "(cf())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "IDSIA/sacred", + "expected": "tmpfile.name", + "got": "tmpfile.name", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "file_content", + "got": "file_content", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "\"hello world\"", + "got": "\"hello world\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "named_configs", + "got": "named_configs", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "not has_nonex", + "got": "not has_nonex", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "random.Random", + "got": "random.Random", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "dedented_body", + "got": "dedented_body", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "[(\"b\", 1234)]", + "got": "[(\"b\", 1234)]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "\"some output\"", + "got": "\"some output\\n\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9285714285714286 + }, + { + "repo": "IDSIA/sacred", + "expected": "\"INTERRUPTED\"", + "got": "\"INTERRUPTED\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "\"log_metrics\"", + "got": "\"log_metrics\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "(expected, [])", + "got": "expected", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7272727272727273 + }, + { + "repo": "IDSIA/sacred", + "expected": "EXAMPLE_DIGEST", + "got": "EXAMPLE_DIGEST", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "EXAMPLE_SOURCE", + "got": "EXAMPLE_SOURCE", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "expected_lines", + "got": "expected_lines", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "copied_l[1][1]", + "got": "copied_l[1][1]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "T2.isoformat()", + "got": "T2.isoformat()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "\"testing_runs\"", + "got": "\"runs\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6 + }, + { + "repo": "IDSIA/sacred", + "expected": "OSError)", + "got": "FileNotFoundError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "IDSIA/sacred", + "expected": "IOError)", + "got": "IOError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "KeyError)", + "got": "TypeError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7368421052631579 + }, + { + "repo": "IDSIA/sacred", + "expected": "TypeError)", + "got": "TypeError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "exception)", + "got": "ObserverError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "IDSIA/sacred", + "expected": "NameError)", + "got": "NameError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "ValueError)", + "got": "ValueError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "SacredError)", + "got": "TypeError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6363636363636364 + }, + { + "repo": "IDSIA/sacred", + "expected": "ImportError)", + "got": "ImportError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "RuntimeError)", + "got": "ObserverError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5185185185185185 + }, + { + "repo": "IDSIA/sacred", + "expected": "SettingError)", + "got": "SettingError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "ObserverError)", + "got": "ObserverError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "AttributeError)", + "got": "AttributeError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "FileExistsError)", + "got": "FileNotFoundError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6470588235294118 + }, + { + "repo": "Lancetnik/Propan", + "expected": "2", + "got": "2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "1", + "got": "1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "3", + "got": "3", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "0", + "got": "0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "4", + "got": "3", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "a", + "got": "a", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "10", + "got": "10", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "q2", + "got": "q2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "s1", + "got": "Singleton()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "\"1\"", + "got": "\"1\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "app", + "got": "app", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "200", + "got": "200", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "404", + "got": "404", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "None", + "got": "\"pong\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "Lancetnik/Propan", + "expected": "\"hi\"", + "got": "\"hi\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "True", + "got": "True", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "\"pong\"", + "got": "\"pong\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "\"test\"", + "got": "\"test\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "broker", + "got": "broker", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "logger", + "got": "logger", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "schema", + "got": "schema[\"channels\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "Lancetnik/Propan", + "expected": "exp_app", + "got": "\"app\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "Lancetnik/Propan", + "expected": "message2", + "got": "message2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "r.stdout", + "got": "r.stderr", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.625 + }, + { + "repo": "Lancetnik/Propan", + "expected": "\"MyField\"", + "got": "\"MyField\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "\"Handler\"", + "got": "\"test\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "Lancetnik/Propan", + "expected": "(True, 1)", + "got": "(False, \"1\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "Lancetnik/Propan", + "expected": "is_casted", + "got": "is_casted", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "APPLICATION", + "got": "\"module:app\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "Singleton()", + "got": "Singleton()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "await use()", + "got": "await use()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "logging.INFO", + "got": "logging.DEBUG", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.64 + }, + { + "repo": "Lancetnik/Propan", + "expected": "await func()", + "got": "await func()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "await use1()", + "got": "await use1()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "await use2()", + "got": "await use2()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "await use3()", + "got": "await use3()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "\"FuncMessage\"", + "got": "\"handler\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "Lancetnik/Propan", + "expected": "result.stdout", + "got": "result.stdout", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "router.broker", + "got": "router.broker", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "func(key=key)", + "got": "await func()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.48 + }, + { + "repo": "Lancetnik/Propan", + "expected": "broker.started", + "got": "False", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.21052631578947367 + }, + { + "repo": "Lancetnik/Propan", + "expected": "{\"b\", \"a\", \"c\"}", + "got": "{\"a\", \"b\"}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.72 + }, + { + "repo": "Lancetnik/Propan", + "expected": "result[\"title\"]", + "got": "\"MyField\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "Lancetnik/Propan", + "expected": "\"HandlerMessage\"", + "got": "\"PerfectArgument\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.42424242424242425 + }, + { + "repo": "Lancetnik/Propan", + "expected": "expected_message", + "got": "expected_message", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "consume.is_set()", + "got": "consume.is_set()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "\"PerfectArgument\"", + "got": "\"Handler\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "Lancetnik/Propan", + "expected": "consume2.is_set()", + "got": "consume2.is_set()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "consume3.is_set()", + "got": "consume3.is_set()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "isinstance(r, str)", + "got": "isinstance(r[\"value\"], str)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "Lancetnik/Propan", + "expected": "not broker.started", + "got": "not broker.started", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "str(excinfo.value)", + "got": "str(excinfo.value)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "isinstance(s, int)", + "got": "isinstance(s, str)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "Lancetnik/Propan", + "expected": "await func(key=key)", + "got": "await func(key=key)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "not mock.exit.called", + "got": "consume.is_set()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "Lancetnik/Propan", + "expected": "schema_path.exists()", + "got": "schema_path.exists()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "not mock.parse.called", + "got": "mock.decode_local.called", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5777777777777777 + }, + { + "repo": "Lancetnik/Propan", + "expected": "not start_mock.called", + "got": "False", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.23076923076923078 + }, + { + "repo": "Lancetnik/Propan", + "expected": "await broker.connect()", + "got": "not broker.started", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "Lancetnik/Propan", + "expected": "await handler(message)", + "got": "await test_broker.publish(message)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6071428571428571 + }, + { + "repo": "Lancetnik/Propan", + "expected": "not mock.decode.called", + "got": "mock.decode_local.called", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.782608695652174 + }, + { + "repo": "Lancetnik/Propan", + "expected": "context[\"sync_called\"]", + "got": "context.started", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5945945945945946 + }, + { + "repo": "Lancetnik/Propan", + "expected": "isinstance(r[\"a\"], int)", + "got": "isinstance(r, dict)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8095238095238095 + }, + { + "repo": "Lancetnik/Propan", + "expected": "await self.ping(broker)", + "got": "broker.started", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.32432432432432434 + }, + { + "repo": "Lancetnik/Propan", + "expected": "Path.cwd() / exp_module", + "got": "\"tests\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.13333333333333333 + }, + { + "repo": "Lancetnik/Propan", + "expected": "app.broker.logger.level", + "got": "level", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.35714285714285715 + }, + { + "repo": "Lancetnik/Propan", + "expected": "context[\"async_called\"]", + "got": "context[\"async_called\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "await func(model=model)", + "got": "await func()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6857142857142857 + }, + { + "repo": "Lancetnik/Propan", + "expected": "isinstance(example, int)", + "got": "isinstance(example, str)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9166666666666666 + }, + { + "repo": "Lancetnik/Propan", + "expected": "{\"a\", \"b\", \"c\", \"m\", \"k\"}", + "got": "{\"a\", \"b\"}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "Lancetnik/Propan", + "expected": "not mock.call_stop2.called", + "got": "not mock.call_stop2.called", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "sqs_async_project.exists()", + "got": "sqs_async_project.exists()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "not mock.call_start2.called", + "got": "mock.call_start1.called", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.88 + }, + { + "repo": "Lancetnik/Propan", + "expected": "nats_async_project.exists()", + "got": "nats_async_project.exists()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "isinstance(examples[0], str)", + "got": "examples[0][\"message\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.48 + }, + { + "repo": "Lancetnik/Propan", + "expected": "not async_mock.on_max.called", + "got": "not async_mock.on_success.called", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "Lancetnik/Propan", + "expected": "{r.result() for r in result}", + "got": "{result[0].value, result[1].value}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4838709677419355 + }, + { + "repo": "Lancetnik/Propan", + "expected": "redis_async_project.exists()", + "got": "redis_async_project.exists()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "kafka_async_project.exists()", + "got": "kafka_async_project.exists()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "isinstance(example[\"a\"], int)", + "got": "isinstance(example, tuple)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "Lancetnik/Propan", + "expected": "rabbit_async_project.exists()", + "got": "rabbit_async_project", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8163265306122449 + }, + { + "repo": "Lancetnik/Propan", + "expected": "await broker.connect(**kwargs)", + "got": "not await broker.connect()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7857142857142857 + }, + { + "repo": "Lancetnik/Propan", + "expected": "not async_mock.on_error.called", + "got": "not async_mock.on_success.called", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8387096774193549 + }, + { + "repo": "Lancetnik/Propan", + "expected": "nats_js_async_project.exists()", + "got": "nats_js_async_project.exists()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "not async_mock.broker_stop.called", + "got": "async_mock.before.called", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7368421052631579 + }, + { + "repo": "Lancetnik/Propan", + "expected": "not async_mock.broker_start.called", + "got": "not async_mock.after.called", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7868852459016393 + }, + { + "repo": "Lancetnik/Propan", + "expected": "await broker.connect(*args, **kwargs)", + "got": "await broker.connect_by_url(settings.url)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6410256410256411 + }, + { + "repo": "Lancetnik/Propan", + "expected": "await broker.connect(url=settings.url)", + "got": "await broker.connect(host=settings.host)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8205128205128205 + }, + { + "repo": "Lancetnik/Propan", + "expected": "isinstance(message1, aio_pika.Message)", + "got": "message1 is not None", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.41379310344827586 + }, + { + "repo": "Lancetnik/Propan", + "expected": "isinstance(broker.js, JetStreamContext)", + "got": "broker.js", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.375 + }, + { + "repo": "Lancetnik/Propan", + "expected": "{\"title\": \"FuncReply\", \"type\": \"string\"}", + "got": "{}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.09523809523809523 + }, + { + "repo": "Lancetnik/Propan", + "expected": "result[\"properties\"][\"b\"][\"description\"]", + "got": "\"BField\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.20833333333333334 + }, + { + "repo": "Lancetnik/Propan", + "expected": "result[\"properties\"][\"a\"][\"description\"]", + "got": "\"a\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.13953488372093023 + }, + { + "repo": "Lancetnik/Propan", + "expected": "code == signal.SIGTERM.value or code == 0", + "got": "code", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.17777777777777778 + }, + { + "repo": "Lancetnik/Propan", + "expected": "not watcher.memory.get(message.message_id)", + "got": "not async_mock.on_error.called", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3055555555555556 + }, + { + "repo": "Lancetnik/Propan", + "expected": "await broker.connect(servers=settings.url)", + "got": "broker.url", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.38461538461538464 + }, + { + "repo": "Lancetnik/Propan", + "expected": "await broker.connect(url=settings.url, **kwargs)", + "got": "broker", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "Lancetnik/Propan", + "expected": "await test_broker.publish(\"\", queue, callback=True)", + "got": "await self._test_broker.send(\"test\", \"test\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5473684210526316 + }, + { + "repo": "Lancetnik/Propan", + "expected": "await broker.connect(bootstrap_servers=settings.url)", + "got": "broker.url", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3225806451612903 + }, + { + "repo": "Lancetnik/Propan", + "expected": "await test_broker.publish(\"\", \"logs.error\", callback=True)", + "got": "await test_broker.publish(\"\", exchange=\"test2\", callback=True)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "Lancetnik/Propan", + "expected": "await test_broker.publish(\"\", exchange=\"test2\", callback=True)", + "got": "await test_broker.publish(\"\", queue, callback=False)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7894736842105263 + }, + { + "repo": "Lancetnik/Propan", + "expected": "await test_broker.publish(exchange=exch, callback=True, headers={})", + "got": "await test_broker.publish(exchange=exch, callback=True)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9016393442622951 + }, + { + "repo": "Lancetnik/Propan", + "expected": "{ \"title\": \"FuncPayload\", \"type\": \"null\", }", + "got": "{ \"properties\": { \"a\": {\"title\": \"A\", \"type\": \"integer\"} }, \"required\": [\"a\"], \"title\": \"Message\", \"type\": \"object\", }", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.38509316770186336 + }, + { + "repo": "Lancetnik/Propan", + "expected": "{ \"protocol\": \"nats\", \"url\": \"nats://localhost:4222\", }", + "got": "\"https://localhost:4222\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5569620253164557 + }, + { + "repo": "Lancetnik/Propan", + "expected": "{ \"protocol\": \"sqs\", \"url\": \"http://localhost:9324/\", }", + "got": "\"http://localhost:8080\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.48717948717948717 + }, + { + "repo": "Lancetnik/Propan", + "expected": "{ \"protocol\": \"redis\", \"url\": \"redis://localhost:6379\", }", + "got": "\"http://localhost:8080\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "Lancetnik/Propan", + "expected": "{ \"description\": \"\", \"title\": \"Propan\", \"version\": \"0.1.0\", }", + "got": "{ \"title\": \"Propan\", \"version\": \"0.0.0\" }", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7843137254901961 + }, + { + "repo": "Lancetnik/Propan", + "expected": "await broker.connect( host=settings.host, port=settings.port, )", + "got": "await self.ping(broker)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3023255813953488 + }, + { + "repo": "Lancetnik/Propan", + "expected": "{ \"protocol\": \"kafka\", \"url\": \"localhost\", \"protocolVersion\": \"auto\", }", + "got": "\"http://localhost:9092\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2978723404255319 + }, + { + "repo": "Lancetnik/Propan", + "expected": "{ \"description\": \"description\", \"title\": \"My App\", \"version\": \"1.0.0\", }", + "got": "{ \"description\": \"description\", \"title\": \"My App\", \"version\": \"1.0.0\", }", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "await test_broker.publish( \"\", \"logs.info\", exchange=exch, callback=True )", + "got": "await test_broker.publish(\"\", \"test.info\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6724137931034483 + }, + { + "repo": "Lancetnik/Propan", + "expected": "await test_broker.publish( \"\", \"logs.error\", exchange=exch, callback=True )", + "got": "await test_broker.publish(\"\", \"logs.error\", exchange=exch)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8721804511278195 + }, + { + "repo": "Lancetnik/Propan", + "expected": "await test_broker.publish( \"\", queue + \"1\", exchange=\"test\", callback=True )", + "got": "await test_broker.publish(\"\", queue + \"1\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.711864406779661 + }, + { + "repo": "Lancetnik/Propan", + "expected": "await test_broker.publish( exchange=exch, callback=True, headers={\"key\": 2} )", + "got": "await test_broker.publish(", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5048543689320388 + }, + { + "repo": "Lancetnik/Propan", + "expected": "{ \"protocol\": \"kafka-secury\", \"url\": \"kafka:9092\", \"protocolVersion\": \"1.0.0\", }", + "got": "{ \"protocol\": \"kafka-secury\", \"url\": \"kafka:9092\", \"protocolVersion\": \"1.0.0\", }", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "{ \"protocol\": \"amqps\", \"url\": \"amqps://rabbithost.com\", \"protocolVersion\": \"0.8.0\", }", + "got": "{ \"protocol\": \"amqps\", \"url\": \"amqps://rabbithost.com\", \"protocolVersion\": \"0.8.0\", }", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "{ \"k\": \"1\", \"k2\": \"1\", \"k3\": True, \"k4\": False, \"k5\": [\"1\", \"1\"], \"some_key\": True, }", + "got": "(\"1\", \"1\", \"1\", \"1\", \"1\", \"1\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.24347826086956523 + }, + { + "repo": "Lancetnik/Propan", + "expected": "await test_broker.publish( exchange=exch, callback=True, headers={\"key\": 2, \"key2\": 2} )", + "got": "await test_broker.publish(\"\", \"test-queue-2\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5263157894736842 + }, + { + "repo": "Lancetnik/Propan", + "expected": "any( (TestRabbitBroker, TestRedisBroker, TestKafkaBroker, TestNatsBroker, TestSQSBroker) )", + "got": "TestRabbitBroker", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3018867924528302 + }, + { + "repo": "Lancetnik/Propan", + "expected": "{ \"protocol\": \"amqp\", \"url\": \"amqp://guest:guest@localhost:5672/\", \"protocolVersion\": \"0.9.1\", }", + "got": "\"http://localhost:5672\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.35294117647058826 + }, + { + "repo": "Lancetnik/Propan", + "expected": "await broker.connect( host=settings.host, login=settings.login, password=settings.password, port=settings.port, )", + "got": "await broker.connect()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.32592592592592595 + }, + { + "repo": "Lancetnik/Propan", + "expected": "{ \"example\": {\"a\": 1}, \"properties\": { \"a\": {\"title\": \"A\", \"type\": \"integer\"}, }, \"required\": [\"a\"], \"title\": \"Message\", \"type\": \"object\", }", + "got": "{ \"title\": \"FuncPayload\", \"type\": \"object\", \"properties\": { \"a\": {\"title\": \"A\", \"type\": \"integer\"} }, \"required\": [\"a\"], }", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7099236641221374 + }, + { + "repo": "Lancetnik/Propan", + "expected": "{ \"examples\": [{\"a\": 1}], \"properties\": { \"a\": {\"title\": \"A\", \"type\": \"integer\"}, }, \"required\": [\"a\"], \"title\": \"Message\", \"type\": \"object\", }", + "got": "{\"title\": \"Message\", \"type\": \"object\"}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4198895027624309 + }, + { + "repo": "Lancetnik/Propan", + "expected": "{ \"Handler\": { \"bindings\": {\"kafka\": {\"bindingVersion\": \"0.4.0\", \"topic\": [\"test\"]}}, \"servers\": [\"dev\"], \"subscribe\": {\"message\": {\"$ref\": \"", + "got": "[\"test\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.10738255033557047 + }, + { + "repo": "Lancetnik/Propan", + "expected": "{ \"Handler\": { \"bindings\": {\"nats\": {\"bindingVersion\": \"custom\", \"subject\": \"test\"}}, \"servers\": [\"dev\"], \"subscribe\": { \"bindings\": {\"nats\": {\"bindingVersion\": \"custom\"}}, \"message\": {\"$ref\": \"", + "got": "[\"test\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.06930693069306931 + }, + { + "repo": "Lancetnik/Propan", + "expected": "{ \"Handler\": { \"bindings\": { \"sqs\": { \"bindingVersion\": \"custom\", \"queue\": {\"fifo\": False, \"name\": \"test\"}, } }, \"servers\": [\"dev\"], \"subscribe\": { \"bindings\": {\"sqs\": {\"bindingVersion\": \"custom\"}}, \"message\": {\"$ref\": \"", + "got": "[\"test\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.03508771929824561 + }, + { + "repo": "Lancetnik/Propan", + "expected": "{ \"Handler\": { \"bindings\": { \"redis\": { \"bindingVersion\": \"custom\", \"channel\": \"test\", \"method\": \"subscribe\", } }, \"servers\": [\"dev\"], \"subscribe\": { \"bindings\": {\"redis\": {\"bindingVersion\": \"custom\"}}, \"message\": {\"$ref\": \"", + "got": "[\"test\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.034482758620689655 + }, + { + "repo": "Lancetnik/Propan", + "expected": "{ \"Handler\": { \"bindings\": { \"nats\": { \"bindingVersion\": \"custom\", \"queue\": \"workers\", \"subject\": \"*test\", } }, \"servers\": [\"dev\"], \"subscribe\": { \"bindings\": { \"nats\": { \"bindingVersion\": \"custom\", \"replyTo\": {\"title\": \"HandlerReply\", \"type\": \"string\"}, } }, \"message\": {\"$ref\": \"", + "got": "{ \"Handler\": { \"bindings\": {\"nats\": {\"bindingVersion\": \"custom\", \"subject\": \"test\"}}, \"servers\": [\"dev\"], \"subscribe\": { \"bindings\": {\"nats\": {\"bindingVersion\": \"custom\"}}, \"message\": {\"$ref\": \"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7115789473684211 + }, + { + "repo": "Lancetnik/Propan", + "expected": "{ \"Handler\": { \"bindings\": { \"sqs\": { \"bindingVersion\": \"custom\", \"queue\": {\"fifo\": False, \"name\": \"test\"}, } }, \"servers\": [\"dev\"], \"subscribe\": { \"bindings\": { \"sqs\": { \"bindingVersion\": \"custom\", \"replyTo\": {\"title\": \"HandlerReply\", \"type\": \"string\"}, } }, \"message\": {\"$ref\": \"", + "got": "{ \"Handler\": { \"bindings\": { \"sqs\": { \"bindingVersion\": \"custom\", \"queue\": {\"fifo\": False, \"name\": \"test\"} }, \"servers\": [\"dev\"], \"subscribe\": { \"bindings\": {\"sqs\": {\"bindingVersion\": \"custom\"}}, \"message\": {\"$ref\": \"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8674698795180723 + }, + { + "repo": "Lancetnik/Propan", + "expected": "{ \"Handler\": { \"bindings\": { \"redis\": { \"bindingVersion\": \"custom\", \"channel\": \"*test\", \"method\": \"psubscribe\", } }, \"servers\": [\"dev\"], \"subscribe\": { \"bindings\": { \"redis\": { \"bindingVersion\": \"custom\", \"replyTo\": {\"title\": \"HandlerReply\", \"type\": \"string\"}, } }, \"message\": {\"$ref\": \"", + "got": "{ \"Handler\": { \"bindings\": { \"redis\": { \"bindingVersion\": \"custom\", \"channel\": \"test\", \"method\": \"subscribe\", \"pattern\": True, } }, \"servers\": [\"dev\"], \"subscribe\": { \"bindings\": {\"redis\": {\"bindingVersion\": \"custom\"}}, \"message\": {\"$ref\": \"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4659090909090909 + }, + { + "repo": "Lancetnik/Propan", + "expected": "{ \"Handler\": { \"bindings\": {\"kafka\": {\"bindingVersion\": \"0.4.0\", \"topic\": [\"test\"]}}, \"servers\": [\"dev\"], \"subscribe\": { \"bindings\": { \"kafka\": { \"bindingVersion\": \"0.4.0\", \"groupId\": {\"enum\": [\"workers\"], \"type\": \"string\"}, \"replyTo\": { \"title\": \"HandlerReply\", \"type\": \"string\", }, } }, \"message\": {\"$ref\": \"", + "got": "{ \"Handler\": { \"bindings\": {\"kafka\": {\"bindingVersion\": \"0.4.0\", \"topic\": [\"test\"]}}, \"servers\": [\"dev\"], \"subscribe\": {\"message\": {\"$ref\": \"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6208425720620843 + }, + { + "repo": "Lancetnik/Propan", + "expected": "{ \"Handler\": { \"bindings\": { \"amqp\": { \"bindingVersion\": \"0.2.0\", \"exchange\": { \"autoDelete\": False, \"durable\": False, \"name\": \"test\", \"type\": \"fanout\", \"vhost\": \"/\", }, \"is\": \"routingKey\", } }, \"servers\": [\"dev\"], \"subscribe\": { \"bindings\": {\"amqp\": {\"ack\": True, \"bindingVersion\": \"0.2.0\"}}, \"description\": \"Test description\", \"message\": {\"$ref\": \"", + "got": "{ \"Handler\": { \"bindings\": { \"amqp\": { \"bindingVersion\": \"0.2.0\", \"exchange\": {\"type\": \"fanout\", \"vhost\": \"/\"}, \"is\": \"routingKey\", \"queue\": { \"autoDelete\": False, \"durable\": False, \"exclusive\": False, \"name\": \"test\", \"vhost\": \"/\", }, }, }, \"servers\": [\"dev\"], \"subscribe\": { \"bindings\": { \"amqp\": {\"ack\": True, \"bindingVersion\": \"0.2.0\", \"cc\": \"test\"}, }, \"message\": {\"$ref\": \"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6510989010989011 + }, + { + "repo": "Lancetnik/Propan", + "expected": "{ \"Handler\": { \"bindings\": { \"amqp\": { \"bindingVersion\": \"0.2.0\", \"exchange\": {\"type\": \"default\", \"vhost\": \"/\"}, \"is\": \"routingKey\", \"queue\": { \"autoDelete\": False, \"durable\": False, \"exclusive\": False, \"name\": \"test\", \"vhost\": \"/\", }, } }, \"servers\": [\"dev\"], \"subscribe\": { \"bindings\": { \"amqp\": {\"ack\": True, \"bindingVersion\": \"0.2.0\", \"cc\": \"test\"} }, \"message\": {\"$ref\": \"", + "got": "{\"test\"}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.005194805194805195 + }, + { + "repo": "Lancetnik/Propan", + "expected": "schema[\"info\"] == { \"contact\": { \"email\": \"my-domain@gmail.com\", \"name\": \"Developer\", \"url\": \"http://my-domain.com/\", }, \"description\": \"description\", \"license\": {\"name\": \"MIT\", \"url\": \"http://mit.com/\"}, \"title\": \"My App\", \"version\": \"1.0.0\", } or schema[\"info\"] == { \"contact\": { \"email\": \"my-domain@gmail.com\", \"name\": \"Developer\", \"url\": \"http://my-domain.com\", }, \"description\": \"description\", \"license\": {\"name\": \"MIT\", \"url\": \"http://mit.com\"}, \"title\": \"My App\", \"version\": \"1.0.0\", }", + "got": "schema[\"info\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.05533596837944664 + }, + { + "repo": "Lancetnik/Propan", + "expected": "{ \"Handler\": { \"bindings\": { \"amqp\": { \"bindingVersion\": \"0.2.0\", \"exchange\": { \"autoDelete\": False, \"durable\": False, \"name\": \"test\", \"type\": \"direct\", \"vhost\": \"/\", }, \"is\": \"routingKey\", \"queue\": { \"autoDelete\": False, \"durable\": False, \"exclusive\": False, \"name\": \"test\", \"vhost\": \"/\", }, } }, \"servers\": [\"dev\"], \"subscribe\": { \"bindings\": { \"amqp\": {\"ack\": True, \"bindingVersion\": \"0.2.0\", \"cc\": \"test\"} }, \"description\": \"Test description\", \"message\": {\"$ref\": \"", + "got": "{ \"Handler\": { \"bindings\": { \"amqp\": { \"bindingVersion\": \"0.2.0\", \"exchange\": { \"autoDelete\": False, \"durable\": False, \"name\": \"test\", \"type\": \"direct\", \"vhost\": \"/\", }, \"is\": \"routingKey\", }, }, \"servers\": [\"dev\"], \"subscribe\": { \"bindings\": {\"amqp\": {\"ack\": True, \"bindingVersion\": \"0.2.0\"}}, \"description\": \"Test description\", \"message\": {\"$ref\": \"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8292682926829268 + }, + { + "repo": "Lancetnik/Propan", + "expected": "ValueError)", + "got": "ValueError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "SkipMessage)", + "got": "SkipMessage)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "TimeoutError)", + "got": "TimeoutError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "AssertionError)", + "got": "TypeError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.56 + }, + { + "repo": "Lancetnik/Propan", + "expected": "ValidationError)", + "got": "ValidationError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "FileNotFoundError)", + "got": "ImportError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5333333333333333 + }, + { + "repo": "Lancetnik/Propan", + "expected": "RequestValidationError)", + "got": "RequestValidationError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "pydantic.ValidationError)", + "got": "pydantic.error_wrappers.ValidationError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7692307692307693 + }, + { + "repo": "Lancetnik/Propan", + "expected": "\"1\")", + "got": "\"1\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "\"test\")", + "got": "\"test\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "\"hi, depends\")", + "got": "\"hi\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5263157894736842 + }, + { + "repo": "Lancetnik/Propan", + "expected": "expected_message)", + "got": "expected_message)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "{\"a\": 1, \"b\": 1, \"args\": (2, 3)})", + "got": "{\"a\": 1, \"b\": 1, \"args\": (1.0, 2.0, 3.0)})", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.88 + }, + { + "repo": "Lancetnik/Propan", + "expected": "{ \"a\": 1, \"b\": 1, })", + "got": "{\"a\": 1, \"b\": 1.0})", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8717948717948718 + }, + { + "repo": "MinishLab/semhash", + "expected": "1", + "got": "2", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "MinishLab/semhash", + "expected": "3", + "got": "3", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MinishLab/semhash", + "expected": "2", + "got": "2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MinishLab/semhash", + "expected": "[]", + "got": "[]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MinishLab/semhash", + "expected": "10", + "got": "10", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MinishLab/semhash", + "expected": "42", + "got": "42", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MinishLab/semhash", + "expected": "32", + "got": "32", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MinishLab/semhash", + "expected": "50", + "got": "50", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MinishLab/semhash", + "expected": "0.4", + "got": "0.8", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "MinishLab/semhash", + "expected": "0.2", + "got": "1.0", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "MinishLab/semhash", + "expected": "0.0", + "got": "0.0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MinishLab/semhash", + "expected": "1.0", + "got": "0.0", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "MinishLab/semhash", + "expected": "img", + "got": "img", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MinishLab/semhash", + "expected": "100", + "got": "10", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "MinishLab/semhash", + "expected": "3.14", + "got": "3.14", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MinishLab/semhash", + "expected": "\"42\"", + "got": "\"42\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MinishLab/semhash", + "expected": "True", + "got": "True", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MinishLab/semhash", + "expected": "texts", + "got": "texts", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MinishLab/semhash", + "expected": "False", + "got": "False", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MinishLab/semhash", + "expected": "{1, 2}", + "got": "{1, 2}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MinishLab/semhash", + "expected": "texts2", + "got": "texts1", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "MinishLab/semhash", + "expected": "(3, 1)", + "got": "(3,)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "MinishLab/semhash", + "expected": "\"3.14\"", + "got": "\"3.14\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MinishLab/semhash", + "expected": "result", + "got": "result", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MinishLab/semhash", + "expected": "result2", + "got": "result2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MinishLab/semhash", + "expected": "expected", + "got": "expected", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MinishLab/semhash", + "expected": "selected", + "got": "\"hello\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.26666666666666666 + }, + { + "repo": "MinishLab/semhash", + "expected": "(2, 128)", + "got": "(2,)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "MinishLab/semhash", + "expected": "[(\"b\", 0.9)]", + "got": "[(\"b\", 0.9), (\"c\", 0.8)]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "MinishLab/semhash", + "expected": "[(\"a\", \"c\", 0.7)]", + "got": "[\"b\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "MinishLab/semhash", + "expected": "[\"a\", \"b\", \"c\", \"e\"]", + "got": "[DuplicateRecord(\"e\", False, [(\"z\", 0.8)])]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.19047619047619047 + }, + { + "repo": "MinishLab/semhash", + "expected": "len(result2.selected)", + "got": "len(result2.selected)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MinishLab/semhash", + "expected": "[\"It's dangerous to go alone!\"]", + "got": "[\"It's dangerous to go alone!\", \"It's not safe to go alone!\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6739130434782609 + }, + { + "repo": "MinishLab/semhash", + "expected": "frozendict({\"a\": \"1\", \"c\": \"3\"})", + "got": "frozendict({\"a\": \"1\", \"c\": \"3\"})", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MinishLab/semhash", + "expected": "{\"grapefruit\", \"banana\", \"apple\"}", + "got": "{\"blueberry\", \"pineapple\", \"grape\"}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4411764705882353 + }, + { + "repo": "MinishLab/semhash", + "expected": "isinstance(embeddings, np.ndarray)", + "got": "np.all(embeddings[0] != embeddings[1])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "MinishLab/semhash", + "expected": "{\"blueberry\", \"pineapple\", \"grape\"}", + "got": "{\"It's dangerous to go alone!\", \"The master sword can seal the darkness.\", \"Ganon has invaded Hyrule!\"}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.30434782608695654 + }, + { + "repo": "MinishLab/semhash", + "expected": "[\"zebra\", \"apple\", \"banana\", \"cherry\"]", + "got": "[\"zebra\", \"apple\", \"banana\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8484848484848485 + }, + { + "repo": "MinishLab/semhash", + "expected": "[DuplicateRecord(\"d\", False, [(\"x\", 0.9)])]", + "got": "[\"e\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.20833333333333334 + }, + { + "repo": "MinishLab/semhash", + "expected": "[SelectedWithDuplicates(record=selected, duplicates=[(filtered, 1.0)])]", + "got": "[SelectedWithDuplicates(record=selected, duplicates=[(filtered, 1.0)])]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MinishLab/semhash", + "expected": "[ {\"question\": \"What is the villain's name?\", \"context\": \"The villain is Ganon\", \"answer\": \"Ganon\"} ]", + "got": "[test_records[0]]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.15254237288135594 + }, + { + "repo": "MinishLab/semhash", + "expected": "[ {\"question\": \"What is the hero's name?\", \"context\": \"The hero is Link\", \"answer\": \"Link\"}, {\"question\": \"Who is the princess?\", \"context\": \"The princess is Zelda\", \"answer\": \"Zelda\"}, ]", + "got": "[\"What is the hero's name?\", \"What is the hero's name?\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.40329218106995884 + }, + { + "repo": "MinishLab/semhash", + "expected": "ValueError)", + "got": "ValueError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MinishLab/semhash", + "expected": "ValueError, match=\"has None value\")", + "got": "ValueError, match=\"None\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "MinishLab/semhash", + "expected": "ValueError, match=\"must be a 2D array\")", + "got": "ValueError, match=\"embeddings must be 2D\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7654320987654321 + }, + { + "repo": "MinishLab/semhash", + "expected": "TypeError, match=\"data type: FakeImage\")", + "got": "TypeError, match=\"Failed to encode column 'img'\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5393258426966292 + }, + { + "repo": "MinishLab/semhash", + "expected": "ValueError, match=\"All records must be\")", + "got": "TypeError, match=\"records must be a list of dicts\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7032967032967034 + }, + { + "repo": "MinishLab/semhash", + "expected": "ValueError, match=\"Missing column 'missing'\")", + "got": "ValueError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.39285714285714285 + }, + { + "repo": "MinishLab/semhash", + "expected": "ValueError, match=\"records must not be empty\")", + "got": "ValueError, match=\"must not be empty\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9047619047619048 + }, + { + "repo": "MinishLab/semhash", + "expected": "ValueError, match=\"outlier_percentage must be between 0 and 1\")", + "got": "ValueError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2972972972972973 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "1", + "got": "1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "2", + "got": "2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "3", + "got": "3", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "0", + "got": "1", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "5", + "got": "4", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "d", + "got": "d", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "e", + "got": "e", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "f", + "got": "f", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "g", + "got": "g", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "4", + "got": "2", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "o", + "got": "o", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "p", + "got": "p", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "v", + "got": "v", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "q", + "got": "q", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "a", + "got": "a.array", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.25 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "u", + "got": "u", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "9", + "got": "10", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "8", + "got": "8", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "7", + "got": "7", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "c2", + "got": "c2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "s1", + "got": "None", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "s2", + "got": "s2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "-1", + "got": "0", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "{}", + "got": "{}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "[]", + "got": "[\"my_name\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "db", + "got": "db", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "24", + "got": "24", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "34", + "got": "34", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "35", + "got": "35", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "10", + "got": "10", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "()", + "got": "[]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "30", + "got": "30", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "20", + "got": "21", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "21", + "got": "21", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "99", + "got": "99", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "50", + "got": "50", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "19", + "got": "19", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "12", + "got": "12", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "25", + "got": "25", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "hp", + "got": "hp", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "ab", + "got": "ab", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "f2", + "got": "f2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "p1", + "got": "p1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "d1", + "got": "d1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "60", + "got": "60", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "d2", + "got": "d2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "76", + "got": "283", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "37", + "got": "37", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "92", + "got": "92", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "18", + "got": "18", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "me", + "got": "me", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "11", + "got": "10", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "23", + "got": "23", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "q2", + "got": "q2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "14", + "got": "14", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "15", + "got": "15", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "qs", + "got": "qs", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "ak", + "got": "ak", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "100", + "got": "100", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"b\"", + "got": "\"a\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"a\"", + "got": "\"a\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"v\"", + "got": "\"v\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "[1]", + "got": "[]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "bob", + "got": "bob", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"@\"", + "got": "\"user\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "key", + "got": "key", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "obj", + "got": "obj", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "one", + "got": "one", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "189", + "got": "189", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"u\"", + "got": "\"u\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "NOW", + "got": "NOW", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "doc", + "got": "doc", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"2\"", + "got": "\"l\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"5\"", + "got": "\"5\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "[4]", + "got": "[1, 2, 3]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"Z\"", + "got": "\"Z\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "371", + "got": "100", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "185", + "got": "185", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"1\"", + "got": "\"1\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "999", + "got": "999", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "Foo", + "got": "Foo", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "Bar", + "got": "Bar", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "150", + "got": "150", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"F\"", + "got": "\"F\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "3.6", + "got": "3.6", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "3.5", + "got": "3.5", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "[b]", + "got": "[b]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "[y]", + "got": "[y]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "240", + "got": "135", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "170", + "got": "130", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "960", + "got": "1080", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "249", + "got": "58", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"A\"", + "got": "\"A\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "Q()", + "got": "Q()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "None", + "got": "None", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "1000", + "got": "100", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "2090", + "got": "sum(a.i for a in A.objects.all())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "True", + "got": "True", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "user", + "got": "user", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "PULL", + "got": "\"PULL\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "info", + "got": "info", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"id\"", + "got": "\"_cls\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "tags", + "got": "tags", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"OK\"", + "got": "\"OK\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "obj2", + "got": "obj2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "obj3", + "got": "obj3", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "base", + "got": "base", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "22.0", + "got": "24.0", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.75 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "23.4", + "got": "23.4", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "23.0", + "got": "23.0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "24.0", + "got": "24.0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "keys", + "got": "keys", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "log1", + "got": "log1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "p.id", + "got": "p.id", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"hi\"", + "got": "\"hi\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "bill", + "got": "bill", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "text", + "got": "text", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "data", + "got": "test_file_dupe.the_file.read()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.11764705882352941 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "8313", + "got": "100", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "4971", + "got": "10000", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "doc2", + "got": "doc1", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.75 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "Base", + "got": "\"foo\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"me\"", + "got": "\"me\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "docs", + "got": "docs", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"p2\"", + "got": "\"p1\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.75 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "[me]", + "got": "[me]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "1360", + "got": "1000", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "1.11", + "got": "Decimal(\"1.11\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.42105263157894735 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "2.22", + "got": "2.22", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"[]\"", + "got": "\"\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.13793103448275862 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"db1\"", + "got": "\"db1\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"db2\"", + "got": "\"db2\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "False", + "got": "False", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "[\"a\"]", + "got": "[\"a\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"bla\"", + "got": "\"bla\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "sarah", + "got": "sarah", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "topic", + "got": "topic", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "p.age", + "got": "p.age", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "10000", + "got": "10000", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"_id\"", + "got": "\"username\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.26666666666666666 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"Bob\"", + "got": "\"Bob\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "[\"b\"]", + "got": "[\"b\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"foo\"", + "got": "\"foo\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"Two\"", + "got": "\"Two\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"Bar\"", + "got": "\"Bar\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "a1.tp", + "got": "\"pj\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "MyDoc", + "got": "MyDoc", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "p1.id", + "got": "p1.id", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "p2.id", + "got": "p2.id", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "error", + "got": "error", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"bar\"", + "got": "\"bar\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"PNG\"", + "got": "\"JPEG\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7272727272727273 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "p1.pk", + "got": "p1.pk", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "not q", + "got": "not q", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"you\"", + "got": "\"you\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"joe\"", + "got": "\"joe\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "alice", + "got": "alice", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "[bar]", + "got": "[bar.id]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7692307692307693 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "test2", + "got": "test2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "a1.pk", + "got": "person.id", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.14285714285714285 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "query", + "got": "query", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"test\"", + "got": "\"test\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "(\"A\",)", + "got": "(\"A\",)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "[1, 2]", + "got": "[1, 2]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "[0, 2]", + "got": "[0, 2]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "[0, 1]", + "got": "[0, 1]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "[user]", + "got": "[1]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "[root]", + "got": "[root]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"Fred\"", + "got": "\"Fred\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"1337\"", + "got": "\"1337\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"John\"", + "got": "\"Bob\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5454545454545454 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "person", + "got": "person", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "[\"id\"]", + "got": "[\"id\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"User\"", + "got": "\"User\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"tutu\"", + "got": "\"tutu\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "book_2", + "got": "book_2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"name\"", + "got": "\"name\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "doc.id", + "got": "doc.id", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "actual", + "got": "actual", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "tuples", + "got": "tuples", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "result", + "got": "result", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "post_1", + "got": "post_1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "link_1", + "got": "link_1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "animal", + "got": "animal", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "events", + "got": "events", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"jane\"", + "got": "\"jane\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"Ross\"", + "got": "\"Ross\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"Adam\"", + "got": "\"Adam\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "{0: 1}", + "got": "{0: 1.0}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "{1, 2}", + "got": "{1, 2}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "KeyError)", + "got": "KeyError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "TypeError)", + "got": "TypeError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "ValueError)", + "got": "ImportError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5217391304347826 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "IndexError)", + "got": "IndexError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "InvalidName)", + "got": "InvalidName)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "CustomError)", + "got": "CustomError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "LookUpError)", + "got": "InvalidQueryError) as exc_info:", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.27906976744186046 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "DoesNotExist)", + "got": "MongoEngineError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "NotRegistered)", + "got": "InvalidDocumentError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.17142857142857143 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "{\"level\": \"local\"})", + "got": "{\"level\": \"local\"})", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "fspath(path)", + "got": "fspath(path)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"/__subdir__/filename\"", + "got": "\"filename\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.625 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "isinstance(contents, bytes)", + "got": "self.fs.isfile(path)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2978723404255319 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "0)", + "got": "0)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "1)", + "got": "1)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "3)", + "got": "3)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "2)", + "got": "2)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "4)", + "got": "4)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "5)", + "got": "5)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "[])", + "got": "self.fs.listdir(\"\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.08695652173913043 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"\")", + "got": "\"/\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "10)", + "got": "10)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "12)", + "got": "12)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"w\")", + "got": "\"w\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "text)", + "got": "text)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "None)", + "got": "None)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "b\"f\")", + "got": "\"b\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"gz\")", + "got": "\"gz\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"foo\")", + "got": "\"foo\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"bar\")", + "got": "\"bar\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "data1)", + "got": "data1)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "data2)", + "got": "data2)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "data3)", + "got": "data3)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"baz\")", + "got": "\"baz\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "b\"m3\")", + "got": "b\"m3\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "b\"m2\")", + "got": "b\"m2\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"1.0\")", + "got": "\"1.0\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "[\"/\"])", + "got": "(\"/\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "0o777)", + "got": "0)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "0o755)", + "got": "0o755)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "0o700)", + "got": "0o555)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "0o764)", + "got": "0o755)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "0o760)", + "got": "0o755)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"/foo\")", + "got": "os.path.abspath(\"foo\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "b\"bar\")", + "got": "b\"bar\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "b\"foo\")", + "got": "b\"foo\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "parsed)", + "got": "parsed)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "result)", + "got": "result)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"foo/\")", + "got": "\"foo\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9230769230769231 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"/bar\")", + "got": "\"bar\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9230769230769231 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "[\"./\"])", + "got": "[\"\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "p_copy)", + "got": "p_copy)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "written)", + "got": "written)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"hello\")", + "got": "\"hello\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "[\"foo\"])", + "got": "[\"foo\"])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "cleanup)", + "got": "cleanup)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "[\"baz\"])", + "got": "[\"baz\"])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "contents)", + "got": "contents)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "expected)", + "got": "expected)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"foofoo\")", + "got": "\"foofoo\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"1 byte\")", + "got": "\"1 byte\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"1.0 KB\")", + "got": "\"1.0 KB\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"1.0 MB\")", + "got": "\"1.0 MB\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"1.0 kB\")", + "got": "\"1.0 KB\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "dates[0])", + "got": "dates[0])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "b\"Hello\")", + "got": "b\"Hello\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"foobar\")", + "got": "\"foobar\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"Source\")", + "got": "\"Source\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "mem_fs_2)", + "got": "mem_fs_2)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"fstest\")", + "got": "\"fstest\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "b\"\\0\\1\\2\")", + "got": "\"\\0\\1\\2\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9473684210526315 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "b\"barbar\")", + "got": "b\"barbar\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "all_bytes)", + "got": "all_bytes)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "test_data)", + "got": "test_data)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "b\"foofoo\")", + "got": "b\"foofoo\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "exception)", + "got": "\"resource not found\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3225806451612903 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"oh dear\")", + "got": "\"oh dear\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"0 bytes\")", + "got": "\"0 bytes\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"2 bytes\")", + "got": "\"2 bytes\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "142214400)", + "got": "\"1974-07-05 00:00:00\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3125 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"Content\")", + "got": "\"Content\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"foo/bar\")", + "got": "\"foo/bar\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"\")", + "got": "\"\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"top.txt\")", + "got": "\"top.txt\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "test_bytes)", + "got": "test_bytes)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "bytes_test)", + "got": "bytes_test)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "self.data1)", + "got": "self.data1)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "self.data2)", + "got": "self.data2)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "self.data3)", + "got": "self.data3)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "b\"read me\")", + "got": "b\"read me\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "b\", World\")", + "got": "b\", World\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"test2.txt\")", + "got": "\"test2.txt\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "os.SEEK_CUR)", + "got": "\"current\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.09090909090909091 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "p.u_r)", + "got": "p.is_user)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.625 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "copied)", + "got": "copied)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "no_meta)", + "got": "no_meta)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "f.closed)", + "got": "f.seekable())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.45454545454545453 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "top.is_dir)", + "got": "top.is_dir)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "info.is_dir)", + "got": "info.is_dir)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "f1.isatty())", + "got": "f1.isatty())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "mode.binary)", + "got": "mode.binary)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "info.is_link)", + "got": "info.is_symlink)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.896551724137931 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "isabs(\"foo\"))", + "got": "isabs(\"foo\"))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "m1.isclosed())", + "got": "m1.closed)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "m2.isclosed())", + "got": "m2.isclosed())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "p.check(\"foo\"))", + "got": "p.check(\"u_r\", \"g_w\", \"g_x\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5454545454545454 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "isdotfile(path))", + "got": "isdotfile(path))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "read_file.closed)", + "got": "read_file.closed)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "mem_fs.isclosed())", + "got": "mem_fs.isclosed())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "iswildcard(\"foo\"))", + "got": "iswildcard(\"foo\"))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "matcher(\"foo.PY\"))", + "got": "matcher(\"bar.py\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7222222222222222 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "self.fs.isclosed())", + "got": "foo_fs.isfile(\"foo\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.55 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "self.fs.exists(\"a\"))", + "got": "self.fs.exists(\"egg/foo.pyc\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.76 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "check_readable(\"w\"))", + "got": "check_readable(\"w\"))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "check_readable(\"a\"))", + "got": "check_readable(\"x\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.95 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "self.fs.exists(path))", + "got": "self.fs.exists(path))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "self.fs.isdir(\"foo\"))", + "got": "self.fs.isdir(\"foo.txt\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9130434782608695 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "self.fs.isdir(\"bar\"))", + "got": "self.fs.isdir(\"bar\"))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "self.fs.isempty(\"/\"))", + "got": "self.fs.isempty(\"/\"))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "src_fs.exists(\"foo\"))", + "got": "src_fs.exists(\"/foo\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9767441860465116 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "self.fs.isfile(\"bar\"))", + "got": "self.fs.isfile(\"bar\"))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "self.fs.islink(\"foo\"))", + "got": "self.fs.islink(\"foo\"))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "self.fs.exists(\"foo\"))", + "got": "self.fs.exists(\"foo\"))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "self.fs.hasurl(\"foo\"))", + "got": "self.fs.hasurl(\"foo\"))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "os.path.exists(_path))", + "got": "os.path.exists(_path))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "m2)", + "got": "m2)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "fs)", + "got": "_tar_file)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"/\")", + "got": "\"/\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "text)", + "got": "text)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"bar\")", + "got": "\"bar\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"foo\")", + "got": "\"foo/bar/baz\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"egg\")", + "got": "\"egg\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"abc\")", + "got": "\"abc\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"Foo\")", + "got": "\"Foo\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"fOO\")", + "got": "\"fOO\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "b\"bar\")", + "got": "b\"bar\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"foo1\")", + "got": "\"foo1\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"foo2\")", + "got": "\"foo2\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"spam\")", + "got": "\"spam\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "NoURL)", + "got": "NoURL)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "IOError)", + "got": "errors.ResourceNotFound)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3125 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "args[0])", + "got": "exc.__class__)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2727272727272727 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "FSError)", + "got": "FSError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "OSError)", + "got": "errors.IOError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6086956521739131 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "TypeError)", + "got": "errors.FileExpected)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "ValueError)", + "got": "ValueError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "MountError)", + "got": "MountError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "errors.NoURL)", + "got": "errors.NoURL)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "NotWriteable)", + "got": "NotWriteableError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8387096774193549 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "BulkCopyFailed)", + "got": "BulkCopyFailed)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "ResourceReadOnly)", + "got": "ResourceReadOnly)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "errors.FileExists)", + "got": "errors.FileExists)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "errors.ParseError)", + "got": "errors.NotURL)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5625 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "tarfile.ReadError)", + "got": "tarfile.ReadError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "errors.InvalidPath)", + "got": "errors.PathError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "errors.OpenerError)", + "got": "errors.UserDataError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.75 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "dict)", + "got": "dict)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "bool)", + "got": "bool)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "list)", + "got": "set)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "OSFS)", + "got": "OSFS)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "SubFS)", + "got": "self.fs.__class__)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.08333333333333333 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "bytes)", + "got": "bytes)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "FTPFS)", + "got": "FTPFS)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "MemoryFS)", + "got": "MemoryFS)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "text_type)", + "got": "text_type)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "copied)", + "got": "copied)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "bar.is_dir)", + "got": "bar.is_dir)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "isabs(\"/\"))", + "got": "isabs(\"/\"))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "foo.is_dir)", + "got": "zip_fs.getinfo(\"foo/bar/baz/egg\").isdir())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.37735849056603776 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "ftps_fs.tls)", + "got": "ftps_fs.secure)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7407407407407407 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "info.is_link)", + "got": "info.is_link)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "m1.isclosed())", + "got": "m1.isclosed())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "isabs(\"/foo\"))", + "got": "isabs(\"/a/b\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7857142857142857 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "p.check(\"u_r\"))", + "got": "p.check(\"u_r\"))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "iswildcard(\"*\"))", + "got": "iswildcard(\"*.txt\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "isdotfile(path))", + "got": "isdotfile(path))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "mem_fs.isclosed())", + "got": "mem_fs.exists(\"/\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7027027027027027 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "matcher(\"foo.py\"))", + "got": "matcher(\"foo.py\"))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "self.fs.exists(\"\"))", + "got": "self.fs.exists(\"/\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9743589743589743 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "accessed - now < 5)", + "got": "accessed > now)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8235294117647058 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "self.fs.isclosed())", + "got": "self.fs.isclosed())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "dst_fs.isdir(\"bar\"))", + "got": "dst_fs.isdir(\"foo/empty\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7391304347826086 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "check_readable(\"r\"))", + "got": "check_readable(\"r\"))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "src_fs.isempty(\"/\"))", + "got": "dst_fs.exists(\"test.txt\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5652173913043478 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "fs.isdir(\"foo/bar\"))", + "got": "fs.isfile(\"foo/bar/test.txt\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.72 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "self.fs.isdir(\"foo\"))", + "got": "self.fs.isdir(\"foo\"))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "self.fs.isempty(\"/\"))", + "got": "self.fs.isempty(\"/\"))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "ftp_fs.isfile(\"foo\"))", + "got": "ftp_fs.isfile(\"foo\"))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "self.fs.exists(\"foo\"))", + "got": "self.fs.exists(\"foo\"))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "self.fs.hasurl(\"foo\"))", + "got": "self.fs.hasurl(\"foo\"))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "dst_fs.isdir(\"empty\"))", + "got": "dst_fs.isdir(\"empty\"))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "self.fs.exists(\"egg\"))", + "got": "self.fs.exists(\"egg\"))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "os.path.isdir(fs_dir))", + "got": "os.path.isdir(fs_dir))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"/file1.txt\" in copied)", + "got": "src_fs.exists(\"/file1.txt.copy\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5357142857142857 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "raw_wrapper.seekable())", + "got": "raw_wrapper.seekable())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "p)", + "got": "p.names)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "pp)", + "got": "pp)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "info)", + "got": "info.namespaces)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.47619047619047616 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "foo_info.raw)", + "got": "foo_info)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8181818181818182 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "self.lrucache)", + "got": "self.lrucache)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "0)", + "got": "10000)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "CreateFailed)", + "got": "CreateFailed)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "[])", + "got": "[])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "None)", + "got": "None)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "self.mem_fs)", + "got": "self.mem_fs)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "levels)", + "got": "levels)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "info.name)", + "got": "info.get(\"missing\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4666666666666667 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "p)", + "got": "p)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "self.lrucache)", + "got": "self.lrucache)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"Memory usage didn't increase after creating files; diff is %0.2f KiB.\" % (diff_create.size_diff / 1024.0))", + "got": "\"File memory wasn't freed.\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2814814814814815 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "1)", + "got": "0)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "r)", + "got": "r)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "0)", + "got": "0)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "3)", + "got": "\"3\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "2)", + "got": "2)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "5)", + "got": "3)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "11)", + "got": "11)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "22)", + "got": "22)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "(1,))", + "got": "(1,))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "None)", + "got": "(None,))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7692307692307693 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "r[0])", + "got": "r[0])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "w[1])", + "got": "w[0])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"bar\")", + "got": "\"'bar'\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"foo\")", + "got": "\"foo\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"2.0\")", + "got": "\"2.0\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"FOO\")", + "got": "\"foo\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"utf8\")", + "got": "\"utf8mb4\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8235294117647058 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "cursor)", + "got": "cursor)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"NULL\")", + "got": "None)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"(bar)\")", + "got": "\"['bar']\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"cp1252\")", + "got": "\"latin1\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "other_db)", + "got": "other_db)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "expected)", + "got": "expected)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"foobar\")", + "got": "\"foobar\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "e.args[0])", + "got": "e.args[0])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "(\"row1\",))", + "got": "('row1',))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "list(cur))", + "got": "cur.fetchall())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "self.rows)", + "got": "self.rows)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "current_db)", + "got": "current_db)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"'foobar'\")", + "got": "\"'foobar'\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "list(cur1))", + "got": "cur1.fetchall())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "(\"latin1\",))", + "got": "(\"latin1\",))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"foo\\\\nbar\")", + "got": "\"foo\\\\nbar\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "c.fetchone())", + "got": "c.fetchone())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"'foo''bar'\")", + "got": "\"'foo''bar'\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"'foo\\\\'bar'\")", + "got": "\"'foo''bar'\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "tuple(values))", + "got": "(pymysql.Binary(b\"\\x00\\xff\\x00\"), \"\\xe4\\xf6\\xfc\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.125 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "sorted(result))", + "got": "sorted(result))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "cur.fetchall())", + "got": "list(cur))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "c.fetchone()[0])", + "got": "c.fetchone()[0])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "c.warning_count)", + "got": "len(c._warning_list))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6486486486486487 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "generator(i, j))", + "got": "data[i][j])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.37037037037037035 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "conn.thread_id())", + "got": "None)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "conn.insert_id())", + "got": "c.insert_id())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9032258064516129 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "c2.fetchone()[0])", + "got": "c2.fetchone()[0])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "ER.NO_SUCH_TABLE)", + "got": "ER.NO_SUCH_TABLE)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"Victoria Bitter\")", + "got": "\"Victoria Bitter\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "list(c.fetchall()))", + "got": "c.fetchall())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8125 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "json.loads(json_str))", + "got": "{\"hello\": \"\u3053\u3093\u306b\u3061\u306f\"})", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.15 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"'2012-03-04 05:06:00'\")", + "got": "\"'2012-03-04 05:06:00'\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "c.execute(\"SELECT @foo\"))", + "got": "c.execute(\"select 1\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6382978723404256 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"incorrect data retrieved\")", + "got": "\"cursor.fetchall retrieved incorrect data\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4857142857142857 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "c.execute(\"SET @foo = 'bar'\"))", + "got": "c.execute(\"SELECT @foo := 1\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"list failed via MyDictCursor\")", + "got": "\"MyDictCursor should be iterable\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"affected_rows does not match\")", + "got": "\"Affected rows\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5957446808510638 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"DictCursor should be iterable\")", + "got": "\"fetchall failed via list comprehension\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2191780821917808 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "c.execute(\"SELECT * FROM test\"))", + "got": "c.execute(\"select * from test\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6875 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "self.connection.literal(3.1415))", + "got": "self.connection.literal(3.1415))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"fetchone via DictCursor failed\")", + "got": "\"bob dict cursor returned wrong row\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.45714285714285713 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"fetchall failed via DictCursor\")", + "got": "\"fetchall failed via DictCursor\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "(\"LINESTRING(1.1 1.1,2.2 2.2)\",))", + "got": "(\"LINESTRING(1.1 1.1,2.2 2.2)\",))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"fetchone() returns MyDictCursor\")", + "got": "\"fetchone failed via MyDictCursor\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7536231884057971 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"NULL value not returned as None\")", + "got": "\"cursor.fetchall returned incorrect data\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4473684210526316 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"callproc produced no result set\")", + "got": "\"callproc() should return a tuple\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6086956521739131 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"incorrect data retrieved or inserted\")", + "got": "\"cursor.mixedfetch retrieved incorrect rows\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"cursor.fetchall returned too few rows\")", + "got": "\"select produced invalid result set\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.38961038961038963 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"Should return 0 before we do anything.\")", + "got": "\"affected_rows didn't return 0.\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3783783783783784 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"cursor.fetchone retrieved incorrect data\")", + "got": "\"cursor.fetchone retrieved incorrect data\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"cursor.fetchall retrieved incorrect rows\")", + "got": "\"incorrect data retrieved\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"fetchall returned incorrect number of rows\")", + "got": "\"fetchall returned incorrect number of rows\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"group 3 not blank, bug in RE_INSERT_VALUES?\")", + "got": "\"error parse %s\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.25396825396825395 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"non-empty result set after other result sets\")", + "got": "\"cursor.nextset should return False after \"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3111111111111111 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"cursor.description describes too many columns\")", + "got": "\"cursor.description should have one element\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5591397849462365 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"cursor.description[x][0] must return column name\")", + "got": "\"cursor.description[x][0] must be 'name'\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8387096774193549 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"cursor.description[x] tuples must have 7 elements\")", + "got": "\"cursor.description describes too few columns\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"cursor.rowcount should be -1 after executing no-result statements\")", + "got": "\"rowcount must be -1 after executing a \" \"statement that can return no rows\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6482758620689655 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"cursor.fetchall retrieved incorrect data, or data inserted incorrectly\")", + "got": "\"cursor.fetchall returned invalid results\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.603448275862069 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"cursor.rowcount not being reset to -1 after executing \" \"no-result statements\")", + "got": "\"cursor.rowcount should be -1 after executing no-result statements\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8513513513513513 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "bytearray( b\"insert into bulkinsert (id, name, age, height) values \" b\"(0,'bob',21,123),(1,'jim',56,45),(2,'fred',100,180)\" ))", + "got": "\"INSERT INTO bulkinsert (id, name, age, height) VALUES (%s,%s,%s,%s)\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.46938775510204084 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "bytearray( b\"\"\"insert into bulkinsert (id, name, age, height) values (0, 'bob' , 21, 123 ),(1, 'jim' , 56, 45 ),(2, 'fred' , 100, 180 )\"\"\" ))", + "got": "b\"insert into bulkinsert (id, name, age, height) values (0,'bob',21,123),\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6944444444444444 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "( b\"\\x01\\x02\\x00\\x00\\x00\\x02\\x00\\x00\\x00\" b\"\\x9a\\x99\\x99\\x99\\x99\\x99\\xf1?\" b\"\\x9a\\x99\\x99\\x99\\x99\\x99\\xf1?\" b\"\\x9a\\x99\\x99\\x99\\x99\\x99\\x01@\" b\"\\x9a\\x99\\x99\\x99\\x99\\x99\\x01@\", ))", + "got": "(b\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x0", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.13872832369942195 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "bytearray( b\"\"\"insert into bulkinsert (id, name, age, height) values (0, 'bob' , 21, 123 ),(1, 'jim' , 56, 45 ),(2, 'fred' , 100, 180 ) on duplicate key update age = values(age)\"\"\" ))", + "got": "bytearray(b\"insert\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.20588235294117646 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "l)", + "got": "l)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "c.open)", + "got": "c.open)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "orphans)", + "got": "orphans)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "kill_id in ids)", + "got": "kill_id in ids)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "c.fetchone()[0])", + "got": "c.fetchone()[0])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "cursor.nextset())", + "got": "cursor.nextset())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "bool(cur.nextset()))", + "got": "cur.nextset())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8235294117647058 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "con.get_autocommit())", + "got": "con.autocommit)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "[(1,)])", + "got": "[(\"1\",), (\"2\",), (\"3\",)])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4375 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "w[2])", + "got": "w[2])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "(2006, 2013))", + "got": "\"gone\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.1 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "w[2].lower())", + "got": "w[2])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5555555555555556 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "(r[0], \"mysql_native_password\"))", + "got": "r[0])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2702702702702703 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"expected Python type None or datetime from SQL timestamp\")", + "got": "\"timestamp\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3380281690140845 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "c2.fetchone())", + "got": "c2.fetchone())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "cur.nextset())", + "got": "cur.nextset())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "cur.fetchone())", + "got": "cur.fetchone())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "cur1.nextset())", + "got": "cur1.nextset())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "TypeError)", + "got": "TypeError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "err.OperationalError)", + "got": "err.OperationalError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "pymysql.err.OperationalError)", + "got": "pymysql.err.OperationalError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "t", + "got": "t", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "None", + "got": "None", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "fileno", + "got": "fileno", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "length", + "got": "length", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "utf8mb4", + "got": "utf8mb4", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"utf8mb3\"", + "got": "\"utf8mb3\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"utf8mb4\"", + "got": "\"utf8mb4\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "rfile.closed", + "got": "not sock.closed", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5185185185185185 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "sock._closed", + "got": "not sock.closed", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8148148148148148 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "ssl.CERT_NONE", + "got": "ssl.CERT_NONE", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "cur.nextset()", + "got": "numberofrows[0]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.21428571428571427 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "lowercase_latin1", + "got": "lowercase_latin1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "len(self.samples)", + "got": "6", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "err.OperationalError", + "got": "err.OperationalError", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "isinstance(self.conn.get_host_info(), str)", + "got": "isinstance(self.conn.host_info(), str)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.95 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "r)", + "got": "r)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "conn.open)", + "got": "conn.read_default_group())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3888888888888889 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "len(l) == 1)", + "got": "l)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "cur.nextset())", + "got": "cur.nextset())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "c.fetchone()[0])", + "got": "c.fetchone()[0])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"x\" * size == r)", + "got": "len(r) > size)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4666666666666667 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "con.get_autocommit())", + "got": "con.get_autocommit())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "isinstance(tid, int))", + "got": "isinstance(tid, int))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "con.Error is drv.Error)", + "got": "con.Error is drv.Error)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "cur.rowcount in (-1, 1))", + "got": "cur.rowcount)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7027027027027027 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "cur.rowcount in (-1, 2))", + "got": "cur.rowcount in (-1, 2))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "cur.rowcount in (-1, 0))", + "got": "cur.fetchone())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "cur.rowcount in (-1, 6))", + "got": "cur.rowcount in (-1, 6))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "isinstance(row[0], bytes))", + "got": "row[0].startswith(b\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "hasattr(cur, \"arraysize\"))", + "got": "hasattr(cur, \"arraysize\"))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "con.Warning is drv.Warning)", + "got": "hasattr(con, \"Warning\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.43137254901960786 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "threadsafety in (0, 1, 2, 3))", + "got": "threadsafety in (0, 1, 2))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9454545454545454 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "hasattr(self.driver, \"ROWID\"))", + "got": "hasattr(self.driver, \"ROWID\"))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "hasattr(self.driver, \"STRING\"))", + "got": "hasattr(self.driver, \"STRING\"))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "hasattr(self.driver, \"BINARY\"))", + "got": "hasattr(self.driver, \"BINARY\"))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "hasattr(self.driver, \"NUMBER\"))", + "got": "hasattr(self.driver, \"NUMBER\"))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "hasattr(self.driver, \"DATETIME\"))", + "got": "hasattr(self.driver, \"DATETIME\"))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "pymysql.STRING == pymysql.STRING)", + "got": "set(('a', 'b')) == set(('a', 'b')))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.20588235294117646 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "pymysql.STRING != pymysql.NUMBER)", + "got": "pymysql.STRING != pymysql.NUMBER)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"2\" == self.connection.literal(2))", + "got": "self.cursor.execute(\"SELECT 1\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.42424242424242425 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "FIELD_TYPE.DATE != pymysql.STRING)", + "got": "FIELD_TYPE.VAR_STRING != pymysql.NUMBER)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7297297297297297 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "isinstance(_mysql.__version__, str))", + "got": "isinstance(_mysql.__version__, str))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "isinstance(_mysql.thread_safe(), int))", + "got": "isinstance(_mysql.is_thread_safe(), bool))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.875 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "con.DatabaseError is drv.DatabaseError)", + "got": "con.DatabaseError is drv.DatabaseError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "isinstance(_mysql.version_info, tuple))", + "got": "isinstance(_mysql.__version_info__, tuple))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9512195121951219 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "FIELD_TYPE.VAR_STRING == pymysql.STRING)", + "got": "pymysql.FIELD_TYPE in pymysql.FIELD_TYPE)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5432098765432098 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "issubclass(self.driver.Error, Exception))", + "got": "issubclass(self.driver.Error, Exception))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"'foo'\" == self.connection.literal(\"foo\"))", + "got": "\"abc\" == self.connection.literal(\"abc\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8292682926829268 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "isinstance(_mysql.get_client_info(), str))", + "got": "isinstance(_mysql.client_info, str))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9230769230769231 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "issubclass(self.driver.Warning, Exception))", + "got": "hasattr(self.driver, \"DatabaseError\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.49382716049382713 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "isinstance(self.conn.get_proto_info(), int))", + "got": "isinstance(self.conn.get_proto_info(), int))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "isinstance(self.conn.get_server_info(), str))", + "got": "isinstance(self.conn.get_server_info(), str))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "isinstance(c.fetchone()[0], datetime.datetime))", + "got": "isinstance(c.fetchone()[0], datetime.datetime))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "isinstance(self.conn.character_set_name(), str))", + "got": "isinstance(self.conn.charset_name(), str))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9333333333333333 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "issubclass(self.driver.DatabaseError, self.driver.Error))", + "got": "issubclass(self.driver.DatabaseError, self.driver.Error))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "paramstyle in (\"qmark\", \"numeric\", \"named\", \"format\", \"pyformat\"))", + "got": "paramstyle in (\"qmark\", \"numeric\", \"named\", \"format\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "TypeError)", + "got": "TypeError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "OperationalError)", + "got": "OperationalError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "pymysql.err.Error)", + "got": "pymysql.err.OperationalError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7659574468085106 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "self.driver.Error)", + "got": "self.driver.Error)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "pymysql.OperationalError)", + "got": "pymysql.err.OperationalError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9259259259259259 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "pymysql.ProgrammingError)", + "got": "StopIteration)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "pymysql.err.OperationalError)", + "got": "pymysql.err.InternalError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8363636363636363 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "\"cert\")", + "got": "\"cert\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "m)", + "got": "m)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "0", + "got": "0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "None", + "got": "None", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"cuda\"", + "got": "\"cuda\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "result", + "got": "result", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"__init__.py\"", + "got": "\"__init__.py\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "result.testVerdict", + "got": "result.testVerdict", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "torch.device(\"cpu\")", + "got": "\"cpu\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4166666666666667 + }, + { + "repo": "ROCm/TheRock", + "expected": "amdsmi_test_bin.exists()", + "got": "amdsmi_test_bin.exists()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "torch.cuda.is_available()", + "got": "torch.cuda.is_available()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "torch.allclose(result, expected)", + "got": "torch.allclose(result, expected)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "run_ci)", + "got": "run_ci)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "output)", + "got": "re.match(r\"rocm_sdk version \\d+\\.\\d+\\.\\d+\", output))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.23728813559322035 + }, + { + "repo": "ROCm/TheRock", + "expected": "so_paths)", + "got": "so_paths)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "bd.optional)", + "got": "bd.optional)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "path.exists())", + "got": "path.exists())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "request.flatten)", + "got": "request.flatten)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "hip_file.exists())", + "got": "hip_file.exists())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "bin_path.exists())", + "got": "bin_path.exists())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "dest_file.exists())", + "got": "dest_file.exists())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "flat_orig.exists())", + "got": "os.stat(flat_orig).st_ino == os.stat(flat_link).st_ino)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.43243243243243246 + }, + { + "repo": "ROCm/TheRock", + "expected": "manifest_path.exists())", + "got": "manifest_path.exists())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "api.is_authenticated())", + "got": "api.is_authenticated())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "self.backend.base_path.exists())", + "got": "Path(self.temp_dir) / \"run-test-run-123-linux\").exists()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3409090909090909 + }, + { + "repo": "ROCm/TheRock", + "expected": "self.backend.base_path.is_dir())", + "got": "(self.backend.base_path / \"run-test-run-123-linux\").exists())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6021505376344086 + }, + { + "repo": "ROCm/TheRock", + "expected": "entry.get(\"build_pytorch\", False))", + "got": "entry.get(\"build_pytorch\", False))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "required_keys.issubset(config.keys()))", + "got": "all(key in config for key in required_keys))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.36585365853658536 + }, + { + "repo": "ROCm/TheRock", + "expected": "any(\"missing_dep\" in e for e in errors))", + "got": "any(\"missing_dep\" in e for e in errors))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "any(\"gfx942\" in k for k in fetched_keys))", + "got": "any(\"gfx942\" in k for k in fetched_keys))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "any(\"generic\" in k for k in fetched_keys))", + "got": "\"test-artifact_lib_generic.tar.zst\" in fetched_keys)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5531914893617021 + }, + { + "repo": "ROCm/TheRock", + "expected": "any(\"missing_group\" in e for e in errors))", + "got": "any(\"missing_group\" in error for error in errors))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9130434782608695 + }, + { + "repo": "ROCm/TheRock", + "expected": "all(\"amdgpu_family\" in t for t in targets))", + "got": "not targets)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.43636363636363634 + }, + { + "repo": "ROCm/TheRock", + "expected": "self.backend.artifact_exists(\"exists.tar.xz\"))", + "got": "self.backend.artifact_exists(\"exists.tar.xz\"))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "self.backend.artifact_exists(\"exists.tar.zst\"))", + "got": "self.backend.artifact_exists(\"exists.tar.zst\"))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "any(\"gfx94X-dcgpu\" in k for k in fetched_keys))", + "got": "any(\"gfx942\" in k for k in fetched_keys))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "ROCm/TheRock", + "expected": "any(\"Circular dependency\" in e for e in errors))", + "got": "any(\"circular dependency\" in e for e in errors))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9791666666666666 + }, + { + "repo": "ROCm/TheRock", + "expected": "configure_ci.determine_long_lived_branch(branch))", + "got": "configure_ci.determine_long_lived_branch(branch))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "all(\"family\" in entry for entry in target_output))", + "got": "all(\"test-runs-on\" in entry for entry in target_output))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8301886792452831 + }, + { + "repo": "ROCm/TheRock", + "expected": "self.backend.artifact_exists(\"test_artifact.tar.xz\"))", + "got": "(self.backend.base_path / \"test_artifact.tar.xz\").exists())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7142857142857143 + }, + { + "repo": "ROCm/TheRock", + "expected": "self.backend.artifact_exists(\"test_artifact.tar.zst\"))", + "got": "self.backend.artifact_exists(\"test_artifact.tar.zst\"))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "any(\"rocblas\" == entry for entry in linux_test_labels))", + "got": "any(\"rocblas\" == entry for entry in linux_test_labels))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "run_ci)", + "got": "run_ci)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "bd.optional)", + "got": "bd.predicate.force_excludes)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "ROCm/TheRock", + "expected": "sanity_check)", + "got": "sanity_check)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "call_args.flatten)", + "got": "call_args.flatten)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "api.is_authenticated())", + "got": "api.is_authenticated())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "entry.get(\"build_pytorch\", False))", + "got": "entry.get(\"build_pytorch\", False))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "any(\"gfx1100\" in k for k in fetched_keys))", + "got": "any(\"gfx1100\" in k for k in fetched_keys))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "any(\"--index-url\" in str(a) for a in cmd))", + "got": "\"--index-url\" in cmd)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "ROCm/TheRock", + "expected": "configure_ci.determine_long_lived_branch(branch))", + "got": "configure_ci.determine_long_lived_branch(branch))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "any(\"family\" in entry for entry in target_output))", + "got": "any(\"family\" in entry for entry in target_output))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "self.backend.artifact_exists(\"nonexistent.tar.xz\"))", + "got": "self.backend.artifact_exists(\"nonexistent.tar.xz\"))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "self.backend.artifact_exists(\"nonexistent.tar.zst\"))", + "got": "self.backend.artifact_exists(\"nonexistent.tar.zst\"))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "run)", + "got": "run)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "cmd)", + "got": "\" \".join(cmd[4:]))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "ROCm/TheRock", + "expected": "data)", + "got": "data)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "graph)", + "got": "graph)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "output)", + "got": "output)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "result)", + "got": "result)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "version)", + "got": "version)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "produced)", + "got": "[a.name for a in produced])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "ROCm/TheRock", + "expected": "filtered)", + "got": "filtered)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "artifacts)", + "got": "artifacts)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "lib_files)", + "got": "lib_files)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "test_names)", + "got": "test_names)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "family_info)", + "got": "family_info)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "family_dict)", + "got": "family_dict)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "call_kwargs)", + "got": "call_kwargs)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "target_names)", + "got": "target_names)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "workflow_run)", + "got": "workflow_run)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "libs_inbound)", + "got": "libs_inbound)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "info.s3_path)", + "got": "info.s3_bucket)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7142857142857143 + }, + { + "repo": "ROCm/TheRock", + "expected": "matched_files)", + "got": "matched_files)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "stage2_inbound)", + "got": "stage2_inbound)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "self.gha_output)", + "got": "self.gha_output)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "data)", + "got": "data)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "names)", + "got": "names)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "output)", + "got": "output)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "result)", + "got": "result)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "produced)", + "got": "produced)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "filtered)", + "got": "filtered)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "lib_files)", + "got": "lib_files)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "0)", + "got": "0)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "1)", + "got": "2)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "ROCm/TheRock", + "expected": "6)", + "got": "3)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "ROCm/TheRock", + "expected": "3)", + "got": "3)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "2)", + "got": "2)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "[])", + "got": "\"gfx94X-dgpu\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.11764705882352941 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"\")", + "got": "\"windows-gfx120X-all-gpu-rocm\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.17647058823529413 + }, + { + "repo": "ROCm/TheRock", + "expected": "[1])", + "got": "[1])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "an2)", + "got": "ArtifactName.from_path(p1))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.12903225806451613 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"-m\")", + "got": "\"-m\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"uv\")", + "got": "\"python\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.42857142857142855 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"pip\")", + "got": "\"pip\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"3.11\")", + "got": "\"3.11\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"3.12\")", + "got": "\"3.12\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"main\")", + "got": "\"main\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"name\")", + "got": "\"name_component\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5833333333333334 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"base\")", + "got": "\"base\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"linux\")", + "got": "\"linux\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"8.1.0\")", + "got": "\"8.1.0\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"7.9.0\")", + "got": "\"7.9.0\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "matches)", + "got": "matches)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "mock_cwd)", + "got": "\"/\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.15384615384615385 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"bench1\")", + "got": "\"bench1\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "expected)", + "got": "expected)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"header\")", + "got": "\"header\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"hipblas\")", + "got": "\"hipblas\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"nightly\")", + "got": "\"nightly\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"generic\")", + "got": "\"tar\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"1000abc\")", + "got": "\"1000\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8235294117647058 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"windows\")", + "got": "\"windows\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"test-key\")", + "got": "\"test-key\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "[\"**/def\"])", + "got": "[\"**/def\"])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"per-arch\")", + "got": "\"per-arch\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "[\"group1\"])", + "got": "[\"group1\"])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "mock_module)", + "got": "mock_module)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"s3-run-id\")", + "got": "\"s3-run-id\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"test-token\")", + "got": "\"test-token\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"README.txt\")", + "got": "\"README.txt\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"release-2.7\")", + "got": "\"release\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "ROCm/TheRock", + "expected": "second_result)", + "got": "second_result)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"cp312-cp312\")", + "got": "\"3.12\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"__init__.py\")", + "got": "\"rocm_sdk\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.32 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"test-secret\")", + "got": "\"test-secret\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"test-bucket\")", + "got": "\"test-bucket\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"gfx94X-dcgpu\")", + "got": "\"gfx94X\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.75 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"rocm-systems\")", + "got": "\"rocm-systems\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"ROCm/TheRock\")", + "got": "\"ROCm/TheRock\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"ROCm-TheRock/\")", + "got": "\"ROCm/TheRock\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9032258064516129 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"cp313t-cp313t\")", + "got": "\"cp313\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"8.0.0~custom1\")", + "got": "\"8.0.0~custom1\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"rocm-libraries\")", + "got": "\"rocm\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5833333333333334 + }, + { + "repo": "ROCm/TheRock", + "expected": "expected_includes)", + "got": "expected_includes)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "[1, 2, 3, 4, 5, 6])", + "got": "[0, 1, 2, 3, 4, 5])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8421052631578947 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"linux-mi300-mgpu\")", + "got": "\"true\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"therock-artifacts\")", + "got": "\"18022609292\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.17647058823529413 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"users-alice-feature\")", + "got": "\"users/alice/feature\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "ROCm/TheRock", + "expected": "{\"rocblas\", \"hipblas\"})", + "got": "{\"hipblas\", \"rocblas\"})", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5217391304347826 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"therock-ci-artifacts\")", + "got": "\"therock-artifacts\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9302325581395349 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"ROCm-rocm-libraries/\")", + "got": "\"ROCm/\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5161290322580645 + }, + { + "repo": "ROCm/TheRock", + "expected": "core_path.parent.parent)", + "got": "core_path.parent)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8292682926829268 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"hardlink test content\")", + "got": "\"hardlink test content\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "devel_path.parent.parent)", + "got": "devel_path.parent)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8372093023255814 + }, + { + "repo": "ROCm/TheRock", + "expected": "TEST_THEROCK_FORK_RUN_ID)", + "got": "\"\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.07142857142857142 + }, + { + "repo": "ROCm/TheRock", + "expected": "os.stat(link_file).st_ino)", + "got": "os.stat(link_file).st_ino)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "TEST_THEROCK_COMMIT_NEWER)", + "got": "TEST_THEROCK_COMMIT_NEWER)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "TEST_THEROCK_COMMIT_OLDER)", + "got": "TEST_THEROCK_COMMIT_OLDER)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"windows-gfx1151-gpu-rocm\")", + "got": "\"windows-mi325-1gpu-ossci-rocm\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.711864406779661 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"test artifact content xz\")", + "got": "\"test artifact content xz\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"--no-include-rocm-systems\")", + "got": "\"--no-include-rocm-systems\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "{\"test1\", \"test2\", \"test3\"})", + "got": "{\"test1\", \"test2\", \"test3\"})", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "\"therock-nightly-artifacts\")", + "got": "\"therock-ci-artifacts\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8627450980392157 + }, + { + "repo": "ROCm/TheRock", + "expected": "GitHubAPI.AuthMethod.GH_CLI)", + "got": "GitHubAPI.AuthMethod.GH_CLI)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "2)", + "got": "2)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "1)", + "got": "2)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "ROCm/TheRock", + "expected": "Exception)", + "got": "GitHubAPIError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.24 + }, + { + "repo": "ROCm/TheRock", + "expected": "TypeError)", + "got": "ValueError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "ROCm/TheRock", + "expected": "ValueError)", + "got": "ValueError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "SystemExit)", + "got": "SystemExit)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "info)", + "got": "info)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "result)", + "got": "result)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "mod)", + "got": "mod)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "rocm_sdk)", + "got": "\"rocm_sdk\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9 + }, + { + "repo": "ROCm/TheRock", + "expected": "core_mod)", + "got": "core_mod)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "mock_module)", + "got": "mock_module)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "libraries_mod)", + "got": "libraries_mod)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "str(dest_path))", + "got": "dest_path)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "ROCm/TheRock", + "expected": "info)", + "got": "info)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "result)", + "got": "result)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "an_invalid1)", + "got": "an_invalid1)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "1)", + "got": "1)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "0)", + "got": "0)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "version_dev)", + "got": "version_dev)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "version_alpha)", + "got": "version_alpha)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "hipblaslt_windows[\"total_shards\"])", + "got": "hipblaslt_windows[\"total_shards\"])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "str)", + "got": "str)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "list)", + "got": "list)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "OSError)", + "got": "OSError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "URLError)", + "got": "URLError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "HTTPError)", + "got": "HTTPError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "TimeoutError)", + "got": "TimeoutError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "ArtifactRunInfo)", + "got": "ArtifactRunInfo)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "GitHubAPI.AuthMethod)", + "got": "GitHubAPI.AuthMethod)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "5)", + "got": "5)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "ValueError, \"illegal key: 'foobar'\")", + "got": "ValueError, \"must be a string\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5671641791044776 + }, + { + "repo": "ROCm/TheRock", + "expected": "ValueError, \"Unmatched artifact files\")", + "got": "ValueError, \"not/default/dir\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6086956521739131 + }, + { + "repo": "ROCm/TheRock", + "expected": "ValueError, \"artifact_name is required\")", + "got": "ValueError, \"artifact_name\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8235294117647058 + }, + { + "repo": "ROCm/TheRock", + "expected": "RuntimeError, \"The program was compiled against a ROCm version matching\")", + "got": "ValueError, \"Version mismatch\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "ROCm/TheRock", + "expected": "[\"lib\"])", + "got": "[\"default\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6 + }, + { + "repo": "ROCm/TheRock", + "expected": "[\"extras\"])", + "got": "[\"extras\"])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "set())", + "got": "set())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "set([\"lib/libfoo.so.1\"]))", + "got": "set([\"lib\", \"bin\"]))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6222222222222222 + }, + { + "repo": "ROCm/TheRock", + "expected": "runtime_idx)", + "got": "runtime_idx)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "r\"^[0-9]+[0-9\\.]*abc$\")", + "got": "r\"^[0-9]+[0-9\\.]*abcd$\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9787234042553191 + }, + { + "repo": "ROCm/TheRock", + "expected": "r\"^[0-9]+[0-9\\.]*rc.*$\")", + "got": "r\"^[0-9]+[0-9\\.]*rc.*$\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "r\"^[0-9]+[0-9\\.]*~rc.*$\")", + "got": "r\"^[0-9]+[0-9\\.]*~rc.*$\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "r\"^[0-9]+[0-9\\.]*~pre.*$\")", + "got": "r\"^[0-9]+[0-9\\.]*~pre.*$\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "r\"^[0-9]+[0-9\\.]*a[0-9]{8}$\")", + "got": "r\"^[0-9]+[0-9\\.]*\\.a[0-9]{8}$\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9666666666666667 + }, + { + "repo": "ROCm/TheRock", + "expected": "r\"^[0-9]+[0-9\\.]*~[0-9]{8}$\")", + "got": "r\"^[0-9]+[0-9\\.]*~[0-9]{8}$\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ROCm/TheRock", + "expected": "r\"^[0-9]+[0-9\\.]*~dev[0-9]{8}$\")", + "got": "r\"^[0-9]+[0-9\\.]*~dev[0-9]{8}$\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "adamchainz/django-upgrade", + "expected": "2", + "got": "1", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "adamchainz/django-upgrade", + "expected": "0", + "got": "0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "adamchainz/django-upgrade", + "expected": "1", + "got": "1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "adamchainz/django-upgrade", + "expected": "\"\"", + "got": "\"\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "adamchainz/django-upgrade", + "expected": "None", + "got": "None", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "adamchainz/django-upgrade", + "expected": "after", + "got": "after", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "adamchainz/django-upgrade", + "expected": "(2, 2)", + "got": "(3, 0)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "adamchainz/django-upgrade", + "expected": "DEDENT", + "got": "DEDENT", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "adamchainz/django-upgrade", + "expected": "source", + "got": "source", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "adamchainz/django-upgrade", + "expected": "expected", + "got": "expected", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "adamchainz/django-upgrade", + "expected": "not invalid", + "got": "not invalid", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "adamchainz/django-upgrade", + "expected": "UNIMPORTANT_WS", + "got": "\"DEDENT\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.18181818181818182 + }, + { + "repo": "adamchainz/django-upgrade", + "expected": "dedented_after", + "got": "dedented_after", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "adamchainz/django-upgrade", + "expected": "'print(\"hi\")\\n'", + "got": "\"no changes\\n\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.27586206896551724 + }, + { + "repo": "adamchainz/django-upgrade", + "expected": "not undocumented", + "got": "not undocumented", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "adamchainz/django-upgrade", + "expected": "initial_contents", + "got": "initial_contents", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "adamchainz/django-upgrade", + "expected": "dedented_contents", + "got": "dedented_contents", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "adamchainz/django-upgrade", + "expected": "f\"Rewriting {path}\\n\"", + "got": "\"example.py:1:0: F401 'QuerySetPaginator' imported but unused\\n\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.25882352941176473 + }, + { + "repo": "adamchainz/django-upgrade", + "expected": "\"Would rewrite stdin\\n\"", + "got": "\"Would rewrite \\n\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9583333333333334 + }, + { + "repo": "adamchainz/django-upgrade", + "expected": "isinstance(operation, Delete)", + "got": "isinstance(operation, Delete)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "adamchainz/django-upgrade", + "expected": "isinstance(node, ast.ImportFrom)", + "got": "isinstance(node, ast.ImportFrom)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "adamchainz/django-upgrade", + "expected": "f\"{path} is non-utf-8 (not supported)\\n\"", + "got": "\"error: file contains non-UTF-8 bytes\\n\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.425 + }, + { + "repo": "adamchainz/django-upgrade", + "expected": "make_state(filename).looks_like_test_file", + "got": "make_state(filename).looks_like_test_file", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "adamchainz/django-upgrade", + "expected": "make_state(filename).looks_like_admin_file", + "got": "looks_like_admin_file(filename)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5753424657534246 + }, + { + "repo": "adamchainz/django-upgrade", + "expected": "make_state(filename).looks_like_models_file", + "got": "make_state(filename).looks_like_models_file", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "adamchainz/django-upgrade", + "expected": "make_state(filename).looks_like_command_file", + "got": "make_state(filename).looks_like_command_file", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "adamchainz/django-upgrade", + "expected": "make_state(filename).looks_like_settings_file", + "got": "make_state(filename).looks_like_settings_file", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "adamchainz/django-upgrade", + "expected": "not make_state(filename).looks_like_test_file", + "got": "not make_state(filename).looks_like_test_file", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "adamchainz/django-upgrade", + "expected": "not make_state(filename).looks_like_admin_file", + "got": "not make_state(filename).looks_like_admin_file", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "adamchainz/django-upgrade", + "expected": "make_state(filename).looks_like_migrations_file", + "got": "make_state(filename).looks_like_migrations_file", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "adamchainz/django-upgrade", + "expected": "not make_state(filename).looks_like_models_file", + "got": "not make_state(filename).looks_like_models_file", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "adamchainz/django-upgrade", + "expected": "\"from django.core.paginator import Paginator\\n\"", + "got": "\"from django.core.paginator import QuerySetPaginator\\n\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9215686274509803 + }, + { + "repo": "adamchainz/django-upgrade", + "expected": "not make_state(filename).looks_like_command_file", + "got": "not make_state(filename).looks_like_command_file", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "adamchainz/django-upgrade", + "expected": "make_state(filename).looks_like_dunder_init_file", + "got": "make_state(filename).looks_like_dunder_init_file", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "adamchainz/django-upgrade", + "expected": "not make_state(filename).looks_like_settings_file", + "got": "not make_state(filename).looks_like_settings_file", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "adamchainz/django-upgrade", + "expected": "proc.stdout.startswith(b\"usage: django-upgrade \")", + "got": "proc.stdout.startswith(\"Usage: \")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7804878048780488 + }, + { + "repo": "adamchainz/django-upgrade", + "expected": "not make_state(filename).looks_like_migrations_file", + "got": "not make_state(filename).looks_like_migrations_file", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "adamchainz/django-upgrade", + "expected": "not make_state(filename).looks_like_dunder_init_file", + "got": "not make_state(filename).looks_like_dunder_init_file", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "adamchainz/django-upgrade", + "expected": "f\"Detected Django version from pyproject.toml: {expected[0]}.{expected[1]}\\n\"", + "got": "\"\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.05063291139240506 + }, + { + "repo": "adamchainz/django-upgrade", + "expected": "SystemExit)", + "got": "SystemExit)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "2", + "got": "ret", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "1", + "got": "1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "r", + "got": "r", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "0", + "got": "0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "3", + "got": "3", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "t", + "got": "t", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "7", + "got": "3", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "6", + "got": "10", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "42", + "got": "(42,)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "10", + "got": "10", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "11", + "got": "11", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "22", + "got": "22", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "ret", + "got": "ret", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "msg", + "got": "msg", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "cnt", + "got": "cnt", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "ids", + "got": "ids", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "res", + "got": "res", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "val", + "got": "val", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "ops", + "got": "ops", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "row", + "got": "row", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "100", + "got": "100", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "r[0]", + "got": "r[0]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "True", + "got": "True", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "None", + "got": "None", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "conn", + "got": "conn", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "(1,)", + "got": "(1,)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "1049", + "got": "1049", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "loop", + "got": "loop", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "rows", + "got": "rows", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "res1", + "got": "res1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "res2", + "got": "res2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "(42,)", + "got": "(42,)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "value", + "got": "value", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "r[10]", + "got": "r[9]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "r[-1]", + "got": "r[-1]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "False", + "got": "False", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "conn1", + "got": "conn2", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "'boom'", + "got": "'boom'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "'utf8'", + "got": "'utf8'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "'root'", + "got": "'root'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "length", + "got": "length", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "values", + "got": "values", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "row[0]", + "got": "row['id']", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "row.id", + "got": "len(row)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.42857142857142855 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "number", + "got": "number", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "charset", + "got": "'utf8mb4'", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.125 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "list(r)", + "got": "r", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.25 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "'mysql'", + "got": "'mysql'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "resp[0]", + "got": "resp[0]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "maxsize", + "got": "maxsize", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "row.val", + "got": "row['val']", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7058823529411765 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "rows[0]", + "got": "rows[0]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "rows[1]", + "got": "rows[1]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "enabled", + "got": "False", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "not resp", + "got": "not resp", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "expected", + "got": "expected", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "other_db", + "got": "other_db", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "not r[0]", + "got": "not r[0]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "list(r1)", + "got": "list(r1)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "[(2, 0)]", + "got": "[(1, 0)]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.875 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "conn.user", + "got": "conn.user", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "conn.echo", + "got": "False", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.14285714285714285 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "e.args[0]", + "got": "e.errno", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.375 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "pool.size", + "got": "len(pool)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "pool.echo", + "got": "pool._echo", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9473684210526315 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "len(DATA)", + "got": "18", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "len(rows)", + "got": "len(rows)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "current_db", + "got": "current_db", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "c.rowcount", + "got": "await c.fetchone()[0]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.1935483870967742 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "pool._used", + "got": "pool._free", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "(big_str,)", + "got": "(big_str,)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "len(value)", + "got": "value", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "res.keys()", + "got": "res.keys()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "rows[1].id", + "got": "rows[0][0]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "res.closed", + "got": "not res.closed", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "cur.closed", + "got": "0 == len(conn._weak_results)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.10526315789473684 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "len(rows2)", + "got": "len(rows2)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "conn.closed", + "got": "isinstance(w[-2].message, DeprecationWarning)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.14285714285714285 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "minfreesize", + "got": "minfreesize", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "pool.closed", + "got": "not pool._used", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.64 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "engine.name", + "got": "engine.name", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "engine.size", + "got": "engine.size", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "tr.is_active", + "got": "not tr.closed", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "\"'foo''bar'\"", + "got": "\"'foo''bar'\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "(2006, 2013)", + "got": "(2006, 2013)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "cur.rowcount", + "got": "await cur.rowcount", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "pool.minsize", + "got": "len(pool)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.38095238095238093 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "pool.maxsize", + "got": "pool.maxsize", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "pool.timeout", + "got": "pool.timeout", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "conn.timeout", + "got": "conn.timeout", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "res.rowcount", + "got": "res.lastrowid", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.56 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "len(set(ids))", + "got": "len(set(ids))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "cursor.closed", + "got": "cursor.closed", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "tr2.is_active", + "got": "tr2.is_active", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "\"'foo\\\\'bar'\"", + "got": "\"foo''bar\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.782608695652174 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "cur.arraysize", + "got": "cur.arraysize", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "cur.rownumber", + "got": "cur.rownumber", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "pool.freesize", + "got": "pool.freesize", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "res.lastrowid", + "got": "await (await conn.execute(tbl.select())).count()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.09836065573770492 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "engine.driver", + "got": "engine.driver", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "tr1.is_active", + "got": "tr1.is_active", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "'sa_twophase'", + "got": "'xid1'", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.21052631578947367 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "sorted(result)", + "got": "sorted(result)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "not cur.closed", + "got": "cur.closed", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "not res.closed", + "got": "res.closed", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "engine.dialect", + "got": "\"mysql\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.09523809523809523 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "engine.minsize", + "got": "engine.minsize", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "engine.maxsize", + "got": "engine.maxsize", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "not conn.closed", + "got": "pool.size", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4166666666666667 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "not task.done()", + "got": "task.done()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8461538461538461 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "engine.freesize", + "got": "engine.freesize", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "not tr.is_active", + "got": "not tr.is_active", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "transaction_flag", + "got": "transaction_flag", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "conn.insert_id()", + "got": "await c.insert_id()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7428571428571429 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "[\"value\", \"xxx\"]", + "got": "[\"value\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.72 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "not cursor.closed", + "got": "isinstance(cursor, Cursor)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4186046511627907 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "connection.closed", + "got": "connection.closed", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "\"hello == world!\"", + "got": "\"hello, world!\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.875 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "'sha256_password'", + "got": "'caching_sha2_password'", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.75 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "not tr2.is_active", + "got": "not tr2.is_active", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "mysql_params['db']", + "got": "mysql_params['db']", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "str(w[-1].message)", + "got": "w[0].message.args[0]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5789473684210527 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "not transaction_flag", + "got": "not transaction_flag", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "json.loads(json_str)", + "got": "{\"hello\": \"\u3053\u3093\u306b\u3061\u306f\"}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.10526315789473684 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "mysql_params['host']", + "got": "mysql_params[\"host\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "conn.get_host_info()", + "got": "conn.unix_socket", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3888888888888889 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "mysql_params['user']", + "got": "mysql_params['user']", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "not connection.closed", + "got": "not connection.closed", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "('latin1', 'utf8mb4')", + "got": "mysql_params['character_set_server']", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.07017543859649122 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "'mysql_clear_password'", + "got": "'clear_password'", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8421052631578947 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "UserDefinedEnum.Value1", + "got": "111", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.08 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "'caching_sha2_password'", + "got": "'caching_sha2_password'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "len(conn._weak_results)", + "got": "len(gc.get_objects())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5454545454545454 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "len(set(mysql_addresses))", + "got": "len(ids)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.42424242424242425 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "conn.character_set_name()", + "got": "conn.set_charset('utf8')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5306122448979592 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "isinstance(cur, MyCursor)", + "got": "isinstance(cur, MyCursor)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "len(await res.fetchall())", + "got": "len(await res.fetchall())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "value[1].startswith('TLS')", + "got": "value[1]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.47058823529411764 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "isinstance(conn, Connection)", + "got": "isinstance(conn, Connection)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "await c.execute(\"SELECT @foo\")", + "got": "await c.execute(\"select 1\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7017543859649122 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "result['Value'].startswith('TLS')", + "got": "result[\"Value\"].startswith(\"TLS\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8787878787878788 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "await c.execute(\"SET @foo = 'bar'\")", + "got": "(await c.execute(\"SELECT @bar\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8059701492537313 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "\"Unknown database 'does_not_exist'\"", + "got": "\"Unknown database 'does_not_exist'\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "isinstance(r[0], datetime.datetime)", + "got": "type(r[0]) in (type(None), datetime.datetime)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.675 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "sorted(result, key=lambda k: k['id'])", + "got": "sorted(result)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5490196078431373 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "cur._last_executed.endswith(b\"(3, 4),(5, 6)\")", + "got": "RE_INSERT_VALUES.match(q) is None", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.20512820512820512 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "issubclass(w[-1].category, DeprecationWarning)", + "got": "isinstance(w[0].message, DeprecationWarning)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7111111111111111 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "( 'mysql_native_password', 'caching_sha2_password')", + "got": "('mysql_clear_password', 'mysql_native_password')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.52 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "bytearray( b\"REPLACE INTO bulkinsert (id, name, age, height) \" + b\"VALUES (0,'bob',21,123),(0,'jim',56,45)\" )", + "got": "bytearray(b\"REPLACE INTO bulkinsert (id, name, age, height) \" + b\"VALUES (0,'bob',21,123),(0,'jim',56,45)\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9907407407407407 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "bytearray(b\"\"\"insert into bulkinsert (id, name, age, height) values (0, 'bob' , 21, 123 ),(1, 'jim' , 56, 45 ),(2, 'fred' , 100, 180 )\"\"\")", + "got": "\"insert into bulkinsert (id, name, age, height) values (0,'bob',21,123), (1,'jim',56,45), (2,'fred',100,180)\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8663967611336032 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "bytearray(b\"\"\"insert into bulkinsert (id, name, age, height) values (0, 'bob' , 21, 123 ),(1, 'jim' , 56, 45 ),(2, 'fred' , 100, 180 ) on duplicate key update age = values(age)\"\"\")", + "got": "bytearray(b\"\"\"insert", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "TypeError)", + "got": "TypeError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "IndexError)", + "got": "IndexError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "ValueError)", + "got": "ValueError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "RuntimeError)", + "got": "ConnectionError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6206896551724138 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "InterfaceError)", + "got": "ProgrammingError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "AttributeError)", + "got": "KeyError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5833333333333334 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "sa.ArgumentError)", + "got": "TypeError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5185185185185185 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "ProgrammingError)", + "got": "ValueError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "OperationalError)", + "got": "OperationalError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "NotSupportedError)", + "got": "IndexError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4827586206896552 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "asyncio.TimeoutError)", + "got": "RuntimeError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5882352941176471 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "sa.ResourceClosedError)", + "got": "sa.TableExistsError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5116279069767442 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "sa.InvalidRequestError)", + "got": "RuntimeError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "aiomysql.OperationalError)", + "got": "aiomysql.OperationalError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "RuntimeError, match=\"Received LOAD_LOCAL packet but local_infile option is false\")", + "got": "OperationalError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.16161616161616163 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "data)", + "got": "data)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "[(0, \"jim\", 56, 45)])", + "got": "data)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.07692307692307693 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "[])", + "got": "(None, None))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.125 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "[[\"arg\"]])", + "got": "[(\"arg\",)])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7619047619047619 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "[{\"foo\": \"bar\"}])", + "got": "[\"foo\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.64 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "[[\"foo\", \"bar\"]])", + "got": "[[\"foo\", \"bar\"]])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "[('foo', 'bar')])", + "got": "[(\"foo\", \"bar\")])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7647058823529411 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "[(\"arg\", \"arg\")])", + "got": "[(\"arg\", \"arg\")])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "[('foo', 'bar'), ('bat', 'hoho')])", + "got": "[[\"foo\", \"bar\"], [\"bat\", \"hoho\"]])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6470588235294118 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "[{'foo': 'bar'}, {'foo': 'hoho'}])", + "got": "[{\"foo\": \"bar\"}, {\"foo\": \"hoho\"}])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7647058823529411 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "([('foo', 'bar')], [('bar', 'bat')]))", + "got": "[('foo', 'bar'), ('bar', 'bat')])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9428571428571428 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "3", + "got": "2", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "1", + "got": "3", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "2", + "got": "3", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "0", + "got": "0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "200", + "got": "429", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "429", + "got": "429", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "\"1\"", + "got": "\"42\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "\"0\"", + "got": "\"0\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "400", + "got": "400", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "\"3\"", + "got": "\"2\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "\"9\"", + "got": "\"9\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "\"2\"", + "got": "\"5\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "\"5\"", + "got": "\"4\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "1000", + "got": "0", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "\"60\"", + "got": "\"60\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "\"10\"", + "got": "\"10\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "b\"uno\"", + "got": "\"uno\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "b\"dos\"", + "got": "b\"dos\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "\"INFO\"", + "got": "\"WARNING\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5333333333333333 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "b\"tres\"", + "got": "b\"tres\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "str(60)", + "got": "\"60\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "str(int(50))", + "got": "str(50)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7368421052631579 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "result.output", + "got": "result.output", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "response.text", + "got": "response.data", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7692307692307693 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "expected_body", + "got": "expected_body", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "fn_a.call_count", + "got": "fn_a.call_count", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "fn_b.call_count", + "got": "fn_b.call_count", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "expected_status", + "got": "expected_status", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "resp.status_code", + "got": "resp.status_code", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "not caplog.records", + "got": "not caplog.records", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "response.status_code", + "got": "response.status_code", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "caplog.records[0].msg", + "got": "caplog.text", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "cli.get(\"/\").status_code", + "got": "cli.get(\"/\").status_code", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "cli.put(\"/\").status_code", + "got": "cli.put(\"/\").status_code", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "cli.post(\"/\").status_code", + "got": "cli.post(\"/\").status_code", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "cli.get(\"/a\").status_code", + "got": "cli.get(\"/a\").status_code", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "cli.get(\"/b\").status_code", + "got": "cli.get(\"/b\").status_code", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "cli.get(\"/t1\").status_code", + "got": "cli.get(\"/t1\").status_code", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "cli.get(\"/t3\").status_code", + "got": "cli.get(\"/t2\").status_code", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9615384615384616 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "cli.get(\"/t2\").status_code", + "got": "cli.get(\"/t2\").status_code", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "cli.get(\"/\").data.decode()", + "got": "cli.get(\"/\").data.decode(\"utf-8\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8813559322033898 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "str(int(time.time() + 50))", + "got": "str(int(time.time() + 50))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "cli.post(\"/t1\").status_code", + "got": "cli.post(\"/t1\").status_code", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "cli.get(\"/t1/1\").status_code", + "got": "cli.get(\"/t1/1\").status_code", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "cli.get(\"/t1/2\").status_code", + "got": "cli.get(\"/t1/2\").status_code", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "cli.get(\"/root\").status_code", + "got": "cli.get(\"/root\").status_code", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "cli.get(\"/test/a\").status_code", + "got": "cli.get(\"/test/a\").status_code", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "str(math.ceil(time.time() + 1))", + "got": "\"0\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "cli.get(\"/default\").status_code", + "got": "cli.get(\"/default\").status_code", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "limiter._in_memory_fallback_enabled", + "got": "not limiter._in_memory", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.631578947368421 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "json.loads(cli.get(\"/\").data.decode())", + "got": "json.loads(cli.get(\"/\").data.decode())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "re.compile(\"Enabled.*True\").search(result.output)", + "got": "\"Flask-Limiter\" in result.output", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.41975308641975306 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "re.compile(\"\u2514\u2500\u2500 Status.*\u2514\u2500\u2500 Error\").search(result.output)", + "got": "\"Storage not available\" in result.output", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3917525773195876 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "cli.get(\"/t1\", headers={\"Test-IP\": \"127.0.0.3\"}).status_code", + "got": "cli.get(\"/t1\").status_code", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6046511627906976 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "cli.get(\"/t1\", headers={\"X_FORWARDED_FOR\": \"127.0.0.2\"}).status_code", + "got": "cli.get(\"/t1\").status_code", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5531914893617021 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "re.compile(r\"health.health: /health/\\n\\s*\u2514\u2500\u2500 Exempt\", re.MULTILINE).search(result.output)", + "got": "\"1000 per 1 minute\" in result.output", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.288 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "ConfigurationError)", + "got": "ConfigurationError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "\"t3\")", + "got": "\"route3\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7142857142857143 + }, + { + "repo": "argoverse/av2-api", + "expected": "1", + "got": "1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "2", + "got": "5", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "3", + "got": "8", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "8", + "got": "5", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "0", + "got": "3", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "4", + "got": "3", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "ok", + "got": "ok", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "16", + "got": "3", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "1.0", + "got": "0", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "argoverse/av2-api", + "expected": "out", + "got": "0.5", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "3.0", + "got": "bsa", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "2.0", + "got": "2.0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "135", + "got": "135", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "0.1", + "got": "0.1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "bSc", + "got": "bSc", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "None", + "got": "None", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "1e-4", + "got": "0.05", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "0.06", + "got": "1.0", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "argoverse/av2-api", + "expected": "bSa_", + "got": "bSa_", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "not ok", + "got": "not ok", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "within", + "got": "within", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "(9, 2)", + "got": "(9,)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "argoverse/av2-api", + "expected": "(6, 2)", + "got": "(6, 2)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "(1, 2)", + "got": "(1, 2)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "(4, 2)", + "got": "(4, 2)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "(7, 2)", + "got": "(5, 2)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "argoverse/av2-api", + "expected": "\"1.2.3\"", + "got": "\"v1.2.3\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9333333333333333 + }, + { + "repo": "argoverse/av2-api", + "expected": "\"1.2.4\"", + "got": "\"v1.2.4\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9333333333333333 + }, + { + "repo": "argoverse/av2-api", + "expected": "\"0.1.0\"", + "got": "\"main\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "argoverse/av2-api", + "expected": "message", + "got": "message", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "\"9.8.7\"", + "got": "\"9.8.7\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "(16, 2)", + "got": "(16000, 2)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8235294117647058 + }, + { + "repo": "argoverse/av2-api", + "expected": "4499430", + "got": "4499430", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "(10, 3)", + "got": "(10, 3)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "[-5, 5]", + "got": "[-5, 5]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "93269421", + "got": "93269421", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "93269520", + "got": "93269520", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "93269526", + "got": "93269501", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.75 + }, + { + "repo": "argoverse/av2-api", + "expected": "93269458", + "got": "93269520", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.75 + }, + { + "repo": "argoverse/av2-api", + "expected": "(172, 3)", + "got": "(3, 3)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7142857142857143 + }, + { + "repo": "argoverse/av2-api", + "expected": "(2637, 2)", + "got": "(1000,)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.375 + }, + { + "repo": "argoverse/av2-api", + "expected": "(4, 4, 3)", + "got": "(4, 4, 3)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "not within", + "got": "not within", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "(100660, 3)", + "got": "(1000, 3)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9 + }, + { + "repo": "argoverse/av2-api", + "expected": "(64, 64, 64)", + "got": "bg", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "(96, 96, 96)", + "got": "(192, 192, 192)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "argoverse/av2-api", + "expected": "[0, 1, 1, 0]", + "got": "np.array([[0, 1], [1, 0]])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.631578947368421 + }, + { + "repo": "argoverse/av2-api", + "expected": "dims_expected", + "got": "dims_expected", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "_TEST_SCENARIO", + "got": "_TEST_SCENARIO", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "mf_scenario_id", + "got": "mf_scenario_id", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.dtype(bool)", + "got": "np.dtype(np.bool_)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.875 + }, + { + "repo": "argoverse/av2-api", + "expected": "message.lower()", + "got": "message", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6363636363636364 + }, + { + "repo": "argoverse/av2-api", + "expected": "expected_result", + "got": "expected_result", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "in_intersection", + "got": "in_intersection", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "(128, 128, 128)", + "got": "(255, 255, 255)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6 + }, + { + "repo": "argoverse/av2-api", + "expected": "wSimg.inverse()", + "got": "wSimg.inverse()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "imgSw.inverse()", + "got": "imgSw.inverse()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "test_scenario_id", + "got": "test_scenario_id", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "width_m_expected", + "got": "width_m_expected", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "range_m_expected", + "got": "range_m_expected", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "(55, 500, 500, 3)", + "got": "(55, 500, 500, 3)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "length_m_expected", + "got": "length_m_expected", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "height_m_expected", + "got": "height_m_expected", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "category_expected", + "got": "category_expected", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.all(dts[:, :4])", + "got": "np.all(dts[\"tp\"] > 0)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(R, R_)", + "got": "np.allclose(R, R_)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "(img_w * img_h, 2)", + "got": "(img_w * img_h, 2)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.dtype(np.uint8)", + "got": "np.uint8", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "argoverse/av2-api", + "expected": "expected_result_det", + "got": "expected_result_det", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "not in_intersection", + "got": "not in_intersection", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "save_fpath.exists()", + "got": "save_fpath.exists()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "track_uuid_expected", + "got": "track_uuid_expected", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "expected_mask.dtype", + "got": "np.uint8", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "argoverse/av2-api", + "expected": "point_cloud.shape[0]", + "got": "point_cloud.shape[0]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.dtype(np.float64)", + "got": "np.float32", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5333333333333333 + }, + { + "repo": "argoverse/av2-api", + "expected": "imgSw.compose(wSimg)", + "got": "wSw.compose(imgSw)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7894736842105263 + }, + { + "repo": "argoverse/av2-api", + "expected": "timestamp_ns_expected", + "got": "timestamp_ns_expected", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.isclose(ts_s, 0.3)", + "got": "np.isclose(ts_s, 3)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.95 + }, + { + "repo": "argoverse/av2-api", + "expected": "isinstance(bSa, Sim2)", + "got": "np.allclose(bSa.R, bRa)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "argoverse/av2-api", + "expected": "scenario_path.exists()", + "got": "Path(scenario_path).exists()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.88 + }, + { + "repo": "argoverse/av2-api", + "expected": "expected_successor_ids", + "got": "expected_successor_ids", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.isclose(ts_s, 0.95)", + "got": "np.isclose(ts_s, 0.95)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.isclose(ts_ms, 300)", + "got": "np.isclose(ts_ms, 300)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(R, R_gtsam)", + "got": "np.allclose(R, R_gtsam)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "not save_fpath.exists()", + "got": "save_fpath.exists()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9047619047619048 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(bSa.R, bRa)", + "got": "np.allclose(bSa.R, bRa)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(bSa.t, bta)", + "got": "np.allclose(bSa.t, bta)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "expected_result_identity", + "got": "expected_result_identity", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "(dts[:, 4:7] == 0).all()", + "got": "np.all(dts[:, 4:])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(gt_pts, pts)", + "got": "np.allclose(pts, gt_pts)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7916666666666666 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(pts, gt_pts)", + "got": "np.allclose(pts, gt_pts)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "expected_lane_segment_ids", + "got": "expected_lane_segment_ids", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.isclose(cam_yaw_ego, 0)", + "got": "np.isclose(cam_yaw_ego, 0.0)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9629629629629629 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.array_equal(cart, cart_)", + "got": "np.allclose(cart, cart_)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7058823529411765 + }, + { + "repo": "argoverse/av2-api", + "expected": "len(expected_ped_crossings)", + "got": "2", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.isclose(bSa_.scale, bsa)", + "got": "np.allclose(bSa_.scale, bsa)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.isclose(ts_ns, 950000000)", + "got": "np.isclose(ts_ns, 950000000)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(img, dilated_img)", + "got": "np.array_equal(img, dilated_img)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7540983606557377 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.isclose(ts_s, ts_ns / 1e9)", + "got": "np.isnan(ts_ns)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5454545454545454 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.isclose(ts_ns, ts_s * 1e9)", + "got": "np.isclose(ts_ns, ts_s * 1e9)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "isinstance(polyline, Polyline)", + "got": "isinstance(polyline, Polyline)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "Point(874.01, -105.15, -19.58)", + "got": "Point(x=874.01, y=-105.15)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7857142857142857 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.isclose(fov_theta_deg, 90.0)", + "got": "np.isclose(fov_theta_deg, 90.0)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "isinstance(ped_crossings, list)", + "got": "isinstance(ped_crossings, list)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(bSa_.rotation, bRa)", + "got": "np.allclose(bSa.R, bSa_.R)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7368421052631579 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(gt_ray_dir, ray_dir)", + "got": "np.allclose(gt_ray_dir, ray_dir)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(polygon, gt_polygon)", + "got": "np.allclose(polygon, gt_polygon)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.array_equal(fde, expected_fde)", + "got": "np.allclose(fde, expected_fde)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7619047619047619 + }, + { + "repo": "argoverse/av2-api", + "expected": "forecasted_trajectories.shape[:1]", + "got": "forecasted_trajectories.shape[:2]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9696969696969697 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.array_equal(img, expected_img)", + "got": "np.allclose(img, expected_img)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7619047619047619 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(transformed_pts, pts)", + "got": "np.allclose(pts, transformed_pts)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8484848484848485 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(pts, transformed_pts)", + "got": "np.allclose(pts, transformed_pts)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "math.isclose(aSc.theta_deg, 45.0)", + "got": "np.allclose(aSc.rotation, rotmat2d(np.deg2rad(45)))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4523809523809524 + }, + { + "repo": "argoverse/av2-api", + "expected": "isinstance(ls_polygon, np.ndarray)", + "got": "isinstance(ls_polygon, Polygon)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7692307692307693 + }, + { + "repo": "argoverse/av2-api", + "expected": "isinstance(centerline, np.ndarray)", + "got": "isinstance(centerline, np.ndarray)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "isinstance(cuboid_list[5], Cuboid)", + "got": "isinstance(cuboid_list[0], Cuboid)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9705882352941176 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.array_equal(Xt_5, expected_Xt_5)", + "got": "np.allclose(Xt_5, expected_Xt_5)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7761194029850746 + }, + { + "repo": "argoverse/av2-api", + "expected": "isinstance(cuboid_list, CuboidList)", + "got": "isinstance(cuboid_list, CuboidList)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.array_equal(mask, expected_mask)", + "got": "np.allclose(mask, expected_mask)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7761194029850746 + }, + { + "repo": "argoverse/av2-api", + "expected": "isinstance(city_SE3_egovehicle, SE3)", + "got": "city_SE3_egovehicle.is_valid()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6060606060606061 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(gt_ray_dir, ray_dirs[i])", + "got": "np.allclose(ray_dirs[i], gt_ray_dir)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(dst_se3_src.rotation, R)", + "got": "np.allclose(dst_se3_src.matrix(), T_mat_gt)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.759493670886076 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.isclose(lane_width, gt_lane_width)", + "got": "np.isclose(lane_width, gt_lane_width)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.array_equal(Xt_10, expected_Xt_10)", + "got": "np.array_equal(Xt_10, expected_Xt_10)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.array_equal(Xt_15, expected_Xt_15)", + "got": "np.array_equal(Xt_15, expected_Xt_15)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(expected_R, bSa.rotation)", + "got": "np.allclose(bSa.R, expected_R)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6865671641791045 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(bSa_expected, bSa.matrix)", + "got": "np.allclose(bSa.matrix, bSa_expected)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6756756756756757 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.isclose(aSb.scale, expected_scale)", + "got": "np.isclose(aSb.scale, expected_scale)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "detection_cfg.max_num_dts_per_category", + "got": "detection_cfg.max_num_dts", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7936507936507936 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(interp_pts, gt_interp_pts)", + "got": "np.allclose(interp_pts, gt_interp_pts)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "isinstance(vector_lane_segments, list)", + "got": "isinstance(vector_lane_segments, list)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "isinstance(bev_img_interp, np.ndarray)", + "got": "np.allclose(bev_img_interp[0, 0], RED_RGB)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.55 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(expected_img_pts, img_pts)", + "got": "np.allclose(expected_img_pts, img_pts)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(interp_pts, gt_interp_pts_)", + "got": "np.allclose(interp_pts, gt_interp_pts_)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "isinstance(drivable_area, DrivableArea)", + "got": "drivable_area.id", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.509090909090909 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(dst_se3_src.translation, t)", + "got": "np.allclose(dst_se3_src.translation, t)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.array_equal(intrinsics.K, K_expected)", + "got": "np.allclose(K_expected, intrinsics.K)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4935064935064935 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(ray_dir_vec, ray_dir_nonvec)", + "got": "np.allclose(ray_dir_vec, ray_dir_nonvec)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(dilated_img, dilated_img_gt)", + "got": "np.allclose(dilated_img, dilated_img_gt)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(expected_t, bSa.translation)", + "got": "np.allclose(expected_t, bSa.translation)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(brier_ade, expected_brier_ade)", + "got": "np.array_equal(brier_ade, expected_brier_ade)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8275862068965517 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(brier_fde, expected_brier_fde)", + "got": "np.allclose(brier_fde, expected_bfde)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9367088607594937 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(top_plane, top_plane_expected)", + "got": "np.allclose(top_plane, top_plane_expected)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(expected_world_pts, world_pts)", + "got": "np.allclose(expected_world_pts, world_pts)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.array_equal(vertices, expected_vertices)", + "got": "np.allclose(vertices, expected_vertices)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8192771084337349 + }, + { + "repo": "argoverse/av2-api", + "expected": "spec is not None and spec.loader is not None", + "got": "spec", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(interp_pts, expected_interp_pts)", + "got": "np.allclose(interp_pts, expected_interp_pts)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(left_plane, left_plane_expected)", + "got": "np.allclose(left_plane, left_plane_expected)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(near_plane, near_plane_expected)", + "got": "np.allclose(near_plane, near_plane_expected)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(aSb.rotation, expected_rotation)", + "got": "np.allclose(aSb.rotation, expected_rotation)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.array_equal(dummy_sweep.xyz, xyz_expected)", + "got": "np.allclose(dummy_sweep.xyz, xyz_expected)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8275862068965517 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(fr2_se3_fr0.rotation, combined_R)", + "got": "np.allclose(fr2_se3_fr0.rotation, combined_R)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(centerline_pts, gt_centerline_pts)", + "got": "np.allclose(centerline_pts, gt_centerline_pts)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(right_plane, right_plane_expected)", + "got": "np.array_equal(right_plane, right_plane_expected)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8421052631578947 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.isclose(wrap_angles(theta), expected_result)", + "got": "np.allclose(wrap_angles(theta), expected_result)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9473684210526315 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(quat_wxyz, quat_to_quat(quat_wxyz))", + "got": "np.array_equal(quat_wxyz, quat_to_quat(geometry_utils.quat_to_mat(quat_wxyz)))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.656 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(bottom_plane, bottom_plane_expected)", + "got": "np.allclose(bottom_plane, bottom_plane_expected)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(transformed_pts, gt_transformed_pts)", + "got": "np.allclose(transformed_pts, gt_transformed_pts)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.array_equal(is_interior, expected_is_interior)", + "got": "np.allclose(expected_is_interior, is_interior)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5684210526315789 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.array_equal(blended_img, blended_img_expected)", + "got": "np.array_equal(blended_img, blended_img_expected)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(fr2_se3_fr0.translation, np.zeros(3))", + "got": "np.allclose(fr2_se3_fr0.translation, t0 + t1)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8085106382978723 + }, + { + "repo": "argoverse/av2-api", + "expected": "\"Angle (deg.): 0.0, Trans.: [0. 1.], Scale: 10.5\"", + "got": "\"Sim2(R=array([[1., 0.], [0., 1.]]), t=array([0., 1.]), s=10.5)\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2831858407079646 + }, + { + "repo": "argoverse/av2-api", + "expected": "math.isclose(dts_assignments[2, ATE_COL_IDX], 2.0)", + "got": "not math.isclose(dts_assignments[2, ATE_COL_IDX], expected_result)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8103448275862069 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(scaled_points, scaled_points_expected)", + "got": "np.allclose(scaled_points, scaled_points_expected)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(aSb.translation, expected_translation)", + "got": "np.allclose(aSb.translation, expected_translation)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.array_equal(is_ground_pt, expected_is_ground_pt)", + "got": "np.all(expected_is_ground_pt == is_ground_pt)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5833333333333334 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.array_equal(interior_pts, expected_interior_pts)", + "got": "np.array_equal(interior_pts, expected_interior_pts)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(dst_se3_src.transform_matrix, T_mat_gt)", + "got": "np.allclose(dst_se3_src.matrix, T_mat_gt)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8913043478260869 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(centerline_pts[0], gt_centerline_pts[0])", + "got": "np.allclose(centerline_pts, gt_centerline_pts)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9387755102040817 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(error_deg, expected_error_deg, atol=1e-2)", + "got": "np.isclose(error_deg, expected_error_deg)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8297872340425532 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(centerline_pts[-1], gt_centerline_pts[-1])", + "got": "np.allclose(centerline_pts[3], gt_centerline_pts[3])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9433962264150944 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(interp_pts, expected_interp_pts, atol=0.03)", + "got": "np.allclose(interp_pts, expected_interp_pts)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "argoverse/av2-api", + "expected": "top_plane_expected[1] > 0 and top_plane_expected[2] > 0", + "got": "np.allclose(top_plane, top_plane_expected)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5773195876288659 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(utm_coords, expected_utm_coords, atol=0.01)", + "got": "np.allclose(utm_coords, expected_utm_coords)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.all(is_missed_prediction == expected_is_missed_label)", + "got": "np.array_equal(is_missed_prediction, expected_is_missed_label)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8813559322033898 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.array_equal(dummy_sweep.intensity, intensity_expected)", + "got": "np.array_equal(dummy_sweep.intensity, intensity_expected)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(transformed_pts, gt_transformed_pts_explicit)", + "got": "np.allclose(transformed_pts, gt_transformed_pts_explicit)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.isclose(aSa_gt.theta_deg, aSa_est.theta_deg, atol=1e-5)", + "got": "np.allclose(aSa_gt.R, aSa_est.R)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(wgs84_coords, expected_wgs84_coords, atol=1e-4)", + "got": "np.allclose(wgs84_coords, expected_wgs84_coords)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.897196261682243 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.allclose(eval.compute_angle_error(dts_zero, gts), ae_zero)", + "got": "np.allclose(eval.compute_angle_error(dts_zero, gts), ae_zero)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "bottom_plane_expected[1] < 0 and bottom_plane_expected[2] > 0", + "got": "bottom_plane_expected[2] > 0", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6292134831460674 + }, + { + "repo": "argoverse/av2-api", + "expected": "ValueError)", + "got": "FileNotFoundError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5517241379310345 + }, + { + "repo": "argoverse/av2-api", + "expected": "IndexError)", + "got": "IndexError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "ZeroDivisionError)", + "got": "ValueError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4827586206896552 + }, + { + "repo": "argoverse/av2-api", + "expected": "yaw)", + "got": "yaw)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "mask)", + "got": "mask)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "mask_)", + "got": "mask_)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "dts_mask_)", + "got": "dts_mask_)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "gts_mask_)", + "got": "gts_mask_)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "cropped_xyz)", + "got": "cropped_xyz)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "expected_ade)", + "got": "expected_ade)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "lidar_tensor)", + "got": "lidar_tensor)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "tensor_expected)", + "got": "tensor_expected)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "rotation_matrix_)", + "got": "rotation_matrix_)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "tait_bryan_angles_)", + "got": "tait_bryan_angles_)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "vector_da.xyz[171])", + "got": "vector_da.xyz[-1])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.918918918918919 + }, + { + "repo": "argoverse/av2-api", + "expected": "expected_ls_polygon)", + "got": "expected_ls_polygon)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "expected_centerline)", + "got": "expected_centerline)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "lidar_tensor[:, 1:3])", + "got": "lidar_tensor[:, :, 0:2])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8444444444444444 + }, + { + "repo": "argoverse/av2-api", + "expected": "rotation_matrix_expected)", + "got": "rotation_matrix_expected)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "0", + "got": "0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "1", + "got": "1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "4", + "got": "3", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "5", + "got": "5", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "2", + "got": "2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "3", + "got": "3", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "8", + "got": "8", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "{}", + "got": "{}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "10", + "got": "10", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "\"\"", + "got": "\"\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "20", + "got": "20", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "300", + "got": "300", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "199", + "got": "199", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "150", + "got": "150", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "\"/\"", + "got": "\"/login\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5454545454545454 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "int", + "got": "int", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "123", + "got": "123", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "UTC", + "got": "UTC", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "True", + "got": "True", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "None", + "got": "None", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "\"id\"", + "got": "\"id\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "path", + "got": "path", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "6379", + "got": "6379", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "False", + "got": "False", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "hash2", + "got": "hash2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "email", + "got": "email", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "99.99", + "got": "99.99", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "result", + "got": "result", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "tables", + "got": "tables", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "parsed", + "got": "parsed", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "\"UUID\"", + "got": "\"uuid\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "\"test\"", + "got": "\"test\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "status", + "got": "status", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "action", + "got": "action", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "pk_info", + "got": "pk_info", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "45 * 60", + "got": "45 * 60", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "20 * 60", + "got": "20 * 60", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "\"admin\"", + "got": "\"/admin\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9333333333333333 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "\"redis\"", + "got": "\"redis\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "user_id", + "got": "user_id", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "30 * 60", + "got": "1800", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "password", + "got": "password", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "username", + "got": "username", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "\"/login\"", + "got": "\"/oauth2/token\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.43478260869565216 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "expected", + "got": "expected", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "id_value", + "got": "\"93c025d9-5831-413c-9460-edb3a28cc729\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.08695652173913043 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "repr_str", + "got": "repr_str", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "\"123.45\"", + "got": "str(decimal_val)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "event_id", + "got": "event_id", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "db_config", + "got": "db_config", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "user_dict", + "got": "user_dict", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "\"custom:\"", + "got": "\"custom:\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "uuid.UUID", + "got": "int", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "\"unknown\"", + "got": "\"unknown\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "result.os", + "got": "result.os", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "\"testuser\"", + "got": "\"testuser\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "user[\"id\"]", + "got": "user[\"id\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "len(hash2)", + "got": "60", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "admin_base", + "got": "admin_base", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "prev_price", + "got": "prev_price", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "model_dict", + "got": "model_dict", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "mock_event", + "got": "mock_event", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "event_type", + "got": "event_type", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "session_id", + "got": "session_id", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "csrf_token", + "got": "csrf_token", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "\"session:\"", + "got": "\"rate_limit:\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.43478260869565216 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "caplog.text", + "got": "caplog.text", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "full_fields", + "got": "full_fields", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "\"localhost\"", + "got": "\"localhost\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "\"memcached\"", + "got": "\"memcached\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "mock_result", + "got": "mock_result", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "session_ids", + "got": "session_ids", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "admin_db_url", + "got": "admin_db_url", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "current_time", + "got": "current_time", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "admin.models", + "got": "admin.admin_site.router.routes[0].name", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.32 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "call_args[1]", + "got": "call_args[0][0]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8148148148148148 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "\"rate_limit:\"", + "got": "\"memory\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "initial_admin", + "got": "initial_admin", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "product_model", + "got": "product_model", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "item[\"price\"]", + "got": "item[\"price\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "prev_category", + "got": "prev_category", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "\"/TestModel/\"", + "got": "\"/TestModel/\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "select_fields", + "got": "select_fields", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "DocumentModel", + "got": "DocumentModel", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "\"Test Item 1\"", + "got": "\"Test UUID\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "AuditLogModel", + "got": "AuditLogModel", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "resource_type", + "got": "resource_type", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "new_csrf_token", + "got": "new_csrf_token", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "memory_storage", + "got": "memory_storage", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "len(test_data)", + "got": "len(test_data)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "sorted(prices)", + "got": "sorted(prices)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "DocumentSelect", + "got": "DocumentSelect", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "expected_value", + "got": "expected_value", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "dt.isoformat()", + "got": "dt", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.25 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "EventLogModel2", + "got": "EventLogModel2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "AuditLogModel2", + "got": "AuditLogModel2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "result.browser", + "got": "isinstance(result, tuple)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.358974358974359 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "expected_simple", + "got": "expected_simple", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "expected_fields", + "got": "expected_fields", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "EventType.LOGIN", + "got": "\"LOGIN\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.45454545454545453 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "\"AdminEventLog\"", + "got": "\"EventLog\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "\"AdminAuditLog\"", + "got": "\"AdminAuditLog\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "bulk_request.ids", + "got": "bulk_request.ids", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "problematic_uuid", + "got": "problematic_uuid", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "EventType.LOGOUT", + "got": "EventType.LOGOUT", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "mock_session_data", + "got": "mock_session_data", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "\"test-session-id\"", + "got": "\"test-session-id\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "custom_admin_user", + "got": "custom_admin_user", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "\"admin_event_log\"", + "got": "\"admin_event_log\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "\"admin_audit_log\"", + "got": "\"admin_audit_log\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "\"test@example.com\"", + "got": "\"test@example.com\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "mock_event_service", + "got": "mock_event_service", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "hasattr(item, \"id\")", + "got": "isinstance(item, product_read_schema)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.39285714285714285 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "\"Product to Delete\"", + "got": "\"Product to Delete\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "events_call_args[1]", + "got": "events_call_args[1]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "audits_call_args[1]", + "got": "audits_call_args[1]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "terminated_sessions", + "got": "terminated_sessions", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "db_config.crud_users", + "got": "db_config.crud_users", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "exc_info.value.detail", + "got": "exc_info.value.detail", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "hasattr(admin, \"app\")", + "got": "hasattr(admin, \"app\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "\"RedisSessionStorage\"", + "got": "\"RedisSessionStorage\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "hasattr(item, \"name\")", + "got": "hasattr(item, \"name\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "timedelta(minutes=30)", + "got": "30", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.17391304347826086 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "timedelta(minutes=15)", + "got": "timedelta(minutes=15)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "{\"message\": \"deleted\"}", + "got": "{\"message\": \"deleted\"}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "call_args[\"new_state\"]", + "got": "call_args", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5806451612903226 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "isinstance(hashed, str)", + "got": "hashed", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.41379310344827586 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "\"extracted-session-456\"", + "got": "\"extracted-session-456\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "os.path.exists(data_dir)", + "got": "os.path.exists(data_dir)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "isinstance(result, dict)", + "got": "isinstance(result, dict)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "isinstance(result, list)", + "got": "result", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "hashed.startswith(\"$2b$\")", + "got": "hashed.startswith(\"$2b$\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "path.endswith(\"admin.db\")", + "got": "path.endswith(\".db\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "{\"extend_existing\": True}", + "got": "({\"schema\": \"public\"})", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3404255319148936 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "isinstance(json_str, str)", + "got": "json_str", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.48484848484848486 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "isinstance(allowed, bool)", + "got": "allowed", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4375 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "str(exc_info.value.detail)", + "got": "exc_info.value.detail", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8936170212765957 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "{\"id\", \"title\", \"content\"}", + "got": "{\"id\", \"title\", \"content\"}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "callable(health_check_func)", + "got": "isinstance(health_check_func, Callable)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6363636363636364 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "DocumentSelect.model_fields", + "got": "set(DocumentSelect.model_fields.keys())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8181818181818182 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "\"First test item with UUID\"", + "got": "\"Test description 1\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.425531914893617 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "AuditLogModel.__tablename__", + "got": "\"AuditLogModel.__tablename__\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9642857142857143 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "status.HTTP_401_UNAUTHORIZED", + "got": "401", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.1935483870967742 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "hasattr(admin, \"admin_site\")", + "got": "admin.admin_site", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6818181818181818 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "sorted(prices, reverse=True)", + "got": "sorted(prices, reverse=True)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "callable(health_content_func)", + "got": "callable(health_content_func)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "item[\"name\"].startswith(name)", + "got": "item[\"name\"].startswith(name)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "isinstance(converted_id, str)", + "got": "isinstance(converted_id, str)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "isinstance(converted_id, int)", + "got": "isinstance(converted_id, int)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "isinstance(admin.app, FastAPI)", + "got": "admin.app", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "isinstance(result.is_pc, bool)", + "got": "isinstance(result.is_bot, bool)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9180327868852459 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "{\"message\": \"Login successful\"}", + "got": "{\"message\": \"Login successful\"}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "isinstance(session, AsyncSession)", + "got": "session", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.35 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "hasattr(EventLogModel, field.key)", + "got": "field.name", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.32558139534883723 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "hasattr(AuditLogModel, field.key)", + "got": "hasattr(AuditLogModel, field.key)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "{\"id\": 123, \"name\": \"created_item\"}", + "got": "{\"id\": 123, \"name\": \"created_item\"}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "{\"id\": 123, \"name\": \"updated_item\"}", + "got": "{\"id\": 123, \"name\": \"updated_item\"}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "hasattr(admin, \"admin_user_service\")", + "got": "admin.admin_authentication.get_current_user", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.379746835443038 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "isinstance(result, RedirectResponse)", + "got": "isinstance(result, RedirectResponse)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "{\"host\": \"localhost\", \"port\": 11211}", + "got": "{\"host\": \"localhost\", \"port\": 11211}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "isinstance(app_session, AsyncSession)", + "got": "isinstance(app_session, AsyncSession)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "isinstance(data.first_attempt, float)", + "got": "not data.is_rate_limited", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.45901639344262296 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "isinstance(item, product_read_schema)", + "got": "isinstance(item, product_read_schema)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "isinstance(result, AdminEventLogRead)", + "got": "isinstance(result, AdminEventLogRead)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "hasattr(admin, \"admin_authentication\")", + "got": "hasattr(admin, \"authentication\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9142857142857143 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "isinstance(admin_session, AsyncSession)", + "got": "admin_session", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "isinstance(item, product_create_schema)", + "got": "isinstance(item, product_create_schema)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "excluded_fields.isdisjoint(safe_fields)", + "got": "excluded_fields.issubset(safe_fields)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.868421052631579 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "isinstance(rate_limiter, SimpleRateLimiter)", + "got": "isinstance(rate_limiter, SimpleRateLimiter)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "re.match(uuid_regex, test_uuid, re.IGNORECASE)", + "got": "re.match(uuid_regex, test_uuid)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8051948051948052 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "ANY)", + "got": "user_id=user_id)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.1 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "CSRFToken)", + "got": "CSRFToken)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "session_id)", + "got": "session_id)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "csrf_token)", + "got": "csrf_token)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "encoded_key)", + "got": "encoded_key)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "SessionData)", + "got": "\"test_agent\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "initial_admin)", + "got": "initial_admin)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "\"test-session-id\")", + "got": "\"test-session-id\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "\"test-csrf-token\")", + "got": "\"test-csrf-token\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "\"invalid-session-id\")", + "got": "\"invalid-session-id\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "\"invalid-csrf-token\")", + "got": "\"invalid-csrf-token\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "f\"test_session:{session_id}\")", + "got": "f\"test_session:{session_id}\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "\"Error in event logging: Audit error\")", + "got": "\"Error in audit logging: Audit error\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8947368421052632 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "\"Error in event logging: Database error\")", + "got": "\"Database error\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5862068965517241 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "Exception)", + "got": "Exception)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "TypeError)", + "got": "TypeError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "ValueError)", + "got": "ValueError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "HTTPException)", + "got": "HTTPException)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "CSRFException)", + "got": "CSRFException)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "AttributeError)", + "got": "ValueError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "ConnectionError)", + "got": "ConnectionError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "sqlalchemy.exc.ArgumentError)", + "got": "sqlalchemy.exc.InvalidRequestError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.71875 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "Exception, match=\"Audit error\")", + "got": "Exception, match=\"Audit error\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "Exception, match=\"bcrypt error\")", + "got": "Exception)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.47619047619047616 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "Exception, match=\"Storage error\")", + "got": "Exception)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.46511627906976744 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "Exception, match=\"Create failed\")", + "got": "Exception, match=\"Create failed\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "Exception, match=\"Logging failed\")", + "got": "Exception)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.45454545454545453 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "Exception, match=\"Database error\")", + "got": "Exception)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.45454545454545453 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "ValueError, match=\"Unknown backend\")", + "got": "ValueError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.46808510638297873 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "Exception, match=\"Security logging error\")", + "got": "Exception, match=\"Security logging error\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "9", + "got": "0", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "datarobot/syftr", + "expected": "5", + "got": "6", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "datarobot/syftr", + "expected": "2", + "got": "study_config.optimization.num_eval_samples", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "datarobot/syftr", + "expected": "0", + "got": "0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "1", + "got": "33", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "datarobot/syftr", + "expected": "7", + "got": "10", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "datarobot/syftr", + "expected": "3", + "got": "3", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "6", + "got": "7", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "datarobot/syftr", + "expected": "4", + "got": "3", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "datarobot/syftr", + "expected": "53", + "got": "1000", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "datarobot/syftr", + "expected": "49", + "got": "53", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "datarobot/syftr", + "expected": "20", + "got": "200", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "datarobot/syftr", + "expected": "out", + "got": "isinstance(out, list)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.25 + }, + { + "repo": "datarobot/syftr", + "expected": "None", + "got": "None", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "docs", + "got": "docs", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "True", + "got": "True", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "nodes", + "got": "nodes", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "study", + "got": "study.study_path", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.47619047619047616 + }, + { + "repo": "datarobot/syftr", + "expected": "params", + "got": "isinstance(params, ParamDict)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.34285714285714286 + }, + { + "repo": "datarobot/syftr", + "expected": "result", + "got": "isinstance(result, list)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "datarobot/syftr", + "expected": "duration", + "got": "duration", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "expected", + "got": "expected", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "call_data", + "got": "call_data", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "df.columns", + "got": "df.columns", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "param_names", + "got": "all(param in df_trials.columns for param in param_names)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3283582089552239 + }, + { + "repo": "datarobot/syftr", + "expected": "len(df_all)", + "got": "len(df_all)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "caplog.text", + "got": "caplog.text", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "len(df_dst)", + "got": "len(df_dst)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "eval_result", + "got": "eval_result[\"correct\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "datarobot/syftr", + "expected": "pareto_flows", + "got": "pareto_flows", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "response.text", + "got": "isinstance(response, str)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.47368421052631576 + }, + { + "repo": "datarobot/syftr", + "expected": "[True, False]", + "got": "(\"onnx\", \"onnxruntime\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "datarobot/syftr", + "expected": "feature_names", + "got": "feature_names", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "len(df_trials)", + "got": "df_trials[\"best_trial\"].isna().any()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "datarobot/syftr", + "expected": "expected_score", + "got": "expected_score", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "_TestException", + "got": "_TestException", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "df_trials.empty", + "got": "not df_trials.empty", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8823529411764706 + }, + { + "repo": "datarobot/syftr", + "expected": "baselines_reused", + "got": "baselines_reused", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "resulting_prompt", + "got": "resulting_prompt", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "study.study_path", + "got": "study.get_study()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "datarobot/syftr", + "expected": "not study.remote", + "got": "study.study_id", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4666666666666667 + }, + { + "repo": "datarobot/syftr", + "expected": "df_trials.columns", + "got": "df_trials", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6923076923076923 + }, + { + "repo": "datarobot/syftr", + "expected": "study_config_path", + "got": "study_config_path", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "pareto_df.columns", + "got": "pareto_df", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6923076923076923 + }, + { + "repo": "datarobot/syftr", + "expected": "expected_rag_card", + "got": "expected_rag_card", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "exception_message", + "got": "exception_message", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "expected_reasoning", + "got": "expected_reasoning", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "not df_trials.empty", + "got": "all(param in df_trials.columns for param in param_names)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.37333333333333335 + }, + { + "repo": "datarobot/syftr", + "expected": "data[0].input_tokens", + "got": "data", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "datarobot/syftr", + "expected": "results[\"num_total\"]", + "got": "results[\"num_total\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "expected_no_rag_card", + "got": "expected_no_rag_card", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "str(response).lower()", + "got": "response", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5517241379310345 + }, + { + "repo": "datarobot/syftr", + "expected": "timeout.should_prune()", + "got": "timeout.should_prune()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "qa_pair.supporting_facts", + "got": "any(\"python\" in text.lower() for text in source_texts)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.23076923076923078 + }, + { + "repo": "datarobot/syftr", + "expected": "len(pareto_flows_holdout)", + "got": "len(pareto_flows_holdout)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "not timeout.should_prune()", + "got": "not timeout.should_prune", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.96 + }, + { + "repo": "datarobot/syftr", + "expected": "isinstance(llm, AzureOpenAI)", + "got": "llm", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.1935483870967742 + }, + { + "repo": "datarobot/syftr", + "expected": "api.SyftrStudyStatus.RUNNING", + "got": "\"STOPPED\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.05405405405405406 + }, + { + "repo": "datarobot/syftr", + "expected": "api.SyftrStudyStatus.STOPPED", + "got": "api.SyftrStudyStatus.STOPPED", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "isinstance(study, api.Study)", + "got": "study.study_path", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.45454545454545453 + }, + { + "repo": "datarobot/syftr", + "expected": "df1_sorted.equals(df2_sorted)", + "got": "df1_sorted.equals(df2_sorted)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "api.SyftrStudyStatus.COMPLETED", + "got": "api.SyftrStudyStatus.COMPLETED", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "api.SyftrStudyStatus.INITIALIZED", + "got": "\"NOT_FOUND\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.13953488372093023 + }, + { + "repo": "datarobot/syftr", + "expected": "index._embed_model._warning_logged", + "got": "not index", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.23255813953488372 + }, + { + "repo": "datarobot/syftr", + "expected": "isinstance(examples, datasets.Dataset)", + "got": "isinstance(examples, datasets.Dataset)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "[\"train\", \"test\", \"sample\", \"holdout\"]", + "got": "[\"train\", \"test\", \"sample\", \"holdout\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "df_pareto[cols].equals(df_pareto[cols])", + "got": "all(c in df_dst.columns for c in cols)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2597402597402597 + }, + { + "repo": "datarobot/syftr", + "expected": "study_config.pareto.replacement_llm_name", + "got": "\"llama-2-7b\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.15384615384615385 + }, + { + "repo": "datarobot/syftr", + "expected": "expected_rag_card + expected_no_rag_card", + "got": "2 * ss.rag_retriever.get_cardinality()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2564102564102564 + }, + { + "repo": "datarobot/syftr", + "expected": "isinstance(dataset, datasets.DatasetDict)", + "got": "\"querypairs\" in dataset", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.375 + }, + { + "repo": "datarobot/syftr", + "expected": "\"This study is not running. Run it first.\"", + "got": "\"Study example-dr-docs is not running.\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5432098765432098 + }, + { + "repo": "datarobot/syftr", + "expected": "are_valid_parameters(ss, baseline_template)", + "got": "are_valid_parameters(ss, baseline_template)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "all(isinstance(res, int) for res in result)", + "got": "isinstance(result, list)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5373134328358209 + }, + { + "repo": "datarobot/syftr", + "expected": "llama_debug.get_event_pairs(CBEventType.LLM)", + "got": "call_data[\"tool_calls\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.1791044776119403 + }, + { + "repo": "datarobot/syftr", + "expected": "trial_exists(study_config.name, trial.params)", + "got": "trial_exists(trial)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.59375 + }, + { + "repo": "datarobot/syftr", + "expected": "llama_debug.get_event_pairs(CBEventType.QUERY)", + "got": "llama_debug.get_event_pairs(CBEventType.AGENT_STEP)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.865979381443299 + }, + { + "repo": "datarobot/syftr", + "expected": "llama_debug.get_event_pairs(CBEventType.AGENT_STEP)", + "got": "llama_debug.get_event_pairs(CBEventType.LLM)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8631578947368421 + }, + { + "repo": "datarobot/syftr", + "expected": "has_valid_hyde_params(*search_space_and_trial_params)", + "got": "has_valid_hyde_params(*search_space_and_trial_params)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "\"Cannot find study non_existent_study in the database.\"", + "got": "\"Cannot find this study in the database\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "datarobot/syftr", + "expected": "any(answer in response.text for answer in valid_answers)", + "got": "any(answer.startswith(a) for a in valid_answers)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7115384615384616 + }, + { + "repo": "datarobot/syftr", + "expected": "has_valid_minimum_params(*search_space_and_trial_params)", + "got": "has_valid_minimum_params(search_space_and_trial_params)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.990990990990991 + }, + { + "repo": "datarobot/syftr", + "expected": "has_valid_few_shot_params(*search_space_and_trial_params)", + "got": "has_valid_few_shot_params(*search_space_and_trial_params)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "has_valid_base_rag_params(*search_space_and_trial_params)", + "got": "has_valid_base_rag_params(*search_space_and_trial_params)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "has_valid_splitter_params(*search_space_and_trial_params)", + "got": "has_valid_splitter_params(*search_space_and_trial_params)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "any(fact.lower().strip() in text for text in source_texts)", + "got": "any(fact.lower().strip() in source_texts for source_texts in source_texts)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8787878787878788 + }, + { + "repo": "datarobot/syftr", + "expected": "all(\"accuracy\" in flow[\"metrics\"] for flow in pareto_flows)", + "got": "all(\"llm_cost_std\" in flow[\"metrics\"] for flow in pareto_flows)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8524590163934426 + }, + { + "repo": "datarobot/syftr", + "expected": "has_valid_rag_fusion_params(*search_space_and_trial_params)", + "got": "has_valid_rag_fusion_params(*search_space_and_trial_params)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "\"Not enough points in the Pareto front to find a knee point.\"", + "got": "\"Cannot find this study in the database\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.39603960396039606 + }, + { + "repo": "datarobot/syftr", + "expected": "has_valid_rag_reranker_params(*search_space_and_trial_params)", + "got": "has_valid_rag_reranker_params(*search_space_and_trial_params)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "all(\"llm_cost_mean\" in flow[\"metrics\"] for flow in pareto_flows)", + "got": "isinstance(pareto_flows, list)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3829787234042553 + }, + { + "repo": "datarobot/syftr", + "expected": "has_valid_additional_context_params(*search_space_and_trial_params)", + "got": "has_valid_additional_context_params(*search_space_and_trial_params)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "has_valid_rag_embedding_model_params(*search_space_and_trial_params)", + "got": "has_valid_rag_embedding_model_params(*search_space_and_trial_params)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "exc.value.args[0].startswith(\"Cannot find this study in the database\")", + "got": "exc.value.args[0].startswith(\"Cannot find this study in the database:\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9929078014184397 + }, + { + "repo": "datarobot/syftr", + "expected": "exc.value.args[0].startswith(\"Cannot find this study in the database:\")", + "got": "\"Study does not exist\" in str(exc.value)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.18018018018018017 + }, + { + "repo": "datarobot/syftr", + "expected": "\"Study hotpot-dev-toy-non-existent has no study config in the database.\"", + "got": "\"Cannot find study hotpot-toy-non-existent in the database.\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7121212121212122 + }, + { + "repo": "datarobot/syftr", + "expected": "sorted( pareto_flows_holdout, key=lambda d: sorted(d.items()) )", + "got": "sorted(", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2 + }, + { + "repo": "datarobot/syftr", + "expected": "is_valid if search_space_and_trial_params[1][\"rag_mode\"] == \"no_rag\" else not is_valid", + "got": "not is_valid", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.24489795918367346 + }, + { + "repo": "datarobot/syftr", + "expected": "all( df_trials[\"user_attrs_metric_num_total\"] == study_config.optimization.num_eval_samples )", + "got": "not df_all[\"pruned\"].any()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2689075630252101 + }, + { + "repo": "datarobot/syftr", + "expected": "has_valid_unique_params( *search_space_and_trial_params, \"lats_rag_agent\", [ \"lats_max_rollouts\", \"lats_num_expansions\", ], )", + "got": "has_valid_unique_params( *search_space_and_trial_params, \"reflection_rag_agent\", [ \"reflection_agent_llm_name\", \"reflection_agent_llm_temperature\", \"reflection_agent_llm_top_p\", \"reflection_agent_llm_use_reasoning\", ] )", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5523255813953488 + }, + { + "repo": "datarobot/syftr", + "expected": "has_valid_unique_params( *search_space_and_trial_params, \"critique_rag_agent\", [ \"critique_agent_llm_name\", \"reflection_agent_llm_name\", ], )", + "got": "has_valid_unique_params(*search_space_and_trial_params)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5612244897959183 + }, + { + "repo": "datarobot/syftr", + "expected": "r.top_k.get_cardinality() * len(r.methods) * len(r.query_decomposition_enabled) * r.hybrid.get_cardinality() * r.query_decomposition.get_cardinality() * r.fusion.get_cardinality()", + "got": "1", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "datarobot/syftr", + "expected": "sorted( [ \"additional_context_enabled\", \"additional_context_num_nodes\", \"coa_enable_calculator\", \"critique_agent_llm_name\", \"critique_agent_llm_temperature\", \"critique_agent_llm_top_p\", \"critique_agent_llm_use_reasoning\", \"few_shot_embedding_model\", \"few_shot_enabled\", \"few_shot_top_k\", \"hyde_enabled\", \"hyde_llm_name\", \"hyde_llm_temperature\", \"hyde_llm_top_p\", \"hyde_llm_use_reasoning\", \"lats_max_rollouts\", \"lats_num_expansions\", \"rag_embedding_model\", \"rag_fusion_mode\", \"rag_hybrid_bm25_weight\", \"rag_method\", \"rag_mode\", \"rag_query_decomposition_enabled\", \"rag_query_decomposition_llm_name\", \"rag_query_decomposition_llm_temperature\", \"rag_query_decomposition_llm_top_p\", \"rag_query_decomposition_llm_use_reasoning\", \"rag_query_decomposition_num_queries\", \"rag_top_k\", \"reflection_agent_llm_name\", \"reflection_agent_llm_temperature\", \"reflection_agent_llm_top_p\", \"reflection_agent_llm_use_reasoning\", \"reranker_enabled\", \"reranker_llm_name\", \"reranker_llm_temperature\", \"reranker_llm_top_p\", \"reranker_llm_use_reasoning\", \"reranker_top_k\", \"response_synthesizer_llm_name\", \"response_synthesizer_llm_temperature\", \"response_synthesizer_llm_top_p\", \"response_synthesizer_llm_use_reasoning\", \"splitter_chunk_exp\", \"splitter_chunk_overlap_frac\", \"splitter_method\", \"subquestion_engine_llm_name\", \"subquestion_engine_llm_temperature\", \"subquestion_engine_llm_top_p\", \"subquestion_engine_llm_use_reasoning\", \"subquestion_response_synthesizer_llm_name\", \"subquestion_response_synthesizer_llm_temperature\", \"subquestion_response_synthesizer_llm_top_p\", \"subquestion_response_synthesizer_llm_use_reasoning\", \"template_name\", ] )", + "got": "['base_rag', 'few_shot', 'hyde', 'minimum', 'rag_embedding_model', 'rag_fusion', 'rag_reranker', 'splitter']", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.01384083044982699 + }, + { + "repo": "datarobot/syftr", + "expected": "ValueError)", + "got": "ValueError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "TrialPruned)", + "got": "TrialPruned)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "AssertionError)", + "got": "ValueError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5384615384615384 + }, + { + "repo": "datarobot/syftr", + "expected": "_TestException)", + "got": "_TestException)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "api.SyftrUserAPIError)", + "got": "FileNotFoundError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "datarobot/syftr", + "expected": "EmbeddingPreemptiveTimeoutError)", + "got": "EmbeddingPreemptiveTimeoutError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "4", + "got": "3", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "0", + "got": "0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "2", + "got": "1", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "1", + "got": "1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "tp", + "got": "str", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "erd", + "got": "erd", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "\"+\"", + "got": "\"public_field\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.23529411764705882 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "\"", + "got": "\"_\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "\"-\"", + "got": "\"__\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "out", + "got": "out", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "None", + "got": "None", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "\"str\"", + "got": "\"name\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "Party", + "got": "Party", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "set()", + "got": "set()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "edge2", + "got": "edge2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "[str]", + "got": "set()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "\"name\"", + "got": "\"name\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "model2", + "got": "model2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "field2", + "got": "field2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "gright", + "got": "gright", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "\"Party\"", + "got": "\"Party\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "expected", + "got": "expected", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "edge_key", + "got": "edge_key", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "\"cf-one\"", + "got": "\"target-arrowhead.shape: cf-one\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "[int, T]", + "got": "[T]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5454545454545454 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "model_key", + "got": "model_key", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "field_key", + "got": "field_key", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "d2_string", + "got": "d2_string", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "full_name2", + "got": "full_name2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "full_name1", + "got": "full_name1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "\"image/png\"", + "got": "\"image/png\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "__version__", + "got": "f\"erdantic, version {__version__}\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4888888888888889 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "sys.modules", + "got": "sys.modules", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "predicate_fn", + "got": "predicate_fn", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "result.stderr", + "got": "result.stderr", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "result.output", + "got": "result.output", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "'\"ValidName\"'", + "got": "'\"ValidName\"'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "get_fields_fn", + "got": "get_fields_fn", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "path1.exists()", + "got": "path1.exists()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "'\"list[Item]\"'", + "got": "\"list[Item]\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9230769230769231 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "list_plugins()", + "got": "list_plugins()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "expected_diagram", + "got": "expected_diagram", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "hash(full_name2)", + "got": "hash(full_name2)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "hash(full_name3)", + "got": "hash(full_name3)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "list[Adventurer]", + "got": "\"this_is_arbitrary\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.22857142857142856 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "repr(FancyInt())", + "got": "\".FancyInt, int, object)> with args\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.22807017543859648 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "GlobalWithFwdRefs", + "got": "GlobalWithFwdRefs", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "\"Optional[Quest]\"", + "got": "\"Quest\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5833333333333334 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "path_base.exists()", + "got": "path_base.exists()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "\"GlobalWithFwdRefs\"", + "got": "\"GlobalWithFwdRefs\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "expected_model_keys", + "got": "expected_model_keys", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "attrs_examples.Party", + "got": "attrs_examples.Party", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "diagram._repr_png_()", + "got": "not IPython_png._repr_png_.was_called()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4745762711864407 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "diagram._repr_svg_()", + "got": "diagram._repr_svg_()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "'\"He said \\\\\"hi\\\\\"\"'", + "got": "'\"He said \\\\\"hi\\\\\"\"'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "GlobalOtherClassBefore", + "got": "GlobalOtherClassBefore", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "FnScopeOtherClassAfter", + "got": "FnScopeOtherClassAfter", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "msgspec_examples.Party", + "got": "msgspec_examples.Party", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "FnScopeOtherModelAfter", + "got": "FnScopeOtherModelAfter", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "GlobalOtherModelBefore", + "got": "GlobalOtherModelBefore", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "not is_attrs_class(str)", + "got": "not is_attrs_class(NotAPydanticModel)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "FnScopeOtherClassBefore", + "got": "FnScopeOtherClassBefore", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "FnScopeOtherModelBefore", + "got": "FnScopeOtherModelBefore", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "expected_pydantic_models", + "got": "expected_pydantic_models", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "\"GlobalOtherClassBefore\"", + "got": "\"GlobalOtherClassBefore\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "\"FnScopeOtherModelAfter\"", + "got": "\"FnScopeOtherModelAfter\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "expected_path.read_text()", + "got": "expected_path.read_text()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "not is_nullable_type(int)", + "got": "not is_nullable_type(int)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "warning_records[0].message", + "got": "warning_records[0].message", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "not is_msgspec_struct(str)", + "got": "not is_msgspec_struct(NotAStruct())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.819672131147541 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "not is_pydantic_model(str)", + "got": "not is_pydantic_model(NotAPydanticModel())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7058823529411765 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "not is_attrs_class(\"Hello\")", + "got": "not is_attrs_class(NotAPydanticModel)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6875 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "not is_dataclass_class(str)", + "got": "not is_dataclass_class(NotADataclass)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.78125 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "mock_rich_print.was_called()", + "got": "mock_rich_print.was_called()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "not is_pydantic_v1_model(str)", + "got": "not is_pydantic_v1_model(NotAPydanticModel())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7297297297297297 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "not is_msgspec_struct(\"Hello\")", + "got": "not is_msgspec_struct(NotAStruct())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7384615384615385 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "not is_pydantic_model(\"Hello\")", + "got": "not is_pydantic_model(NotAPydanticModel())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6944444444444444 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "exc_info.match(\"'not_a_field'\")", + "got": "exc_info.match(f\"'not_a_field'\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9841269841269841 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "not is_dataclass_class(\"Hello\")", + "got": "not is_dataclass_class(NotADataclass)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7352941176470589 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "expected_path.read_text().strip()", + "got": "expected_path.read_text().strip()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "not is_msgspec_struct(NotAStruct)", + "got": "not is_msgspec_struct(int)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8135593220338984 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "not is_pydantic_v1_model(\"Hello\")", + "got": "not is_pydantic_v1_model(NotAPydanticModel())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.717948717948718 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "filecmp.cmp(out_path, expected_path)", + "got": "filecmp.cmp(out_path, expected_path)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "is_attrs_class(attrs_examples.Party)", + "got": "not is_attrs_class(NotAPydanticModel)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5205479452054794 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "not is_dataclass_class(NotADataclass)", + "got": "not is_dataclass_class(int)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.78125 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "is_nullable_type(typing.Optional[int])", + "got": "is_nullable_type(typing.Optional[int])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "not is_pydantic_model(NotAPydanticModel)", + "got": "not is_pydantic_model(int)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7575757575757576 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "not is_collection_type_of(Target, Target)", + "got": "not is_collection_type_of(typing.List[Target], int)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.782608695652174 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "is_nullable_type(typing.Union[int, None])", + "got": "not is_nullable_type(int)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6363636363636364 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "not is_pydantic_v1_model(NotAPydanticModel)", + "got": "not is_pydantic_v1_model(int)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7777777777777778 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "re.findall(rf\"\\[X\\]\\s*{key}\", result.output)", + "got": "re.findall(r\"\\[X\\]\\s*{}\".format(key), result.output)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8541666666666666 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "re.findall(r\"\\[X\\]\\s*pydantic\", result.output)", + "got": "\"erdantic.plugins\" in result.stdout", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4691358024691358 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "is_collection_type_of(typing.List[Target], Target)", + "got": "is_collection_type_of(list, Target)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "not is_collection_type_of(typing.List[int], Target)", + "got": "not is_collection_type_of(typing.Any, Target)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8958333333333334 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "exc_info.match(f\"'{FullyQualifiedName.from_object(Party)}'\")", + "got": "str(exc_info.value)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3291139240506329 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "all(isinstance(edge, CustomEdge) for edge in diagram.edges.values())", + "got": "all(isinstance(edge, CustomEdge) for edge in diagram.edges.values())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "filecmp.cmp(out_dir / f\"{filename}.png\", expected_dir / f\"{filename}.png\")", + "got": "not os.path.exists(out_dir / f\"{filename}.png\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.512396694214876 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "all(isinstance(model_info, CustomModelInfo) for model_info in diagram.models.values())", + "got": "isinstance(diagram.models[\"Party\"], CustomModelInfo)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4927536231884058 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "\".FancyInt, int, object)>\"", + "got": "\"FancyInt\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.20408163265306123 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "expected_path.read_text())", + "got": "expected_path)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "ValueError)", + "got": "ValueError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "RuntimeError)", + "got": "ModelOrModuleNotFoundError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "FieldNotFoundError)", + "got": "FieldNotFoundError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "PluginNotFoundError)", + "got": "PluginNotFoundError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "ModuleNotFoundError)", + "got": "ImportError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "UnknownModelTypeError)", + "got": "UnknownModelTypeError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "_UnevaluatedForwardRefError)", + "got": "_UnevaluatedForwardRefError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "erdantic.exceptions.UnevaluatedForwardRefError)", + "got": "ImportError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3050847457627119 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "UnresolvableForwardRefError, match=\"'FnScopeOtherModelAfter'\")", + "got": "UnresolvableForwardRefError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6222222222222222 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "UnresolvableForwardRefError, match=\"'FnScopeOtherModelBefore'\")", + "got": "UnresolvableForwardRefError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "UnresolvableForwardRefError, match=\"'FnScopeManuallyResolvable'\")", + "got": "UnresolvableForwardRefError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6021505376344086 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "1", + "got": "\"42\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "0", + "got": "1", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "2", + "got": "2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "3", + "got": "3", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "4", + "got": "2", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "6", + "got": "7", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "5", + "got": "5", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "8", + "got": "7", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "v", + "got": "v", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "24", + "got": "24", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "42", + "got": "42", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "[]", + "got": "[\"blarg.toml\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.25 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "23", + "got": "23", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "-1", + "got": "-1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "99", + "got": "99", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "10", + "got": "10", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "43", + "got": "43", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "80", + "got": "80", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "12", + "got": "12", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"\"", + "got": "\"\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "45", + "got": "21", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "{}", + "got": "Person()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "14", + "got": "99", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "20", + "got": "20", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "30", + "got": "30", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "36", + "got": "30", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "123", + "got": "123", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "500", + "got": "500", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "[1]", + "got": "[1]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "4.2", + "got": "3.14", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"A\"", + "got": "\"A\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"D\"", + "got": "\"D\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "1.1", + "got": "1.1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "999", + "got": "999", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "val", + "got": "val", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "100", + "got": "100", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"1\"", + "got": "\"test\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"2\"", + "got": "\"2\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "789", + "got": "1", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"7\"", + "got": "\"7\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"a\"", + "got": "\"secret_key\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.26666666666666666 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\".\"", + "got": "\"test\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "True", + "got": "True", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "None", + "got": "None", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "42.1", + "got": "\"VALUE\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "5000", + "got": "8080", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "1234", + "got": "1234", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "8080", + "got": "8080", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "5432", + "got": "5432", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "data", + "got": "data", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "42.0", + "got": "42.0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "list", + "got": "list", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "dict", + "got": "dict", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"aA\"", + "got": "\"a='aA'\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"bA\"", + "got": "\"aB\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"cA\"", + "got": "\"cA\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"aC\"", + "got": "\"aC\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"bC\"", + "got": "\"bC\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "4040", + "got": "4040", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "core", + "got": "core", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"db\"", + "got": "\"db\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"jj\"", + "got": "\"jj\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "_dir", + "got": "_dir", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"42\"", + "got": "\"42\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "34.0", + "got": "34.0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "23.2", + "got": "23.2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "[{}]", + "got": "[\"\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "42.2", + "got": "1.0", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"i3\"", + "got": "\"i3\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "9999", + "got": "\"9999\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"22\"", + "got": "\"22\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"OK\"", + "got": "\"OK\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "2000", + "got": "5432", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.25 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "5555", + "got": "\"80\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "5433", + "got": "5432", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.75 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "False", + "got": "False", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"500\"", + "got": "\"500\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"bar\"", + "got": "\"bar\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"new\"", + "got": "\"new\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"foo\"", + "got": "\"bar\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"baz\"", + "got": "\"baz\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "value", + "got": "value", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"dev\"", + "got": "\"admin\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"ham\"", + "got": "\"ham\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"naz\"", + "got": "\"naz\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"a,b\"", + "got": "\"a,b\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "core1", + "got": "core1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"zaz\"", + "got": "\"zaz\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "77777", + "got": "\"88888\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"FOO\"", + "got": "\"FOO\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"456\"", + "got": "\"456\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"BAR\"", + "got": "\"BAR\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"bye\"", + "got": "\"bye\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"7.2\"", + "got": "\"6.10\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5454545454545454 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"sff\"", + "got": "\"dynaconf\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"NAR\"", + "got": "\"bar\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "500.42", + "got": "\"500.4\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7692307692307693 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "[1, 2]", + "got": "[1, 2]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "id(s2)", + "got": "id(s2)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "result", + "got": "result.stderr", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.631578947368421 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"nest\"", + "got": "\"nest\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"zxcv\"", + "got": "\"zxcv\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"tofu\"", + "got": "\"tofu\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"None\"", + "got": "\"all\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"prod\"", + "got": "\"prod\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"blaz\"", + "got": "\"blaz\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "748632", + "got": "748632", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"\ud83d\ude00\ud83d\ude00\ud83d\ude00\ud83d\ude00\"", + "got": "\"\ud83d\ude00\ud83d\ude00\ud83d\ude00\ud83d\ude00\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"Gray\"", + "got": "\"Gray\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"Erik\"", + "got": "\"Erik\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"true\"", + "got": "True", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"\\n\\n\"", + "got": "\"\\n\\n\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"unit\"", + "got": "\"unit_testing\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"0.30\"", + "got": "\"0.30\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"pulp\"", + "got": "\"pulp\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"TRUE\"", + "got": "\"https://github.com/rook/rook.git\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.1 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"9999\"", + "got": "\"9999\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"data\"", + "got": "\"https://www.google.com\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"LEAF\"", + "got": "\"root\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "[9999]", + "got": "[9999]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "stdout", + "got": "stdout", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"ciao\"", + "got": "\"another_setting\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.34782608695652173 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"INFO\"", + "got": "\"DEBUG\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"info\"", + "got": "\"DEBUG\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"hello\"", + "got": "\"hello\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "180.235", + "got": "180.235", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"admin\"", + "got": "\"admin\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"Bruno\"", + "got": "\"Bruno\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"first\"", + "got": "\"first\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"third\"", + "got": "\"third\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"white\"", + "got": "\"white\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "core.id", + "got": "core", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7272727272727273 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"Karla\"", + "got": "\"Karla\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"teste\"", + "got": "\"teste\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"false\"", + "got": "\"false\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"POMME\"", + "got": "\"POMME\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"value\"", + "got": "\"value\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "+172800", + "got": "\"+172800\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.875 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"world\"", + "got": "\"world\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"Black\"", + "got": "\"Black\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"Rocha\"", + "got": "\"Silva\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.42857142857142855 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"Booba\"", + "got": "\"test\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"Viana\"", + "got": "\"Viana\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "[\"env\"]", + "got": "\"py\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "DB_PATH", + "got": "DB_PATH", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"debug\"", + "got": "\"DEBUG\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"value1\"", + "got": "\"value1\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"value2\"", + "got": "\"value2\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "expected", + "got": "expected", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"500.42\"", + "got": "500.42", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "[\"core\"]", + "got": "[\"core\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "{\"c\": 3}", + "got": "{\"c\": 3}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"abcdef\"", + "got": "\"abcdef\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"foobar\"", + "got": "\"foobar\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"second\"", + "got": "\"second\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"banana\"", + "got": "\"banana\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"Yellow\"", + "got": "\"Yellow\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"plugin\"", + "got": "\"plugin\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"164110\"", + "got": "\"164110\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "dyna_box", + "got": "dyna_box", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"global\"", + "got": "\"global\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"@none \"", + "got": "\"@null\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5333333333333333 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"Wilsom\"", + "got": "\"Wilsom\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"FOOBAR\"", + "got": "\"FOOBAR\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"BATATA\"", + "got": "\"BATATA\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"POTATO\"", + "got": "\"{{this.FILENAME}}\".format(this=filename)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.12244897959183673 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"SECRET\"", + "got": "\"SECRET\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "EXPECTED", + "got": "EXPECTED", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "settings", + "got": "settings.foo", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"s3cr3t\"", + "got": "\"jenkins\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.35294117647058826 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"Task1-1\"", + "got": "\"Task1-1\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"Task2-2\"", + "got": "\"Task2-2\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "[\"item1\"]", + "got": "[\"item1\", \"item1\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"baz_qux\"", + "got": "\"baz_qux\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "{\"x\": 26}", + "got": "{\"x\": 26}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"", + "got": "\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"foo bar\"", + "got": "\"@format {this.a.b} {this.a.c}\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.25 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "KeyError)", + "got": "KeyError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "TypeError)", + "got": "TypeError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "Exception)", + "got": "SyntaxError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.18181818181818182 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "ValueError)", + "got": "TypeError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "IndexError)", + "got": "KeyError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "ImportError)", + "got": "ModuleNotFoundError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "RuntimeError)", + "got": "ValueError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "AttributeError)", + "got": "AttributeError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "AssertionError)", + "got": "ValueError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5384615384615384 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "ValidationError)", + "got": "ValidationError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "KeyNotFoundError)", + "got": "KeyError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6923076923076923 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "EnvNotFoundError)", + "got": "EnvNotFoundError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "OutputFormatError)", + "got": "ValueError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4827586206896552 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "FileNotFoundError)", + "got": "FileNotFoundError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "DynaconfParseError)", + "got": "DynaconfFormatError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7692307692307693 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "DynaconfFormatError)", + "got": "ValueError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.45161290322580644 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "container)", + "got": "container)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "1)", + "got": "1)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "19)", + "got": "19)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "30)", + "got": "30)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "596)", + "got": "596)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"c\")", + "got": "\"c\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "settings.ANOTHER_DEBUG)", + "got": "settings.DEBUG)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7894736842105263 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "Bip32Depth(1) < 2)", + "got": "Bip32KeyIndex(1) < Bip32KeyIndex(2))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5185185185185185 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "Bip32Depth(2) > 1)", + "got": "Bip32KeyIndex(1) < Bip32KeyIndex(2))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4074074074074074 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "path.IsAbsolute())", + "got": "not path.IsAbsolute())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "Bip32Depth(1) == 1)", + "got": "Bip32KeyIndex(1) < Bip32KeyIndex(2))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.43636363636363634 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "fprint.IsMasterKey())", + "got": "fprint.IsDefault())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.65 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "deser_key.IsPublic())", + "got": "deser_key.IsExtended())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6818181818181818 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "bip_ctx.IsPublicOnly())", + "got": "bip_ctx.IsPublicOnly())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "bip32_ctx.IsPublicOnly())", + "got": "bip32_ctx.IsPublicOnly())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "EosAddr is EosAddrEncoder)", + "got": "isinstance(EosAddr.EOS_ADDR_CLASS, str))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "EthAddr is EthAddrEncoder)", + "got": "isinstance(EthAddr(), EthAddrEncoder))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.71875 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "IcxAddr is IcxAddrEncoder)", + "got": "isinstance(IcxAddr, IcxAddrEncoder))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7419354838709677 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "InjAddr is InjAddrEncoder)", + "got": "isinstance(InjAddr, InjAddrDecoder))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6774193548387096 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "NimAddr is NimAddrEncoder)", + "got": "isinstance(NimAddr.NimAddr, NimAddr))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5396825396825397 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "OneAddr is OneAddrEncoder)", + "got": "isinstance(OneAddr, OneAddrEncoder))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7419354838709677 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "SolAddr is SolAddrEncoder)", + "got": "isinstance(SolAddr, SolAddrEncoder))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7419354838709677 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "SuiAddr is SuiAddrEncoder)", + "got": "isinstance(SuiAddr(), SuiAddrEncoder))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.71875 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "TonAddr is TonAddrEncoder)", + "got": "isinstance(TonAddr, TonAddrEncoder))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7419354838709677 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "TrxAddr is TrxAddrEncoder)", + "got": "issubclass(TrxAddr, TrxAddrEncoder))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7419354838709677 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "XlmAddr is XlmAddrEncoder)", + "got": "issubclass(XlmAddr.XlmAddr, XlmAddr.XlmAddrBase))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.48 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "XmrAddr is XmrAddrEncoder)", + "got": "isinstance(XmrAddrDecoder(), XmrAddr))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "XrpAddr is XrpAddrEncoder)", + "got": "isinstance(XrpAddr.XrpAddr(), XrpAddr.XrpAddr))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4931506849315068 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "XtzAddr is XtzAddrEncoder)", + "got": "issubclass(XtzAddr.XtzAddr, XtzAddr.XtzAddrBase))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.48 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "ZilAddr is ZilAddrEncoder)", + "got": "isinstance(ZilAddr(), ZilAddrEncoder))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.71875 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "P2SHAddr is P2SHAddrEncoder)", + "got": "isinstance(P2SHAddrDecoder(), P2SHAddr))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5294117647058824 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "P2TRAddr is P2TRAddrEncoder)", + "got": "isinstance(P2TRAddr(), P2TRAddrEncoder))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7352941176470589 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "AlgoAddr is AlgoAddrEncoder)", + "got": "issubclass(AlgoAddr.Ed25519Address, AlgoAddr.Ed25519AddressBase))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4731182795698925 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "AtomAddr is AtomAddrEncoder)", + "got": "isinstance(AtomAddrDecoder._old_addr_cls, AtomAddrDecoder))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5517241379310345 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "EgldAddr is EgldAddrEncoder)", + "got": "isinstance(EgldAddr, EgldAddrEncoder))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7575757575757576 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "MvrkAddr is MvrkAddrEncoder)", + "got": "isinstance(MvrkAddr, MvrkAddrEncoder))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7575757575757576 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "NanoAddr is NanoAddrEncoder)", + "got": "isinstance(NanoAddr, NanoAddrEncoder))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7575757575757576 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "NearAddr is NearAddrEncoder)", + "got": "isinstance(NearAddr.NEAR_ADDR_CLASS, type))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3380281690140845 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "OkexAddr is OkexAddrEncoder)", + "got": "isinstance(OkexAddr(), OkexAddr))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5901639344262295 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "P2PKHAddr is P2PKHAddrEncoder)", + "got": "isinstance(P2PKHAddrDecoder(), P2PKHAddrDecoder))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6329113924050633 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "AptosAddr is AptosAddrEncoder)", + "got": "isinstance(AptosAddr, AptosAddrEncoder))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7714285714285715 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "NeoN3Addr is NeoN3AddrEncoder)", + "got": "isinstance(NeoN3Addr.NEO_N3_ADDR_CLASS, type))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.39473684210526316 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "P2WPKHAddr is P2WPKHAddrEncoder)", + "got": "issubclass(P2WPKHAddr, P2WPKHAddrEncoder))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7837837837837838 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "point.CurveType() is curve_type)", + "got": "point.X() == TEST_X)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "BchP2SHAddr is BchP2SHAddrEncoder)", + "got": "issubclass(BchP2SHAddr, BchP2SHAddrEncoder))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7948717948717948 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "bip_ctx.IsLevel(Bip44Levels.COIN))", + "got": "bip_ctx.IsLevel(Bip44Levels.COIN))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "bip44_key.Bip32Key() is bip32_key)", + "got": "bip32_key.IsPrivate())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.35714285714285715 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "isinstance(mnemonic, TonMnemonic))", + "got": "TonMnemonicValidator(lang).IsValid(mnemonic, passphrase))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "isinstance(point.Raw(), DataBytes))", + "got": "isinstance(point, Ed25519Point))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6268656716417911 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "BchP2PKHAddr is BchP2PKHAddrEncoder)", + "got": "issubclass(BchP2PKHAddr, BchP2PKHAddrEncoder))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8048780487804879 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "mnemonic_validator.IsValid(mnemonic))", + "got": "mnemonic_validator.IsValid(test[\"mnemonic\"]))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9024390243902439 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "Sr25519.PointClass() is Sr25519Point)", + "got": "Sr25519.PointClass() is Sr25519Point)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "ErgoP2PKHAddr is ErgoP2PKHAddrEncoder)", + "got": "isinstance(ErgoP2PKHAddr(), ErgoP2PKHAddrEncoder))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7954545454545454 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "NeoLegacyAddr is NeoLegacyAddrEncoder)", + "got": "isinstance(NeoLegacyAddr.NEO_LEGACY_ADDR_CLASS, type))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3695652173913043 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "isinstance(priv_key.Raw(), DataBytes))", + "got": "isinstance(priv_key.KeyObject().Raw(), bytes))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7857142857142857 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "pub_key.Data() is TEST_BIP32_KEY_DATA)", + "got": "pub_key.Data() is TEST_BIP32_KEY_DATA)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "Bip32Ed25519Slip is Bip32Slip10Ed25519)", + "got": "isinstance(Bip32Slip10Ed25519, Bip32Base))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5185185185185185 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "Bip32Nist256p1 is Bip32Slip10Nist256p1)", + "got": "isinstance(Bip32Slip10Nist256p1, Bip32Slip10Nist256p1Cls))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7422680412371134 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "Bip32Secp256k1 is Bip32Slip10Secp256k1)", + "got": "isinstance(Bip32Slip10Secp256k1, Bip32Slip10MstKeyGenerator))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.46 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "Bip32Utils.IsHardenedIndex(0x80000000))", + "got": "Bip32Utils.IsHardenedIndex(0x80000000))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "AdaShelleyAddr is AdaShelleyAddrEncoder)", + "got": "AdaShelleyAddrDecoder is AdaShelleyAddr)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.825 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "AvaxPChainAddr is AvaxPChainAddrEncoder)", + "got": "isinstance(AvaxPChainAddr(), AvaxPChainAddrEncoder))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8043478260869565 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "AvaxXChainAddr is AvaxXChainAddrEncoder)", + "got": "AvaxXChainAddr is AvaxXChainAddrEncoder)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "Bip32Ed25519Kholaw is Bip32KholawEd25519)", + "got": "isinstance(Bip32KholawEd25519, Bip32Base))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5060240963855421 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "isinstance(bip32_obj.Depth(), Bip32Depth))", + "got": "isinstance(bip32_obj.Depth(), int))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8831168831168831 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "FilSecp256k1Addr is FilSecp256k1AddrEncoder)", + "got": "isinstance(FilSecp256k1Addr(), FilSecp256k1AddrEncoder))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.82 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "bip32_ctx.ParentFingerPrint().IsMasterKey())", + "got": "bip32_ctx.IsPrivateOnly())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "issubclass(Ed25519MoneroPoint, Ed25519Point))", + "got": "issubclass(Ed25519MoneroPoint, Ed25519Point))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "AdaShelleyRewardAddr is AdaShelleyStakingAddr)", + "got": "isinstance(AdaShelleyRewardAddrDecoder(), AdaShelleyStakingAddrDecoder))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7288135593220338 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "XmrIntegratedAddr is XmrIntegratedAddrEncoder)", + "got": "XmrIntegratedAddr is XmrIntegratedAddrEncoder)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "isinstance(pub_key.RawCompressed(), DataBytes))", + "got": "isinstance(pub_key.Raw(), DataBytes))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8809523809523809 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "AdaByronLegacyAddr is AdaByronLegacyAddrEncoder)", + "got": "isinstance(AdaByronLegacyAddrDecoder(), AdaByronLegacyAddr))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7037037037037037 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "AdaByronIcarusAddr is AdaByronIcarusAddrEncoder)", + "got": "AdaByronIcarusAddr is AdaByronIcarusAddrEncoder)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "isinstance(bip_mst_ctx.CoinConf(), BipCoinConf))", + "got": "bip_mst_ctx.IsCoinConfigured())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5822784810126582 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "isinstance(substrate_obj.Path(), SubstratePath))", + "got": "isinstance(substrate_obj.PublicKey().ToAddress(), str))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6796116504854369 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "Bip32KholawEd25519.IsPublicDerivationSupported())", + "got": "self.bip32.is_supported_derivation())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4186046511627907 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "CardanoIcarusBip32.IsPublicDerivationSupported())", + "got": "self._is_supported_derivation())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.37037037037037035 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "mnemonic_validator.IsValid(mnemonic, passphrase))", + "got": "mnemonic_validator.IsValid(mnemonic))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8604651162790697 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "isinstance(monero_obj.CoinConf(), MoneroCoinConf))", + "got": "monero_obj.IsWatchOnly())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "Bip32Slip10Nist256p1.IsPublicDerivationSupported())", + "got": "self.bip32.slip10_nist256p1.is_supported_derivation())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5904761904761905 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "Bip32Slip10Secp256k1.IsPublicDerivationSupported())", + "got": "self._is_supported_derivation())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3614457831325301 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "AdaShelleyRewardAddr is AdaShelleyRewardAddrEncoder)", + "got": "AdaShelleyAddrDecoder is AdaShelleyAddrDecoder)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7676767676767676 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "SubstrateEd25519Addr is SubstrateEd25519AddrEncoder)", + "got": "issubclass(SubstrateEd25519Addr, SubstrateEd25519AddrEncoder))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8596491228070176 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "SubstrateSr25519Addr is SubstrateSr25519AddrEncoder)", + "got": "SubstrateSr25519Addr is SubstrateSr25519AddrEncoder)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "ex)", + "got": "ex)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "ex_type)", + "got": "ex_type)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "TypeError)", + "got": "TypeError, addr_enc_class.EncodeKey, key)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.39215686274509803 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "ValueError)", + "got": "ValueError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "test[\"ex\"])", + "got": "test[\"ex\"])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "TonKeyError)", + "got": "TonKeyError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "RuntimeError)", + "got": "NotImplementedError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5454545454545454 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "Bip32KeyError)", + "got": "Bip32KeyError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "Bip32PathError)", + "got": "Bip32PathError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "MoneroKeyError)", + "got": "ValueError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5384615384615384 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "Bip44DepthError)", + "got": "Bip32KeyError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "test[\"exception\"])", + "got": "MnemonicChecksumError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "0)", + "got": "test[\"raw\"][:4])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.1111111111111111 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "33)", + "got": "ED25519_COMPRESSED_LENGTH)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.06896551724137931 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "64)", + "got": "32)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "32)", + "got": "32)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "65)", + "got": "65)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "enc)", + "got": "enc)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "rnd)", + "got": "rnd)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "pda)", + "got": "pda)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "addr)", + "got": "addr)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "elem)", + "got": "elem)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "ex_pub)", + "got": "ex_pub)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "subaddr)", + "got": "subaddr.Address())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "ex_priv)", + "got": "ex_priv)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "def_addr)", + "got": "def_addr)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "conv_addr)", + "got": "conv_addr)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "str(path))", + "got": "path.ToStr())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5217391304347826 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "int(elem))", + "got": "elem)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "str(elem))", + "got": "elem.ToStr())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5217391304347826 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "curve_type)", + "got": "curve_type)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "\"BIP-0044\")", + "got": "\"BIP44\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8421052631578947 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "\"BIP-0049\")", + "got": "\"BIP49\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8421052631578947 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "\"BIP-0084\")", + "got": "\"BIP84\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8421052631578947 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "\"BIP-0086\")", + "got": "\"BIP86\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8421052631578947 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "\"CIP-1852\")", + "got": "\"CIP-1852\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "test[\"raw\"])", + "got": "test[\"raw\"])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "test[\"wif\"])", + "got": "bip32_key.ToWif())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.26666666666666666 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "mnemonic_len)", + "got": "mnemonic_len)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "path.ToStr())", + "got": "path.ToStr())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "pub_key_mode)", + "got": "pub_key_mode)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "str(mnemonic))", + "got": "mnemonic.ToStr(lang))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "path.Length())", + "got": "len(path))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "test_words_num)", + "got": "test_words_num)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "str(path_elem))", + "got": "path_elem.ToBytes())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6285714285714286 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "data_bytes_obj)", + "got": "data_bytes_obj)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "key_net_ver_pub)", + "got": "key_net_ver_pub)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "pub_key.Curve())", + "got": "pub_key.Curve())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "bip_ctx.Level())", + "got": "bip_ctx.Level())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "test[\"address\"])", + "got": "test[\"address\"])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "mnemonic.ToStr())", + "got": "mnemonic)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6923076923076923 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "key_net_ver_priv)", + "got": "key_net_ver_pub)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8484848484848485 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "priv_key.Curve())", + "got": "priv_key.CurveType())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8947368421052632 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "Bip44Levels.COIN)", + "got": "Bip44Levels.COIN)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "ton.GetAddress())", + "got": "ton.GetAddress(TonAddrVersions.DEFAULT))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5964912280701754 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "test_bit_len // 8)", + "got": "test_bit_len)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8387096774193549 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "bip32_ctx.Depth())", + "got": "bip32_ctx.Depth())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "bip32_ctx.Index())", + "got": "bip32_ctx.FingerPrint())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7142857142857143 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "bip32_obj.Depth())", + "got": "bip32_obj.Depth())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "path_elem.ToStr())", + "got": "path_elem.ToString())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9230769230769231 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "path_elem.IsHard())", + "got": "path_elem.IsHard())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "pub_key.CurveType())", + "got": "pub_key.CurveType())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "pub_key.ToAddress())", + "got": "pub_key.Address())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9473684210526315 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "priv_key.CurveType())", + "got": "priv_key.CurveType())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "deser_key.KeyBytes())", + "got": "deser_key.PublicKey().RawCompressed().ToBytes())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6086956521739131 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "binascii.hexlify(dec))", + "got": "dec)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "mnemonic.WordsCount())", + "got": "len(mnemonic))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5555555555555556 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "bip32_key.ToExtended())", + "got": "bip32_key.ToExtended())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "binascii.hexlify(seed))", + "got": "seed)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.35714285714285715 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "mnemonic_len_no_chksum)", + "got": "mnemonic_len_no_chksum)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "shelley_key.ToAddress())", + "got": "shelley_key.AddressKey().Address())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7457627118644068 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "priv_key.Raw().ToBytes())", + "got": "priv_key.Raw().ToBytes())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "binascii.hexlify(seed_hd))", + "got": "seed_hd)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.47058823529411764 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "electrum_v1.IsPublicOnly())", + "got": "electrum_v1.IsPublicKeyOnly())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9473684210526315 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "mnemonic_len_no_chksum + 1)", + "got": "mnemonic_len_no_chksum + 1)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "binascii.hexlify(seed_priv))", + "got": "binascii.hexlify(seed_priv))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "Bip32KeyIndex.HardenIndex(1))", + "got": "Bip32KeyIndex.HARDENED_INDEX_1)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "electrum_v2.GetAddress(0, i))", + "got": "electrum_v2.GetAddress(0, i))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "substrate_obj.IsPublicOnly())", + "got": "substrate_obj.IsWatchOnly())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8421052631578947 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "bip32_ctx.ChainCode().ToHex())", + "got": "bip32_ctx.ChainCode().ToHex())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "byron_legacy.GetAddress(0, i))", + "got": "byron_legacy.GetAddress(0, i).ToHex())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8823529411764706 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "shelley_key.ToRewardAddress())", + "got": "shelley_key.StakingAddress())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7796610169491526 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "binascii.hexlify(address_hash))", + "got": "address_hash)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5909090909090909 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "ton.PrivateKey().Raw().ToHex())", + "got": "binascii.hexlify(ton.PrivateKey().Raw().ToBytes()))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7073170731707317 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "bip_ctx.PublicKey().ToExtended())", + "got": "bip_ctx.PublicKey().ToExtended())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "bip_mst_ctx.PrivateKey().ToWif())", + "got": "bip_mst_ctx.PrivateKey().ToWif())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "pub_skey.RawCompressed().ToHex())", + "got": "pub_skey)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.42857142857142855 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "pub_vkey.RawCompressed().ToHex())", + "got": "pub_vkey.RawCompressed().ToHex())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "bip_ctx.PrivateKey().ToExtended())", + "got": "bip_ctx.PrivateKey().ToExtended())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "pub_key.RawCompressed().ToBytes())", + "got": "pub_key.RawCompressed().ToBytes())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "bip32_ctx.PublicKey().ToExtended())", + "got": "bip32_ctx.PublicKey().ToBytes())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8656716417910447 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "bip_ctx.PrivateKey().Raw().ToHex())", + "got": "bip_ctx.PrivateKey().Raw().ToHex())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "bip_ex_ctx.PublicKey().ToExtended())", + "got": "bip_ex_ctx.PublicKey().ToExtended())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "brainwallet.PublicKey().ToAddress())", + "got": "brainwallet.Address())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7586206896551724 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "pub_key.RawUncompressed().ToBytes())", + "got": "pub_key.RawUncompressed().ToBytes())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "bip32_ctx.PrivateKey().Raw().ToHex())", + "got": "bip32_ctx.PrivateKey())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7666666666666667 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "(ss58_format, binascii.hexlify(dec)))", + "got": "(dec))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.27906976744186046 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "bip32_ctx.ParentFingerPrint().ToHex())", + "got": "bip32_ctx.ParentFprint().ToHex())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9014084507042254 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "substrate_ctx.PublicKey().ToAddress())", + "got": "substrate_ctx.Address())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7741935483870968 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "brainwallet.PrivateKey().Raw().ToHex())", + "got": "brainwallet.PrivateKey())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.78125 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "shelley_key.AddressKey().Raw().ToHex())", + "got": "shelley_key.PrivateKey())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.625 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "shelley_key.StakingKey().Raw().ToHex())", + "got": "shelley_key.StakingKey().Raw().ToHex())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "EllipticCurveGetter.FromType(curve_type))", + "got": "EllipticCurveGetter.FromType(curve_type))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "shelley_addr_ctx.PublicKeys().ToAddress())", + "got": "shelley_addr_ctx.PublicKeys().ToAddress())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "binascii.hexlify(scale_enc.Encode(value)))", + "got": "scale_enc.encode(value))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.696969696969697 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "fprint.IsMasterKey())", + "got": "fprint.IsMasterKey())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "deser_key.IsPublic())", + "got": "deser_key.IsPublic())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "bip_ctx.IsPublicOnly())", + "got": "bip_ctx.IsPublicOnly())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "bip32_ctx.IsPublicOnly())", + "got": "bip32_ctx.IsPrivate())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.723404255319149 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "bip_ex_ctx.IsPublicOnly())", + "got": "bip_ex_ctx.PrivateKey().IsExtended())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5396825396825397 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "electrum_v2.IsPublicOnly())", + "got": "electrum_v2.IsPrivate())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7450980392156863 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "Bip32Utils.IsHardenedIndex(1))", + "got": "Bip32Utils.IsHardenedIndex(0))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9666666666666667 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "shelley_acc_ctx.IsPublicOnly())", + "got": "shelley_acc_ctx.IsPublicOnly())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "TonMnemonicValidator().IsValid(mnemonic))", + "got": "TonMnemonicValidator().IsValid(mnemonic))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "Bip32Slip10Ed25519.IsPublicDerivationSupported())", + "got": "self._is_supported_derivation(\"m/0'/2147483647'/1'/2147483646'/2'\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2564102564102564 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "TonMnemonicValidator().IsValid(mnemonic, \"test\"))", + "got": "TonMnemonicValidator().IsValid(mnemonic))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9111111111111111 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "TonMnemonicValidator(lang).IsValid(test[\"mnemonic\"]))", + "got": "TonMnemonicValidator(lang).IsValid(test[\"mnemonic\"]))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "Bip39MnemonicValidator(lang).IsValid(test[\"mnemonic\"]))", + "got": "Bip39MnemonicValidator(lang).IsValid(test[\"mnemonic\"]))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "Bip32Slip10Ed25519Blake2b.IsPublicDerivationSupported())", + "got": "self.bip32.slip10_ed25519_bls12381.is_supported_derivation())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5128205128205128 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "Ed25519PublicKey.IsValidBytes(binascii.unhexlify(test)))", + "got": "Ed25519PublicKey.FromBytes(TEST_VECT_ED25519_PUB_KEY_INVALID[0]).IsValid())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4122137404580153 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "Sr25519PublicKey.IsValidBytes(binascii.unhexlify(test)))", + "got": "Sr25519PublicKey.IsValidBytes(binascii.unhexlify(test)))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "MoneroMnemonicValidator(lang).IsValid(test[\"mnemonic\"]))", + "got": "MoneroMnemonicValidator(lang).IsValid(test[\"mnemonic\"]))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "Ed25519PrivateKey.IsValidBytes(binascii.unhexlify(test)))", + "got": "Ed25519PrivateKey.IsValidBytes(binascii.unhexlify(test)))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/sense2vec", + "expected": "0", + "got": "2", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "explosion/sense2vec", + "expected": "1", + "got": "1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/sense2vec", + "expected": "2", + "got": "2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/sense2vec", + "expected": "10", + "got": "10", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/sense2vec", + "expected": "[]", + "got": "[\"a|A\", \"b|A\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.25 + }, + { + "repo": "explosion/sense2vec", + "expected": "123", + "got": "123", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/sense2vec", + "expected": "1.0", + "got": "1.0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/sense2vec", + "expected": "s2v", + "got": "s2v.most_similar(\"beekeepers\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.18181818181818182 + }, + { + "repo": "explosion/sense2vec", + "expected": "2.0", + "got": "2.0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/sense2vec", + "expected": "3.0", + "got": "3.0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/sense2vec", + "expected": "456", + "got": "456", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/sense2vec", + "expected": "0.8", + "got": "0.5", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "explosion/sense2vec", + "expected": "0.0", + "got": "0.0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/sense2vec", + "expected": "\"I\"", + "got": "\"I\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/sense2vec", + "expected": "True", + "got": "True", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/sense2vec", + "expected": "None", + "got": "0", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "explosion/sense2vec", + "expected": "\"hll\"", + "got": "\"hll\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/sense2vec", + "expected": "False", + "got": "False", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/sense2vec", + "expected": "\"a|A\"", + "got": "\"a|A\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/sense2vec", + "expected": "\"you\"", + "got": "\"You\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "explosion/sense2vec", + "expected": "\"wrld\"", + "got": "\"world\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9230769230769231 + }, + { + "repo": "explosion/sense2vec", + "expected": "\"cool\"", + "got": "\"cool\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/sense2vec", + "expected": "[\"b|A\"]", + "got": "[\"b|A\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/sense2vec", + "expected": "expected", + "got": "expected", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/sense2vec", + "expected": "\"duck|VERB\"", + "got": "\"duck|NOUN\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6363636363636364 + }, + { + "repo": "explosion/sense2vec", + "expected": "\"duck|NOUN\"", + "got": "\"duck|NOUN\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/sense2vec", + "expected": "other_senses", + "got": "other_senses", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/sense2vec", + "expected": "(word, sense)", + "got": "(word, sense)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/sense2vec", + "expected": "\"beekeepers|NOUN\"", + "got": "\"beekeepers|NOUN\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/sense2vec", + "expected": "\"honey_bees|NOUN\"", + "got": "\"beekeepers|NOUN\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6470588235294118 + }, + { + "repo": "explosion/sense2vec", + "expected": "\"Beekeepers|NOUN\"", + "got": "\"beekeeper|NOUN\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "explosion/sense2vec", + "expected": "(\"honey bees\", \"NOUN\")", + "got": "\"beekeepers|NOUN\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5128205128205128 + }, + { + "repo": "explosion/sense2vec", + "expected": "KeyError)", + "got": "KeyError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/sense2vec", + "expected": "ValueError)", + "got": "KeyError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7 + }, + { + "repo": "explosion/sense2vec", + "expected": "1.0)", + "got": "0.9)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "explosion/thinc", + "expected": "0", + "got": "eps", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "explosion/thinc", + "expected": "2", + "got": "2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "1", + "got": "1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "3", + "got": "3", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "5", + "got": "5", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "6", + "got": "len(model.layers)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "explosion/thinc", + "expected": "4", + "got": "4", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "[]", + "got": "numpy.zeros((0, 0))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "explosion/thinc", + "expected": "xp", + "got": "xp", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "64", + "got": "64", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "\"\"", + "got": "\"None\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "explosion/thinc", + "expected": "10", + "got": "10", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "{}", + "got": "{}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "0.2", + "got": "0.2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "0.0", + "got": "pytest.approx(0, eps)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "explosion/thinc", + "expected": "1.0", + "got": "1.0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "100", + "got": "2.0", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "explosion/thinc", + "expected": "123", + "got": "123", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "\"a\"", + "got": "\"a\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "rate", + "got": "rate", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "None", + "got": "None", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "1000", + "got": "64", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "explosion/thinc", + "expected": "1337", + "got": "1337", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "prev", + "got": "prev", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "True", + "got": "True", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "\"ab\"", + "got": "\"a+b\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "explosion/thinc", + "expected": "relu", + "got": "relu", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "0.001", + "got": "decaying", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "explosion/thinc", + "expected": "(12,)", + "got": "(12,)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "rate2", + "got": "rate2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "rate0", + "got": "rate0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "rate1", + "got": "rate1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "data2", + "got": "data2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "\"foo\"", + "got": "\"foo\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "width", + "got": "width", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "\"FOO\"", + "got": "\"FOO\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "shape", + "got": "shape", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "dtype", + "got": "dtype", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "numpy", + "got": "numpy", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "(nH,)", + "got": "(nH,)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "(nO,)", + "got": "(nO,)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "loss2", + "got": "loss2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "loss1", + "got": "loss1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "child", + "got": "child", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "relu2", + "got": "relu2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "b\"foo\"", + "got": "\"foo\".encode(\"utf8\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.38461538461538464 + }, + { + "repo": "explosion/thinc", + "expected": "(2, 3)", + "got": "(2,)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "explosion/thinc", + "expected": "len(Y)", + "got": "len(Y)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "n_args", + "got": "n_args", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "(5, 4)", + "got": "(5,)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "explosion/thinc", + "expected": "(2, 0)", + "got": "(5, 0)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "explosion/thinc", + "expected": "(5, 0)", + "got": "(2, 0)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "explosion/thinc", + "expected": "Y.size", + "got": "Y.size", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "\"cupy\"", + "got": "\"cupy\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "\"noop\"", + "got": "\"clone\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "explosion/thinc", + "expected": "Y.ndim", + "got": "Y.ndim", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "answer", + "got": "answer", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "\"cool\"", + "got": "\"cool\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "\"gelu\"", + "got": "\"gelu\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "[True]", + "got": "[True]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "OUTPUT", + "got": "OUTPUT", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "(2, 45)", + "got": "(2, 45)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "(1, 45)", + "got": "(1, 45)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "(0, 10)", + "got": "(0,)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7272727272727273 + }, + { + "repo": "explosion/thinc", + "expected": "X.shape", + "got": "X.shape", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "\"numpy\"", + "got": "\"numpy\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "x_thinc", + "got": "None", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.18181818181818182 + }, + { + "repo": "explosion/thinc", + "expected": "len(Xs)", + "got": "2", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "explosion/thinc", + "expected": "summary", + "got": "summary", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "tuple()", + "got": "(\"W\",)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "explosion/thinc", + "expected": "y_debug", + "got": "y_debug", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "chained", + "got": "relu", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.18181818181818182 + }, + { + "repo": "explosion/thinc", + "expected": "t.shape", + "got": "(10, 2)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "explosion/thinc", + "expected": "lr_next1", + "got": "lr", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "explosion/thinc", + "expected": "lr_next2", + "got": "lr_next2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "\"uint32\"", + "got": "numpy.uint64", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "explosion/thinc", + "expected": "n_params", + "got": "n_params", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "dY_thinc", + "got": "x_thinc", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "explosion/thinc", + "expected": "seen_ids", + "got": "seen_ids", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "(nH, nI)", + "got": "(nH, nI)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "\"linear\"", + "got": "\"linear\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "(nN, nO)", + "got": "X.shape", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "explosion/thinc", + "expected": "(nN, nI)", + "got": "(nN, nI)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "\"float32\"", + "got": "\"float32\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "len(flat)", + "got": "len(flat)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "row.sum()", + "got": "row.sum()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "losses[0]", + "got": "1e-3", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.15384615384615385 + }, + { + "repo": "explosion/thinc", + "expected": "scores[0]", + "got": "scores[0]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "seen_keys", + "got": "seen_keys", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "len(data)", + "got": "2", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "explosion/thinc", + "expected": "model2.id", + "got": "model2.id", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "vec1.shape", + "got": "vec1.shape", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "learn_rate", + "got": "learn_rate", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "len(data2)", + "got": "len(data2)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "\"shimdata\"", + "got": "\"shimdata\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "(\"a\", \"b\")", + "got": "[\"a\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "explosion/thinc", + "expected": "len(shape)", + "got": "i + 1", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "explosion/thinc", + "expected": "byte_order", + "got": "byte_order", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "data.shape", + "got": "data.shape", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "(1000, 64)", + "got": "(1000, 64)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "INPUT_FLAT", + "got": "INPUT_FLAT", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "expected[i]", + "got": "expected[i]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "model.attrs", + "got": "model.attrs", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "((\"a\", 1),)", + "got": "((\"a\", 1), (\"c\", 3))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7096774193548387 + }, + { + "repo": "explosion/thinc", + "expected": "base_params", + "got": "base_params", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "Y.data.ndim", + "got": "Y.data.ndim", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "Xs[0].dtype", + "got": "\"i\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "explosion/thinc", + "expected": "(\"param1\",)", + "got": "(\"param1\",)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "new_ops.name", + "got": "new_ops.name", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "(ones, ones)", + "got": "ones", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "explosion/thinc", + "expected": "Xs[0].mean()", + "got": "1", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "explosion/thinc", + "expected": "Xs[1].mean()", + "got": "Xs[1].mean()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "len(strings)", + "got": "2", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "explosion/thinc", + "expected": "[True, True]", + "got": "[True, True]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "(\"nO\", \"nI\")", + "got": "(\"nI\", \"nO\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "explosion/thinc", + "expected": "expected_err", + "got": "expected_err", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "not GENERATE", + "got": "not os.path.exists(\".mypy_cache\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.17777777777777778 + }, + { + "repo": "explosion/thinc", + "expected": "guesses.shape", + "got": "labels.shape", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.64 + }, + { + "repo": "explosion/thinc", + "expected": "rate2 - rate1", + "got": "rate2 - rate1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "base_defaults", + "got": "base_defaults", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "vector2.sum()", + "got": "vector2.sum()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "(2, 1, 1, nO)", + "got": "((nO * 4) + (nO * 4 * nI))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.20512820512820512 + }, + { + "repo": "explosion/thinc", + "expected": "m2.to_bytes()", + "got": "m2.to_bytes()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "\"dummy(relu)\"", + "got": "\"dummy(concat)\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6428571428571429 + }, + { + "repo": "explosion/thinc", + "expected": "guesses1.shape", + "got": "guesses[0].shape", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8666666666666667 + }, + { + "repo": "explosion/thinc", + "expected": "guesses2.shape", + "got": "labels2.shape", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "explosion/thinc", + "expected": "gradient.shape", + "got": "gradient.shape", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "list(Xs[0][0])", + "got": "[1]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.23529411764705882 + }, + { + "repo": "explosion/thinc", + "expected": "list(Xs[1][0])", + "got": "list(Xs[0][1])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7857142857142857 + }, + { + "repo": "explosion/thinc", + "expected": "X.dataXd.shape", + "got": "X.dataXd.shape", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "list(Xs[0][1])", + "got": "list(Xs[0].max(axis=1))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6486486486486487 + }, + { + "repo": "explosion/thinc", + "expected": "list(Xs[1][1])", + "got": "list(Xs[1][0])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9285714285714286 + }, + { + "repo": "explosion/thinc", + "expected": "(dX == 0).all()", + "got": "not ops.xp.isnan(dX).any()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3902439024390244 + }, + { + "repo": "explosion/thinc", + "expected": "list(Xs[0][-1])", + "got": "list(Xs[0][-1])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "shim.to_bytes()", + "got": "shim.to_bytes()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "BACKPROP_OUTPUT", + "got": "BACKPROP_OUTPUT", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "model2.layers[2]", + "got": "model", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.47619047619047616 + }, + { + "repo": "explosion/thinc", + "expected": "[0, 4, 6, 14, 15]", + "got": "[0, 4, 6, 7, 11]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8484848484848485 + }, + { + "repo": "explosion/thinc", + "expected": "y.dtype.byteorder", + "got": "byte_order", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "explosion/thinc", + "expected": "y)", + "got": "y)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "Y)", + "got": "Y)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "Yt)", + "got": "Yt)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "Yu)", + "got": "Yu)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "dXt)", + "got": "dXt)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "dXu)", + "got": "dXu)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "cols)", + "got": "cols)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "data)", + "got": "data)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "True)", + "got": "is_train)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "explosion/thinc", + "expected": "truth)", + "got": "truth)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "ngrad)", + "got": "ngrad)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "unflat)", + "got": "unflat)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "d_seqs)", + "got": "d_seqs)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "in_data)", + "got": "in_data)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "values2)", + "got": "values2)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "values3)", + "got": "values3)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "outputs)", + "got": "outputs)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "expected)", + "got": "expected)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "out_data)", + "got": "out_data)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "Y_script)", + "got": "Y_script)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "predicted)", + "got": "predicted)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "p_nonzero)", + "got": "p_nonzero)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "[0.0, 0.0])", + "got": "0.0)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5333333333333333 + }, + { + "repo": "explosion/thinc", + "expected": "via_update)", + "got": "via_update)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "ragged.data)", + "got": "numpy.zeros((0, 4), dtype=\"f\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.09302325581395349 + }, + { + "repo": "explosion/thinc", + "expected": "reference[0])", + "got": "reference[0])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "scores[0, 1])", + "got": "0.0)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.23529411764705882 + }, + { + "repo": "explosion/thinc", + "expected": "X[i] * length)", + "got": "X[i])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5263157894736842 + }, + { + "repo": "explosion/thinc", + "expected": "Y_not_batched)", + "got": "Y_not_batched)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "target_scores)", + "got": "target_scores)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "predicted_best)", + "got": "predicted_best)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "cols_gpu.get())", + "got": "cols_gpu)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.75 + }, + { + "repo": "explosion/thinc", + "expected": "[0.0, 1.0, 3.0])", + "got": "[1.0])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5454545454545454 + }, + { + "repo": "explosion/thinc", + "expected": "[1.0, 3.0, 4.0])", + "got": "[1.0, 3.0, 4.0])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "[3.0, 4.0, 5.0])", + "got": "[3.0, 4.0, 5.0])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "reference[1][2])", + "got": "reference[2])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.896551724137931 + }, + { + "repo": "explosion/thinc", + "expected": "reference[1][1])", + "got": "reference[1][1])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "expected_output)", + "got": "expected_output)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "ragged.data[4:6])", + "got": "ragged.data[i])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.875 + }, + { + "repo": "explosion/thinc", + "expected": "d_seqs_gpu.get())", + "got": "d_seqs_gpu.get())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "target_scores[0])", + "got": "target_scores[0])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "ops.alloc2f(10, 2))", + "got": "t)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.09523809523809523 + }, + { + "repo": "explosion/thinc", + "expected": "[[1.0], [1.0], [1.0]])", + "got": "numpy.zeros((3, 1)))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.23809523809523808 + }, + { + "repo": "explosion/thinc", + "expected": "table[indices].sum(1))", + "got": "table[indices])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8108108108108109 + }, + { + "repo": "explosion/thinc", + "expected": "ops.asarray1i([2, 4]))", + "got": "ops.asarray1i([2, 4]))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "fwd_via_predict_batch)", + "got": "fwd_via_predict_batch)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "ops.asarray1i([0, 3, 5]))", + "got": "[0, 3, 5])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "explosion/thinc", + "expected": "[[1.0, 6.0], [4.0, 9.0]])", + "got": "[1.0, 6.0, 2.0, 7.0, 3.0])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7450980392156863 + }, + { + "repo": "explosion/thinc", + "expected": "0)", + "got": "0)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "5)", + "got": "5, eps)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "explosion/thinc", + "expected": "KeyError)", + "got": "AttributeError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5833333333333334 + }, + { + "repo": "explosion/thinc", + "expected": "TypeError)", + "got": "TypeError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "ValueError)", + "got": "ValueError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "IndexError)", + "got": "ValueError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6363636363636364 + }, + { + "repo": "explosion/thinc", + "expected": "ValidationError)", + "got": "ValidationError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "DataValidationError)", + "got": "DataValidationError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "ValueError, match=error)", + "got": "TypeError, match=error)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.851063829787234 + }, + { + "repo": "explosion/thinc", + "expected": "ValueError, match=\"Invalid order\")", + "got": "ValueError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4888888888888889 + }, + { + "repo": "explosion/thinc", + "expected": "ValueError, match=r\"lengths must be\")", + "got": "IndexError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2916666666666667 + }, + { + "repo": "explosion/thinc", + "expected": "ValueError, match=r\"not supported.*1.9.0\")", + "got": "ImportError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "explosion/thinc", + "expected": "ValueError, match=r\"Cannot calculate.*truth\")", + "got": "ValueError, match=r\"Cannot calculate.*targets\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9347826086956522 + }, + { + "repo": "invoice-x/invoice2data", + "expected": "0)", + "got": "1)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "invoice-x/invoice2data", + "expected": "\"b\")", + "got": "\"b\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "invoice-x/invoice2data", + "expected": "len(test_list))", + "got": "len(test_list))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "invoice-x/invoice2data", + "expected": "\"Lowercase test failed\")", + "got": "\"lowercase test failed\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9583333333333334 + }, + { + "repo": "invoice-x/invoice2data", + "expected": "\"remove whitespace test failed\")", + "got": "\"Remove whitespace function failed, output not equal\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6511627906976745 + }, + { + "repo": "invoice-x/invoice2data", + "expected": "\"Remove accents function failed, output not equal\")", + "got": "\"Remove accents not working\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.55 + }, + { + "repo": "invoice-x/invoice2data", + "expected": "[ \"WARNING:invoice2data.extract.loader:JSON Loader Failed to load template stream\\nExpecting value: line\" \" 1 column 1 (char 0)\" ])", + "got": "[\"DEBUG:__main__:Invalid JSON stream\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.24705882352941178 + }, + { + "repo": "invoice-x/invoice2data", + "expected": "SystemExit)", + "got": "SystemExit)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "invoice-x/invoice2data", + "expected": "have)", + "got": "have)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "invoice-x/invoice2data", + "expected": "True)", + "got": "isinstance(tesseract.to_text(file), str))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.13043478260869565 + }, + { + "repo": "invoice-x/invoice2data", + "expected": "False)", + "got": "False)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "invoice-x/invoice2data", + "expected": "res == ref_json)", + "got": "all(res[i] == ref_json[i] for i in range(len(res))))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.47058823529411764 + }, + { + "repo": "invoice-x/invoice2data", + "expected": "type(res) is str)", + "got": "isinstance(res, dict))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5128205128205128 + }, + { + "repo": "invoice-x/invoice2data", + "expected": "type(res) is dict)", + "got": "isinstance(res, dict))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6 + }, + { + "repo": "invoice-x/invoice2data", + "expected": "isinstance(res, dict))", + "got": "res)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "invoice-x/invoice2data", + "expected": "os.path.exists(test_file))", + "got": "os.path.exists(test_file))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "invoice-x/invoice2data", + "expected": "os.path.exists(file_path))", + "got": "os.path.exists(file_path))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "invoice-x/invoice2data", + "expected": "all( os.path.exists(os.path.join(copy_dir, v[\"output_fname\"])) for k, v in data.items() ))", + "got": "os.path.exists(os.path.join(copy_dir, \"2015-07-02.pdf\")))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6122448979591837 + }, + { + "repo": "invoice-x/invoice2data", + "expected": "2", + "got": "2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "invoice-x/invoice2data", + "expected": "0", + "got": "0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "invoice-x/invoice2data", + "expected": "[]", + "got": "[]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "invoice-x/invoice2data", + "expected": "\"\u00e4\"", + "got": "\"test\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "invoice-x/invoice2data", + "expected": "True", + "got": "True", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "invoice-x/invoice2data", + "expected": "False", + "got": "False", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "invoice-x/invoice2data", + "expected": "\"test text\"", + "got": "\"test text\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "invoice-x/invoice2data", + "expected": "\"cached text\"", + "got": "\"cached text\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "invoice-x/invoice2data", + "expected": "[\"Basic Test\"]", + "got": "\"keyword\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2608695652173913 + }, + { + "repo": "invoice-x/invoice2data", + "expected": "qty_templ_files", + "got": "qty_templ_files", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "invoice-x/invoice2data", + "expected": "[\"Exclude_this\"]", + "got": "[\"Basic Test\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "invoice-x/invoice2data", + "expected": "qty_templ_files + 2", + "got": "qty_templ_files", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8823529411764706 + }, + { + "repo": "invoice-x/invoice2data", + "expected": "\"thisnameisimportant.yml\"", + "got": "\"thisnameisimportant\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9130434782608695 + }, + { + "repo": "invoice-x/invoice2data", + "expected": "all(isinstance(template, InvoiceTemplate) for template in templates)", + "got": "all(os.path.exists(tpl) for tpl in templates)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6194690265486725 + }, + { + "repo": "invoice-x/invoice2data", + "expected": "\"test.pdf\")", + "got": "\"test.pdf\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "invoice-x/invoice2data", + "expected": "\"cloud-vision-84893\")", + "got": "\"cloud-vision-84893\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "invoice-x/invoice2data", + "expected": "\"test/output-1-to-1.json\")", + "got": "\"output-1-to-1.json\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8936170212765957 + }, + { + "repo": "invoice-x/invoice2data", + "expected": "have)", + "got": "have)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "2", + "got": "2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "1", + "got": "2", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "0", + "got": "0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "4", + "got": "2", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "1988", + "got": "10", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "0.99", + "got": ".99", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "\".xml\"", + "got": "\"_details.xml\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "\".html\"", + "got": "\".htm\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9230769230769231 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "expected", + "got": "expected", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "Path.cwd()", + "got": "\"./\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.14285714285714285 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "after_date", + "got": "after_date", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "custom_path", + "got": "custom_path", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "before_date", + "got": "before_date", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "\"0000320193\"", + "got": "\"0000320193\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "is_cik(789019)", + "got": "not is_cik(\"000078901\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7027027027027027 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "not is_cik(\"\")", + "got": "not is_cik(\"00007890190\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.717948717948718 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "len(list(forms))", + "got": "1", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "is_cik(\"789019\")", + "got": "not is_cik(\"000078901\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7692307692307693 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "Path(\"folder-foo\")", + "got": "Path(\"folder-foo\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "save_path.exists()", + "got": "save_path.exists()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "json_path.exists()", + "got": "json_path.exists()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "str(exc_info.value)", + "got": "str(exc_info.value)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "expected_num_results", + "got": "expected_num_results", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "result_from_datetime", + "got": "\"2023-01-01\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "not save_path.exists()", + "got": "not save_path.exists()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "\"0000320193-22-000108\"", + "got": "\"0000320193-22-000108\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "\"0000950103-23-014290\"", + "got": "\"0000950103-23-014290\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "datetime(1994, 1, 1).date()", + "got": "dt.strftime(DATE_FORMAT_TOKENS)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.27586206896551724 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "directory_is_empty(dl_path)", + "got": "directory_is_empty(dl_path)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "all(form.stat() > 0 for form in forms)", + "got": "downloaded_file_path.exists()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.14925373134328357 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "mocked_fetch.call_args_list[1].args[0].cik", + "got": "\"0000320193\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.037037037037037035 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "within_requested_date_range(download_metadata, \"2022-1-1\")", + "got": "within_requested_date_range(download_metadata)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8846153846153846 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "not within_requested_date_range(download_metadata, \"2020-1-1\")", + "got": "not within_requested_date_range(download_metadata, \"2023-1-1\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9838709677419355 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "\"https://www.sec.gov/Archives/edgar/data/320193/000032019322000108/ny20007635x4_8k.htm\"", + "got": "\"https://www.sec.gov/Archives/edgar/data/320193/000032019322000108/wf-form8k_168444912415136.html\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "\"https://www.sec.gov/Archives/edgar/data/320193/000032019322000108/0000320193-22-000108.txt\"", + "got": "\"https://www.sec.gov/Archives/edgar/data/320193/000032019322000108/4333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3670212765957447 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "\"https://www.sec.gov/Archives/edgar/data/312070/000095010323014290/0000950103-23-014290.txt\"", + "got": "\"https://www.sec.gov/Archives/edgar/data/0000312070/000095010323014290/dp200734_424b2-5333pps.htm\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7894736842105263 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "Path( \"./sec-edgar-filings/AAPL/10-K/0000320193-22-000108/foobar.txt\" )", + "got": "Path(\"./sec-edgar-filings/AAPL/0000320193/10-K/0000320193-22-000108/foobar.txt\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9139072847682119 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "Path( \"./sec-edgar-filings/0000320193/10-K/0000320193-22-000108/foobar.txt\" )", + "got": "\"./foobar.txt\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "{ \"AAPL\": \"0000320193\", \"MSFT\": \"0000789019\", \"GOOGL\": \"0001652044\", \"AMZN\": \"0001018724\", }", + "got": "sample_cik_ticker_payload", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "TypeError)", + "got": "TypeError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "ValueError)", + "got": "ValueError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jadchaar/sec-edgar-downloader", + "expected": "RequestException)", + "got": "RequestException)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "0", + "got": "0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "1", + "got": "1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "2", + "got": "3", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "5", + "got": "5", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "3", + "got": "3", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "16", + "got": "16", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "12", + "got": "12", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "35", + "got": "35", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "{}", + "got": "{}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "60", + "got": "60", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"\"", + "got": "\"Cannot connect\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "25", + "got": "25.0", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "50", + "got": "50", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "[]", + "got": "[]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "0.1", + "got": "0.1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "id2", + "got": "id2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "1.5", + "got": "1.5", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "0.5", + "got": "0.5", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "123", + "got": "123", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "0.0", + "got": "0.0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "5.5", + "got": "5.5", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "2.3", + "got": "2.3", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "3.2", + "got": "3.2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "5.0", + "got": "5.0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "1.0", + "got": "1.5", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "890", + "got": "1012.25", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "100", + "got": "100", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "45.0", + "got": "37.7749", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.18181818181818182 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "True", + "got": "True", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "None", + "got": "None", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "32.0", + "got": "pytest.approx(32.0)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.34782608695652173 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "hass", + "got": "hass", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "data", + "got": "\"hello\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "0.05", + "got": "0.05", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "25.0", + "got": "25.0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "30.0", + "got": "30.0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "20.0", + "got": "20.0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "0.01", + "got": "0.0001", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "1000", + "got": "1000", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "1020", + "got": "0", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "18.0", + "got": "18.0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "False", + "got": "False", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "value", + "got": "value", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "150.0", + "got": "150.0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "200.0", + "got": "200.0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"auth\"", + "got": "\"invalid_api_key\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2608695652173913 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"init\"", + "got": "\"step2\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "result", + "got": "result", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"form\"", + "got": "\"form\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "DOMAIN", + "got": "\"/panels/smart_irrigation\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "config", + "got": "config", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "method", + "got": "method", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "1013.25", + "got": "1000", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "51.5074", + "got": "51.5074", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "trigger", + "got": "trigger", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"07:30\"", + "got": "\"07:30\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "azimuth", + "got": "azimuth", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"step1\"", + "got": "\"weather_service\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.25 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"value\"", + "got": "\"value\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "message", + "got": "message", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"world\"", + "got": "\"world\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"Hello\"", + "got": "\"Hello\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"Hallo\"", + "got": "\"Hallo\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "session", + "got": "session", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "api_key", + "got": "api_key", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"result\"", + "got": "\"result\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "str(exc)", + "got": "str(exc)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "repr_str", + "got": "repr_str", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "zone_ids", + "got": "zone_ids", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "estimate", + "got": "estimate", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"success\"", + "got": "\"success\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "hass.data", + "got": "hass.data", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"Enabled\"", + "got": "\"Enabled\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "PANEL_URL", + "got": "\"/panels/\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "entity_id", + "got": "entity_id", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "zone_data", + "got": "zone_data", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "const.NAME", + "got": "\"Test Smart Irrigation\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.12121212121212122 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "user_input", + "got": "user_input", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "mock_store", + "got": "mock_store", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"00:00:00\"", + "got": "\"00:00:00\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "zone_names", + "got": "zone_names", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "module_ids", + "got": "module_ids", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "month_data", + "got": "month_data", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "sync_config", + "got": "sync_config", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "caplog.text", + "got": "caplog.text", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "api_version", + "got": "api_version", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"Test Zone\"", + "got": "\"Test Zone\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"Side Yard\"", + "got": "\"Side Yard\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"Daily Test\"", + "got": "\"Daily Test\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"[redacted]\"", + "got": "\"***\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.23529411764705882 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "long_message", + "got": "long_message", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "module_names", + "got": "module_names", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"Auth failed\"", + "got": "\"Auth failed\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"InvalidAuth\"", + "got": "\"InvalidAuth\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "expected_data", + "got": "expected_data", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "error_message", + "got": "error_message", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"Hello World\"", + "got": "\"Hello World\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"old_api_key\"", + "got": "\"old_api_key\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "mapping_names", + "got": "mapping_names", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"wind_sensor\"", + "got": "\"wind_sensor\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "updated_names", + "got": "updated_names", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "calendar_data", + "got": "calendar_data", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"Test message\"", + "got": "\"Test message\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"Summer Boost\"", + "got": "\"Summer Boost\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"async_result\"", + "got": "\"async_result\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"async_success\"", + "got": "\"async_success\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"valid_api_key\"", + "got": "\"valid_api_key\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "special_message", + "got": "special_message", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "expected_volume", + "got": "expected_volume", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "not invalid_keys", + "got": "not invalid_keys", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "result[\"config\"]", + "got": "result[\"config\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"test_operation\"", + "got": "\"test_operation\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"Updated Module\"", + "got": "\"Updated Module\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "mock_config_entry", + "got": "mock_config_entry", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"nonexistent.key\"", + "got": "\"nonexistent.key\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"options_api_key\"", + "got": "\"options_api_key\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "performance_timer", + "got": "performance_timer", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"New Test Module\"", + "got": "\"New Test Module\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "initial_count - 1", + "got": "initial_count - 1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"Updated Schedule\"", + "got": "\"Updated Schedule\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "pressure_sea_level", + "got": "pressure_sea_level", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"validate_api_key\"", + "got": "\"test_api_key\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.75 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "{\"hello\": \"world\"}", + "got": "{\"nonexistent\": \"nonexistent\"}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4166666666666667 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "not missing_fields", + "got": "not missing_fields", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "FlowResultType.FORM", + "got": "\"form\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.08 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "US_CUSTOMARY_SYSTEM", + "got": "US_CUSTOMARY_SYSTEM", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"Connection failed\"", + "got": "\"Connection failed\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "TRIGGER_TYPE_SUNRISE", + "got": "TRIGGER_TYPE_SUNRISE", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "FlowResultType.ABORT", + "got": "FlowResultType.ABORT", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "datetime.datetime.min", + "got": "None", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.08 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"api_key_with_spaces\"", + "got": "\"api_key_with_spaces\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "isinstance(zones, list)", + "got": "isinstance(zones, list)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"Authentication failed\"", + "got": "\"Authentication failed\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "isinstance(config, dict)", + "got": "config", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "parsed_timestamps[i + 1]", + "got": "parsed_timestamps[i + 1]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "CONF_WEATHER_SERVICE_OWM", + "got": "CONF_WEATHER_SERVICE_OWM", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "mock_zone_config[\"name\"]", + "got": "\"Test Zone\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.22857142857142856 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "isinstance(result, bool)", + "got": "not result", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.47058823529411764 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "const.SCHEDULE_TYPE_DAILY", + "got": "const.SCHEDULE_TYPE_DAILY", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"single_instance_allowed\"", + "got": "\"single_instance\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8095238095238095 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "isinstance(modules, list)", + "got": "isinstance(modules, list)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "isinstance(exc, Exception)", + "got": "isinstance(exc, CannotConnect)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7142857142857143 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "isinstance(mappings, list)", + "got": "isinstance(mappings, list)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "f\"Updated {original_name}\"", + "got": "f\"Updated {original_name}\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "isinstance(pressure, float)", + "got": "pressure", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.45714285714285713 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "id1.startswith(\"schedule_\")", + "got": "id1.startswith(\"schedule_\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "id2.startswith(\"schedule_\")", + "got": "id2.startswith(\"schedule_\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "FlowResultType.CREATE_ENTRY", + "got": "FlowResultType.FORM", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6956521739130435 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "isinstance(absolute, float)", + "got": "absolute < 1000.1", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4090909090909091 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "result[\"data_schema\"].schema", + "got": "result[\"data_schema\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "id1.startswith(\"adjustment_\")", + "got": "id1.startswith(\"adjustment_\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "id2.startswith(\"adjustment_\")", + "got": "id2.startswith(\"adjustment_\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "[\"Front Lawn\", \"Back Garden\"]", + "got": "\"Test Zone\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "pytest.approx(898.7, rel=1e-1)", + "got": "pytest.approx(964.07, rel=1e-2)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8852459016393442 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"This is a string, not a dict\"", + "got": "\"This is a string, not a dict\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "const.CONF_WEATHER_SERVICE_OWM", + "got": "const.CONF_WEATHER_SERVICE_OWM", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "not iu_integration.is_enabled()", + "got": "not iu_integration.enabled", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9122807017543859 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "isinstance(calendar_data, dict)", + "got": "not calendar_data", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.625 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "isinstance(weather_records, list)", + "got": "all(isinstance(record, dict) for record in weather_records)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5869565217391305 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "mock_connection.send_result.called", + "got": "mock_connection.send_result.called", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "issubclass(InvalidAuth, Exception)", + "got": "issubclass(InvalidAuth, Exception)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "[\"temp_sensor\", \"humidity_sensor\"]", + "got": "[\"Front Lawn\", \"Back Garden\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.38095238095238093 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "[\"Test Module 1\", \"Test Module 2\"]", + "got": "[\"module1\", \"module2\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7142857142857143 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"switch.irrigation_unlimited_c1_z1\"", + "got": "\"switch.irrigation_unlimited_c1_z1\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "isinstance(absolute_pressure, float)", + "got": "absolute_pressure > relative_pressure", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5753424657534246 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "pytest.approx(expected_mm, rel=1e-3)", + "got": "pytest.approx(expected_mm)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8387096774193549 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "CONF_DEFAULT_DAYS_BETWEEN_IRRIGATION", + "got": "CONF_DEFAULT_DAYS_BETWEEN_IRRIGATION", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "issubclass(CannotConnect, Exception)", + "got": "issubclass(CannotConnect, Exception)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "pytest.approx(expected_psi, rel=1e-3)", + "got": "pytest.approx(expected_psi)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.84375 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "pytest.approx(expected_kmh, rel=1e-3)", + "got": "pytest.approx(expected_kmh)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.84375 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "pytest.approx(expected_mph, rel=1e-3)", + "got": "pytest.approx(expected_mph)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.84375 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "pytest.approx(expected_inhg, rel=1e-3)", + "got": "pytest.approx(expected_inhg)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8484848484848485 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "pytest.approx(expected_inch, rel=1e-3)", + "got": "pytest.approx(expected_inch)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8484848484848485 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "callable(websocket_get_watering_calendar)", + "got": "True", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.08888888888888889 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "mock_coordinator.handle_reset_bucket.called", + "got": "mock_coordinator.reset_bucket.called", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9113924050632911 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "{\"hello\": \"Hello\", \"nested\": {\"world\": \"World\"}}", + "got": "\"nonexistent\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.22950819672131148 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "TypeError)", + "got": "TypeError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "InvalidAuth)", + "got": "InvalidAuth) as exc_info:", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 0.6486486486486487 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "ClientError)", + "got": "CannotConnect)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "CannotConnect)", + "got": "CannotConnect)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "SmartIrrigationError)", + "got": "SmartIrrigationError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "(TypeError, ValueError))", + "got": "TypeError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5882352941176471 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "ValueError, match=\"Test error\")", + "got": "ValueError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5238095238095238 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "ValueError, match=\"Async test error\")", + "got": "ValueError, match=\"Async test error\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "(InvalidAuth, CannotConnect, OSError))", + "got": "InvalidAuth)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.48 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "ValueError, match=\"Invalid schedule type\")", + "got": "ValueError, match=\"Invalid schedule type\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "ValueError, match=\"Missing required field\")", + "got": "ValueError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4074074074074074 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "898.7, rel=1e-1)", + "got": "pressure_sea_level * 0.99)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.19047619047619047 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "expected_mm, rel=1e-3)", + "got": "expected_mm)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7058823529411765 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "expected_psi, rel=1e-3)", + "got": "expected_psi)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7222222222222222 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "expected_kmh, rel=1e-3)", + "got": "expected_kmh)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7222222222222222 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "hass)", + "got": "hass)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "DOMAIN)", + "got": "DOMAIN)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "user_input)", + "got": "options_flow)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.25 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "const.DOMAIN)", + "got": "\"smart_irrigation\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.1875 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "\"custom_components\")", + "got": "\"frontend/dist/smart-irrigation.js\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.32142857142857145 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "const.DOMAIN + \"_update_frontend\")", + "got": "\"unit_system_changed\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.35714285714285715 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "const.DOMAIN + \"_unit_system_changed\")", + "got": "\"unit_system_change\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.711864406779661 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "{const.CONF_DAYS_SINCE_LAST_IRRIGATION: 0})", + "got": "{const.CONF_DAYS_SINCE_LAST_IRRIGATION: 0})", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeroenterheerdt/HAsmartirrigation", + "expected": "{const.CONF_DAYS_SINCE_LAST_IRRIGATION: 2})", + "got": "{const.CONF_DAYS_SINCE_LAST_IRRIGATION: 2})", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "0", + "got": "0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "1", + "got": "False", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "2", + "got": "2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "3", + "got": "3", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "6", + "got": "10", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "60", + "got": "TEST_DURATION", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "15", + "got": "\"hour\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "None", + "got": "None", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "host", + "got": "host", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "\"hub\"", + "got": "\"hub\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "False", + "got": "False", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "result", + "got": "result", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "offset", + "got": "offset", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "option", + "got": "option", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "pressed", + "got": "pressed", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "\"device\"", + "got": "\"device\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "trans[1]", + "got": "trans[1]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "trans[2]", + "got": "trans[2]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "not result", + "got": "not result", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "\"finalize\"", + "got": "\"device\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "krahabb/meross_lan", + "expected": "percentage", + "got": "percentage", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "MLLightMp3", + "got": "MLLightMp3", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "credentials", + "got": "credentials", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "preset_mode", + "got": "preset_mode", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "ComponentApi", + "got": "ComponentApi", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "emulator.key", + "got": "emulator.key", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "entity.is_on", + "got": "not await entity.async_turn_on()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "krahabb/meross_lan", + "expected": "Mts960Climate", + "got": "Mts960Climate", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "MLLightEffect", + "got": "MLLightEffect", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "DHCP_GOOD_HOST", + "got": "DHCP_GOOD_HOST", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "expected_state", + "got": "expected_state", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "entity.min_temp", + "got": "entity.min_temp", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "entity.max_temp", + "got": "entity.max_temp", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "excpected_state", + "got": "excpected_state", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "consumptionstate", + "got": "consumptionstate", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "not entity.is_on", + "got": "not entity.is_on", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "entity.max_value", + "got": "entity.max_value", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "entity.min_value", + "got": "entity.min_value", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "entity.available", + "got": "True", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.1 + }, + { + "repo": "krahabb/meross_lan", + "expected": "MLEmulatedSwitch", + "got": "MLEmulatedSwitch", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "entity_hvac_modes", + "got": "entity_hvac_modes", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "{ColorMode.ONOFF}", + "got": "[ColorMode.BRIGHTNESS]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5641025641025641 + }, + { + "repo": "krahabb/meross_lan", + "expected": "entity.speed_count", + "got": "entity.is_on", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7333333333333333 + }, + { + "repo": "krahabb/meross_lan", + "expected": "not entity.enabled", + "got": "not entity_component_test.async_test_enabled_callback", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5070422535211268 + }, + { + "repo": "krahabb/meross_lan", + "expected": "type(entity_types)", + "got": "dict", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.18181818181818182 + }, + { + "repo": "krahabb/meross_lan", + "expected": "manager._trace_file", + "got": "manager.is_tracing", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7567567567567568 + }, + { + "repo": "krahabb/meross_lan", + "expected": "diagnostic[\"state\"]", + "got": "diagnostic[\"config\"][\"name\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6808510638297872 + }, + { + "repo": "krahabb/meross_lan", + "expected": "diagnostic[\"trace\"]", + "got": "diagnostic[\"state\"][\"config\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7083333333333334 + }, + { + "repo": "krahabb/meross_lan", + "expected": "round(0.1 * 16) / 16", + "got": "0.1", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2608695652173913 + }, + { + "repo": "krahabb/meross_lan", + "expected": "entity.hass_connected", + "got": "state.state != STATE_UNAVAILABLE", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.11320754716981132 + }, + { + "repo": "krahabb/meross_lan", + "expected": "not expected_entities", + "got": "not unexpected_entities", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9545454545454546 + }, + { + "repo": "krahabb/meross_lan", + "expected": "mlc.CONF_TRACE_VERSION", + "got": "ConfigFlow.VERSION", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.45 + }, + { + "repo": "krahabb/meross_lan", + "expected": "not device._trace_file", + "got": "device._trace_file", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9 + }, + { + "repo": "krahabb/meross_lan", + "expected": "ConfigEntryState.LOADED", + "got": "ConfigEntryState.LOADED", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "not manager._trace_file", + "got": "not manager._trace_file", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "haec.CoverState.OPENING", + "got": "haec.CoverState.OPENING", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "self.config_entry_loaded", + "got": "self.config_entry_loaded", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "entity.supported_features", + "got": "entity.supported_features", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "not entity.hass_connected", + "got": "not entity.hass_connected", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "await context.async_setup()", + "got": "context.config_entry", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5106382978723404 + }, + { + "repo": "krahabb/meross_lan", + "expected": "FlowResultType.CREATE_ENTRY", + "got": "FlowResultType.CREATE_ENTRY", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "isinstance(container, dict)", + "got": "isinstance(container, dict)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "isinstance(container, list)", + "got": "isinstance(container, list)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "ConfigEntryState.SETUP_RETRY", + "got": "ConfigEntryState.NOT_READY", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8148148148148148 + }, + { + "repo": "krahabb/meross_lan", + "expected": "isinstance(sub_digest, dict)", + "got": "isinstance(sub_digest, dict)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "isinstance(device, HubMixin)", + "got": "not unavailable_entities", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.19230769230769232 + }, + { + "repo": "krahabb/meross_lan", + "expected": "self.config_entry.data.keys()", + "got": "mlc.CONF_TRACE_CONFIG_KEYS", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.10909090909090909 + }, + { + "repo": "krahabb/meross_lan", + "expected": "isinstance(api, ComponentApi)", + "got": "isinstance(api, ComponentApi)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "RollerShutterMixin.SIGNALOPEN", + "got": "RollerShutterMixin.SIGNALOPEN", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "RollerShutterMixin.SIGNALCLOSE", + "got": "\"close\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.10810810810810811 + }, + { + "repo": "krahabb/meross_lan", + "expected": "state and state.state.isdigit()", + "got": "state", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2777777777777778 + }, + { + "repo": "krahabb/meross_lan", + "expected": "isinstance(entity, MLLightBase)", + "got": "entity.is_on", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.37209302325581395 + }, + { + "repo": "krahabb/meross_lan", + "expected": "tc.MOCK_PROFILE_CREDENTIALS_SIGNIN", + "got": "tc.MOCK_PROFILE_EMAIL", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6909090909090909 + }, + { + "repo": "krahabb/meross_lan", + "expected": "state := states.get(self.entity_id)", + "got": "haec.CoverEntityFeature.OPEN in entity.supported_features", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2608695652173913 + }, + { + "repo": "krahabb/meross_lan", + "expected": "tc.MOCK_CLOUDAPI_DEVICE_LATESTVERSION", + "got": "list(tc.MOCK_CLOUDAPI_DEVICE_LATESTVERSION.values())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8314606741573034 + }, + { + "repo": "krahabb/meross_lan", + "expected": "isinstance(entity_dnd, MLDNDLightEntity)", + "got": "entity_dnd", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "krahabb/meross_lan", + "expected": "1 - digest[mc.KEY_TOGGLEX][0][mc.KEY_ONOFF]", + "got": "not initialstate", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.13559322033898305 + }, + { + "repo": "krahabb/meross_lan", + "expected": "list(tc.MOCK_CLOUDAPI_DEVICE_DEVLIST.values())", + "got": "tc.MOCK_DEVICE_LIST", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5846153846153846 + }, + { + "repo": "krahabb/meross_lan", + "expected": "not hass.config_entries.async_has_entries(mlc.DOMAIN)", + "got": "not hass.config_entries.async_entries_by_domain(mlc.DOMAIN)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.875 + }, + { + "repo": "krahabb/meross_lan", + "expected": "haec.CoverEntityFeature.OPEN | haec.CoverEntityFeature.CLOSE", + "got": "{mlc.FEATURE_OPEN_CLOSE}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "krahabb/meross_lan", + "expected": "api._mqtt_connection and api._mqtt_connection.mqtt_is_subscribed", + "got": "isinstance(api.mqtt, hac.MQTTClient)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.22 + }, + { + "repo": "krahabb/meross_lan", + "expected": "state.attributes[haec.ATTR_BRIGHTNESS] == brightness and entity._light[mc.KEY_LUMINANCE] == luminance", + "got": "state.attributes[haec.ATTR_BRIGHTNESS] == brightness", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6797385620915033 + }, + { + "repo": "krahabb/meross_lan", + "expected": "state.attributes[haec.ATTR_COLOR_TEMP_KELVIN] == kelvin and entity._light[mc.KEY_TEMPERATURE] == temperature", + "got": "state.attributes[haec.ATTR_COLOR_TEMP_KELVIN] == kelvin", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6748466257668712 + }, + { + "repo": "krahabb/meross_lan", + "expected": "1)", + "got": "\"MQTT not ready\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.10526315789473684 + }, + { + "repo": "krahabb/meross_lan", + "expected": "ANY)", + "got": "json_dumps({\"method\": mc.METHOD_GET}))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.047619047619047616 + }, + { + "repo": "krahabb/meross_lan", + "expected": "safe_start_calls)", + "got": "safe_start_calls)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "krahabb/meross_lan", + "expected": "helpers.MessageMatcher(header=header_clock_reply))", + "got": "header_clock_reply)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5507246376811594 + }, + { + "repo": "krahabb/meross_lan", + "expected": "helpers.MessageMatcher(header=header_consumption_reply))", + "got": "helpers.MessageMatcher(header_consumption_reply))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9333333333333333 + }, + { + "repo": "krahabb/meross_lan", + "expected": "[ mock.call( mqttconnections[0], HostAddress(tc.MOCK_PROFILE_MSS310_DOMAIN, 443) ), mock.call( mqttconnections[1], HostAddress(tc.MOCK_PROFILE_MSH300_DOMAIN, 443) ), ])", + "got": "mqttconnections)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.17391304347826086 + }, + { + "repo": "krahabb/meross_lan", + "expected": "helpers.MessageMatcher( header=helpers.DictMatcher( { mc.KEY_MESSAGEID: message_bind_set[mc.KEY_HEADER][ mc.KEY_MESSAGEID ], mc.KEY_NAMESPACE: mn.Appliance_Control_Bind.name, mc.KEY_METHOD: mc.METHOD_SETACK, mc.KEY_TRIGGERSRC: mc.HEADER_TRIGGERSRC_CLOUDCONTROL, mc.KEY_FROM: topic_publish, } ) ))", + "got": "\"\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.006688963210702341 + }, + { + "repo": "lark-parser/lark", + "expected": "0", + "got": "1", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "lark-parser/lark", + "expected": "1", + "got": "1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "2", + "got": "2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "5", + "got": "2", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "lark-parser/lark", + "expected": "b", + "got": "b", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "3", + "got": "3", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "9", + "got": "9", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "4", + "got": "2", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "lark-parser/lark", + "expected": "s", + "got": "s", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "cb", + "got": "cb", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "l2", + "got": "l2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "\"t\"", + "got": "\"t\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "'a'", + "got": "Tree('start', ['a', 'a', 'b', 'b'])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.15789473684210525 + }, + { + "repo": "lark-parser/lark", + "expected": "'A'", + "got": "\"A\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "lark-parser/lark", + "expected": "'B'", + "got": "'B'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "'C'", + "got": "'C'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "123", + "got": "123", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "161", + "got": "161", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "153", + "got": "153", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "res1", + "got": "res1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "tree", + "got": "tree", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "None", + "got": "Discard", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "lark-parser/lark", + "expected": "tree2", + "got": "tree2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "False", + "got": "False", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "[3, 5]", + "got": "[1, 2, 3, 4]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5555555555555556 + }, + { + "repo": "lark-parser/lark", + "expected": "[2, 6]", + "got": "[3, 4, 5, 6]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5555555555555556 + }, + { + "repo": "lark-parser/lark", + "expected": "str(e)", + "got": "str(e)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "TestCls", + "got": "TestCls", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "'_ambig'", + "got": "'_ambig'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "internal", + "got": "not internal", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "lark-parser/lark", + "expected": "[3, 4, 6]", + "got": "[3, 5, 7]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7777777777777778 + }, + { + "repo": "lark-parser/lark", + "expected": "self.tree1", + "got": "self.tree1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "OtherClass", + "got": "OtherClass", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "self._parse", + "got": "template.startswith(_TEMPLATE_MARKER)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.20833333333333334 + }, + { + "repo": "lark-parser/lark", + "expected": "not children", + "got": "meta.line", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.19047619047619047 + }, + { + "repo": "lark-parser/lark", + "expected": "parser.grammar", + "got": "parser.grammar", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "list(range(7))", + "got": "[0, 1, 2, 3, 4, 5, 6]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "lark-parser/lark", + "expected": "list('abccbadd')", + "got": "[\"a\", \"b\", \"c\", \"d\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "lark-parser/lark", + "expected": "Tree('start', [])", + "got": "'a'", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3 + }, + { + "repo": "lark-parser/lark", + "expected": "p.parse(\"a b\")", + "got": "\"ab\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.38095238095238093 + }, + { + "repo": "lark-parser/lark", + "expected": "self.mock_fs.files", + "got": "self.mock_fs.files", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "p.parse(\"cbabbbb\")", + "got": "p.parse(\"abccbba\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "lark-parser/lark", + "expected": "list('abc cba dd')", + "got": "list('abc cba dd')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "Tree('start', [n, n])", + "got": "n", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.09090909090909091 + }, + { + "repo": "lark-parser/lark", + "expected": "copy.copy(self.tree1)", + "got": "copy.copy(self.tree1)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "[\"@TEST1!\", \"test2!\"]", + "got": "[\"@test1!\", \"@test2!\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7906976744186046 + }, + { + "repo": "lark-parser/lark", + "expected": "isinstance(tree, Tree)", + "got": "tree.children", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.34285714285714286 + }, + { + "repo": "lark-parser/lark", + "expected": "eval(repr(self.tree1))", + "got": "eval(repr(self.tree1))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "isinstance(node, Token)", + "got": "token_count[0]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2702702702702703 + }, + { + "repo": "lark-parser/lark", + "expected": "copy.deepcopy(self.tree1)", + "got": "copy.deepcopy(self.tree1)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "isinstance(tree, CustomTree)", + "got": "isinstance(tree, CustomTree)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "0)", + "got": "0)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "1)", + "got": "2)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "lark-parser/lark", + "expected": "2)", + "got": "3)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "lark-parser/lark", + "expected": "3)", + "got": "3)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "4)", + "got": "4)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "\"\")", + "got": "\"\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "10)", + "got": "len(s))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2 + }, + { + "repo": "lark-parser/lark", + "expected": "[])", + "got": "[])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "t2)", + "got": "[1, 1, 1, 1])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.125 + }, + { + "repo": "lark-parser/lark", + "expected": "\"b\")", + "got": "'b')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "lark-parser/lark", + "expected": "\"c\")", + "got": "\"c\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "\"a\")", + "got": "'b')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.25 + }, + { + "repo": "lark-parser/lark", + "expected": "\"A\")", + "got": "'A')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "lark-parser/lark", + "expected": "'b')", + "got": "'a')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.75 + }, + { + "repo": "lark-parser/lark", + "expected": "'a')", + "got": "'a')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "'A')", + "got": "'A')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "2.9)", + "got": "4.0)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "lark-parser/lark", + "expected": "text)", + "got": "text)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "'ab')", + "got": "'a' + 'b')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "lark-parser/lark", + "expected": "['N'])", + "got": "['N'])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "[\"2\"])", + "got": "[Token('ANY', '1'), Token('ANY', '2'), Token('ANY', '3')])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.125 + }, + { + "repo": "lark-parser/lark", + "expected": "'abc')", + "got": "'abc')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "[\"b\"])", + "got": "[\"b\"])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "[\"B\"])", + "got": "[\"B\"])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "['a'])", + "got": "['a'])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "['A'])", + "got": "['A'])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "['1'])", + "got": "['int', '1'])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.631578947368421 + }, + { + "repo": "lark-parser/lark", + "expected": "tree2)", + "got": "tree2)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "[\"x!\"])", + "got": "\"X!\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "lark-parser/lark", + "expected": "[\"X!\"])", + "got": "[\"X\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9230769230769231 + }, + { + "repo": "lark-parser/lark", + "expected": "['ab'])", + "got": "['ab', 'a'])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7368421052631579 + }, + { + "repo": "lark-parser/lark", + "expected": "['AB'])", + "got": "['AB'])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "[u'\u00d8'])", + "got": "['\u00d8'])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9230769230769231 + }, + { + "repo": "lark-parser/lark", + "expected": "[None])", + "got": "[\"a\", None, \"c\", \"e\", \"f\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4117647058823529 + }, + { + "repo": "lark-parser/lark", + "expected": "\"RULE\")", + "got": "\"file_input\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3 + }, + { + "repo": "lark-parser/lark", + "expected": "[\"\"])", + "got": "[\"\", \"x\", \"\", \"\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.45714285714285713 + }, + { + "repo": "lark-parser/lark", + "expected": "[\"\"])", + "got": "[\"\", \"\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6956521739130435 + }, + { + "repo": "lark-parser/lark", + "expected": "'hello')", + "got": "'hello')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "expected)", + "got": "expected)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "'_ambig')", + "got": "'ab')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7142857142857143 + }, + { + "repo": "lark-parser/lark", + "expected": "'aaabaa')", + "got": "'abab')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.75 + }, + { + "repo": "lark-parser/lark", + "expected": "terminal)", + "got": "terminal)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "[1, 1, 1])", + "got": "[1, 1, 1])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "tree_copy)", + "got": "tree_copy)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "'aaabaaa')", + "got": "'aaabaaba')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9523809523809523 + }, + { + "repo": "lark-parser/lark", + "expected": "'ab_b_a_')", + "got": "'abba')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8235294117647058 + }, + { + "repo": "lark-parser/lark", + "expected": "'\u0bb5\u0ba3\u0b95\u0bcd\u0b95\u0bae\u0bcd')", + "got": "Tree('NAME', []))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "lark-parser/lark", + "expected": "[1, 2, 1])", + "got": "[1, 2, 1])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "statement)", + "got": "statement)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "'handled')", + "got": "'handled')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "(t, 1, 2))", + "got": "(1, 2))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8235294117647058 + }, + { + "repo": "lark-parser/lark", + "expected": "[\"tested\"])", + "got": "[\"x\", \"tested\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8148148148148148 + }, + { + "repo": "lark-parser/lark", + "expected": "['a', 'A'])", + "got": "['a', 'A'])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "['a', 'a'])", + "got": "['a', 'a'])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "['A', 'a'])", + "got": "['A', 'a'])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "['abcdef'])", + "got": "['abcdef'])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "['a', 'b'])", + "got": "['a', 'b', 'a', 'b'])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6875 + }, + { + "repo": "lark-parser/lark", + "expected": "{'aa', 'a'})", + "got": "set(\"aaa\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.34782608695652173 + }, + { + "repo": "lark-parser/lark", + "expected": "['\"hello\"'])", + "got": "['\"hello\"'])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "[\"'hello'\"])", + "got": "[\"'hello'\"])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "['a', 'bc'])", + "got": "['a', 'b', 'c'])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "lark-parser/lark", + "expected": "list('BCd'))", + "got": "list('Bd'))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9565217391304348 + }, + { + "repo": "lark-parser/lark", + "expected": "list('BCde'))", + "got": "['B', 'C', 'd', 'e'])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4117647058823529 + }, + { + "repo": "lark-parser/lark", + "expected": "composed_res)", + "got": "composed_res)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "expected.data)", + "got": "'_ambig')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.17391304347826086 + }, + { + "repo": "lark-parser/lark", + "expected": "'indirection')", + "got": "'abbb_a_')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.25 + }, + { + "repo": "lark-parser/lark", + "expected": "Tree('a', []))", + "got": "Tree('a', []))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "Tree('b', []))", + "got": "Tree('b', []))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "template.tree)", + "got": "template)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.782608695652174 + }, + { + "repo": "lark-parser/lark", + "expected": "composed_res2)", + "got": "composed_res2)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "['foo', 'bar'])", + "got": "['foo', 'bar'])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "['aBc', 'AbC'])", + "got": "['abc', 'abc', 'abc'])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6486486486486487 + }, + { + "repo": "lark-parser/lark", + "expected": "['a', 'a', 'a'])", + "got": "['a', 'a', 'a'])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "['12', 'lions'])", + "got": "['12', 'lions'])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "ip.parser_state)", + "got": "ip.parser_state)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "{\"$END\", \"SYM\"})", + "got": "[1, 2, 1])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.23076923076923078 + }, + { + "repo": "lark-parser/lark", + "expected": "_remove_ws(new))", + "got": "_remove_ws(new))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "expected_result)", + "got": "expected_result)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "Tree('start', []))", + "got": "Tree('start', ['A'] * 8191))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.782608695652174 + }, + { + "repo": "lark-parser/lark", + "expected": "['12', 'toucans'])", + "got": "['12', 'toucans'])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "[Token('B', 'A')])", + "got": "['A'])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "lark-parser/lark", + "expected": "['16', 'candles'])", + "got": "['16', 'candles'])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "expected_top_down)", + "got": "expected_top_down)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "str(cm2.exception))", + "got": "\"Unexpected input: ' '\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.32558139534883723 + }, + { + "repo": "lark-parser/lark", + "expected": "expected_botton_up)", + "got": "expected_top_down)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7027027027027027 + }, + { + "repo": "lark-parser/lark", + "expected": "['12', 'capybaras'])", + "got": "['12', 'capybaras'])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "['12', 'elephants'])", + "got": "['12', 'elephants'])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "Tree('start', ['A']))", + "got": "Tree('start', ['A']))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "template.match(tree))", + "got": "template.match(tree))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "json.loads(test_json))", + "got": "json.loads(test_json))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "Tree('start', [3, 7]))", + "got": "sum([1, 2, 4, 3, 4]))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4186046511627907 + }, + { + "repo": "lark-parser/lark", + "expected": "Tree('start', [\"AAA\"]))", + "got": "Tree('start', [Token('AAA', 'AAAAA')]))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6774193548387096 + }, + { + "repo": "lark-parser/lark", + "expected": "set(expected.children))", + "got": "expected.children)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8780487804878049 + }, + { + "repo": "lark-parser/lark", + "expected": "Tree('start', [str(i)]))", + "got": "Tree('start', [str(i)]))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "[1, 1, 2, 3, 3, 3, 6, 6])", + "got": "[1, 2, 3, 4])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.631578947368421 + }, + { + "repo": "lark-parser/lark", + "expected": "conf.test_var(test_case))", + "got": "conf.test_var(test_case))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "Tree('start', [u'\\xb1a']))", + "got": "Tree('start', [Tree('A', [])]))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7017543859649122 + }, + { + "repo": "lark-parser/lark", + "expected": "template.match(test_case))", + "got": "template.match(test_case))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "Tree('start', ['BB', 'C']))", + "got": "Tree('start', ['B', 'B', 'C']))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9310344827586207 + }, + { + "repo": "lark-parser/lark", + "expected": "Tree('a', [Tree('b', [])]))", + "got": "Tree('a', []))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6829268292682927 + }, + { + "repo": "lark-parser/lark", + "expected": "translator.translate(tree))", + "got": "translator.translate(tree))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "\"A isn't associated with /a/\")", + "got": "'\"a\" should be considered anonymous')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3283582089552239 + }, + { + "repo": "lark-parser/lark", + "expected": "Tree('start', ['123', 'abc']))", + "got": "\"123abc\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.358974358974359 + }, + { + "repo": "lark-parser/lark", + "expected": "Tree('start', [Tree('x', [])]))", + "got": "Tree('start', [Tree('x', []), Tree('x', []), Tree('x', []), Tree('x', []), Tree('x', []), Tree('x', []), Tree('x', []), Tree('x', []), Tree('x', []), Tree('x', []), Tree('x', []), Tree('x', []), Tree('x', []), Tree('x', []), Tree('x', []), Tree('x', []), Tree('x', []), Tree('x', []), Tree('x', []), Tree('x', []), Tree('x', []), Tree('x', []), Tree('x', []), Tree('x', []), Tree('x', []), Tree('x', []), Tree('x', []), Tree('x', []), Tree('x', []), Tree('x', []), Tree('x', []), Tree('x', []), Tree('x', []), Tree('x', []), Tree('x', []), Tree('x', []), Tree('x', []), Tree('x', []), Tree('x', []), Tree('x', []), Tree('x', []), Tree('x', []", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.09212481426448738 + }, + { + "repo": "lark-parser/lark", + "expected": "Tree('start', [Tree('a', [])]))", + "got": "'a')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.22857142857142856 + }, + { + "repo": "lark-parser/lark", + "expected": "Tree('start', [Tree('b', [])]))", + "got": "'ABC')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.16216216216216217 + }, + { + "repo": "lark-parser/lark", + "expected": "Tree('start', [Token('A', 'A')]))", + "got": "['A'])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "lark-parser/lark", + "expected": "list(template.search(test_case)))", + "got": "list(template.search(test_case)))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "[\"ERROR:lark:dummy message\"])", + "got": "[\"dummy message\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7659574468085106 + }, + { + "repo": "lark-parser/lark", + "expected": "IOError)", + "got": "ImportError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7 + }, + { + "repo": "lark-parser/lark", + "expected": "LexError)", + "got": "GrammarError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5454545454545454 + }, + { + "repo": "lark-parser/lark", + "expected": "TypeError)", + "got": "UnexpectedToken)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "lark-parser/lark", + "expected": "ParseError)", + "got": "ParseError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "GrammarError)", + "got": "GrammarError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "expected_error)", + "got": "expected_error, g.parse, \"1.2.3\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.625 + }, + { + "repo": "lark-parser/lark", + "expected": "AttributeError)", + "got": "ValueError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "lark-parser/lark", + "expected": "UnexpectedInput)", + "got": "UnexpectedInput)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "ConfigurationError)", + "got": "FileNotFoundError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4864864864864865 + }, + { + "repo": "lark-parser/lark", + "expected": "NotImplementedError)", + "got": "UnexpectedToken)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "lark-parser/lark", + "expected": "UnexpectedCharacters)", + "got": "UnicodeDecodeError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.45 + }, + { + "repo": "lark-parser/lark", + "expected": "MissingVariableError)", + "got": "MissingVariableError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "context['UnexpectedToken'])", + "got": "SyntaxError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2564102564102564 + }, + { + "repo": "lark-parser/lark", + "expected": "\"Expected that more than one rule will be generated\")", + "got": "\"no rules\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3125 + }, + { + "repo": "lark-parser/lark", + "expected": "b)", + "got": "b)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "['ab'])", + "got": "['ab'])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "log)", + "got": "log)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "())", + "got": "())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "['1'])", + "got": "['1'])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "('item',))", + "got": "['a'])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.375 + }, + { + "repo": "lark-parser/lark", + "expected": "('list',))", + "got": "('list',))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "['World'])", + "got": "('Hello', 'World'))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5517241379310345 + }, + { + "repo": "lark-parser/lark", + "expected": "['starts'])", + "got": "['starts'])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "[\"A\", \"B\"])", + "got": "[Tree('expr', [])])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "lark-parser/lark", + "expected": "['HelloWorld'])", + "got": "['Hello', 'World'])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8823529411764706 + }, + { + "repo": "lark-parser/lark", + "expected": "[Tree('b',[])])", + "got": "['B'])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.47619047619047616 + }, + { + "repo": "lark-parser/lark", + "expected": "[Tree('b', [])])", + "got": "[Tree('expr', []), Tree('expr', [])])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5660377358490566 + }, + { + "repo": "lark-parser/lark", + "expected": "('item', 'item'))", + "got": "('item', 'item'))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "[Tree('expr', [])])", + "got": "['C'])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "lark-parser/lark", + "expected": "[Tree('sep', ['1'])])", + "got": "[Tree('sep', ['1'])])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "['1', '2', '3', '4'])", + "got": "[Tree('sep', ['1', '2', '3', '4'])])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7368421052631579 + }, + { + "repo": "lark-parser/lark", + "expected": "[Tree('expr', [\"A\", \"B\"])])", + "got": "['A'])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.30303030303030304 + }, + { + "repo": "lark-parser/lark", + "expected": "ip.parser_state)", + "got": "ip.parser_state)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "c)", + "got": "c)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "code)", + "got": "code)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "visited_ab[0])", + "got": "visited_ab[0])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "visited_bc[0])", + "got": "visited_bc[0])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "visited_cd[0])", + "got": "visited_cd[0])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "conf.test_var(test_case))", + "got": "conf.test_var(test_case))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "lark-parser/lark", + "expected": "template.match(test_case))", + "got": "template.match(test_case))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "llimllib/limbo", + "expected": "0", + "got": "0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "llimllib/limbo", + "expected": "9", + "got": "3", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "llimllib/limbo", + "expected": "2", + "got": "3", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "llimllib/limbo", + "expected": "5", + "got": "10", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "llimllib/limbo", + "expected": "1", + "got": "3", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "llimllib/limbo", + "expected": "[]", + "got": "[]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "llimllib/limbo", + "expected": "''", + "got": "\"Unknown command\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "llimllib/limbo", + "expected": "msg", + "got": "\"I\u00f1t\u00ebrn\u00e2ti\u00f4n\u00e0liz\u00e6ti\u00f8n\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "llimllib/limbo", + "expected": "ret", + "got": "ret", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "llimllib/limbo", + "expected": "None", + "got": "[(\"message\", {\"text\": msg})]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0625 + }, + { + "repo": "llimllib/limbo", + "expected": "True", + "got": "True", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "llimllib/limbo", + "expected": "hooks", + "got": "hooks", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "llimllib/limbo", + "expected": "heads", + "got": "heads", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "llimllib/limbo", + "expected": "bananas_gifs", + "got": "bananas_gifs", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "llimllib/limbo", + "expected": "\"user 2 left\"", + "got": "\"user 2 left\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "llimllib/limbo", + "expected": "bananas_images", + "got": "bananas_images", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "llimllib/limbo", + "expected": "\":cloud: 74\u00b0f\"", + "got": "\"**Current Conditions**\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.21052631578947367 + }, + { + "repo": "llimllib/limbo", + "expected": "\":cloud: 23\u00b0c\"", + "got": "\":cloud: 24.4\u00b0C\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "llimllib/limbo", + "expected": "\"user 2 joined\"", + "got": "\"saw user 2 join\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8125 + }, + { + "repo": "llimllib/limbo", + "expected": "\"saw user 2 join\"", + "got": "\"User has joined\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6470588235294118 + }, + { + "repo": "llimllib/limbo", + "expected": "sorted(b.items())", + "got": "sorted(b.items())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "llimllib/limbo", + "expected": "[u\"!echo bananas\"]", + "got": "\"bananas\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "llimllib/limbo", + "expected": "hooks[\"help\"][\"test\"]", + "got": "\"test\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "llimllib/limbo", + "expected": "attachment[\"pretext\"]", + "got": "attachment[\"text\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9230769230769231 + }, + { + "repo": "llimllib/limbo", + "expected": "\"Changed: {}\".format(newmsg)", + "got": "newmsg", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.35294117647058826 + }, + { + "repo": "llimllib/limbo", + "expected": "\"Deleted: {}\".format(oldmsg)", + "got": "\"Deleted: {}\".format(oldmsg)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "llimllib/limbo", + "expected": "u\"!echo I\u00f1t\u00ebrn\u00e2ti\u00f4n\u00e0liz\u00e6ti\u00f8n\"", + "got": "msg", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "llimllib/limbo", + "expected": "hooks[\"extendedhelp\"][\"test\"]", + "got": "hooks[\"extendedhelp\"][\"test\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "llimllib/limbo", + "expected": "'No help found for not_there'", + "got": "''", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.12903225806451613 + }, + { + "repo": "llimllib/limbo", + "expected": "u\"limbo> {0}\\nlimbo>\".format(msg)", + "got": "msg", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "llimllib/limbo", + "expected": "type(sqlite3.connect(\":memory:\"))", + "got": "sqlite3.Connection", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5882352941176471 + }, + { + "repo": "llimllib/limbo", + "expected": "\"```-- ---. - ---. .-. .... . .- -..```\"", + "got": "\"```.-.. .-.. .. -- .-.. .-.. .. -...```\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.691358024691358 + }, + { + "repo": "llimllib/limbo", + "expected": "\"```.-.. .-.. .. -- .-.. .-.. .. -...```\"", + "got": "\"llimllib\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0784313725490196 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "1", + "got": "1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "0", + "got": "0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "i", + "got": "i", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "2", + "got": "2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "3", + "got": "3", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "50", + "got": "max_tokens", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "40", + "got": "40", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "10", + "got": "10", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "-1", + "got": "-1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "\"\"", + "got": "\"True\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "422", + "got": "400", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "100", + "got": "100", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "None", + "got": "None", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "True", + "got": "False", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "key2", + "got": "key2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "key3", + "got": "key3", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "model", + "got": "model", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "False", + "got": "False", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "\"NYC\"", + "got": "\"NYC\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "\"text\"", + "got": "\"text\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "prompt", + "got": "prompt", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "\"celsius\"", + "got": "\"celsius\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "\"call_123\"", + "got": "\"call_123\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "events_order", + "got": "events_order", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "response.text", + "got": "response.text", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "\"Final answer\"", + "got": "\"Final answer\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "has_usage_info", + "got": "has_usage_info", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "model_id_to_test", + "got": "model_id_to_test", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "expected_content", + "got": "expected_content", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "expected_thinking", + "got": "expected_thinking", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "\"chat.completion\"", + "got": "\"chat.completion\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "top_logprobs_count", + "got": "top_logprobs_count", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "\"Reasoning process\"", + "got": "\"Reasoning process\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "text_content.strip()", + "got": "text_content.strip()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "message_stop_received", + "got": "message_stop_received", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "len(full_response.data)", + "got": "1", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "isinstance(prompt, str)", + "got": "\"test_value\" in prompt", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "\"conversion successful\"", + "got": "\"conversion successful\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "choices.message.content", + "got": "choices.message.role", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8372093023255814 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "hasattr(response, \"text\")", + "got": "\"transcription\" in response", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.38461538461538464 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "prompt.endswith(\"\")", + "got": "\"thinking\" in prompt", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2608695652173913 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "\"Here is the final answer.\"", + "got": "\"Here is the final answer.\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "isinstance(model.details, dict)", + "got": "model.details.model_id", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5660377358490566 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "prompt.endswith(\"Hi there! How\")", + "got": "\"continue_final_message\" in prompt", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.18181818181818182 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "isinstance(result, GenerationResult)", + "got": "isinstance(result, GenerationResult)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "isinstance(result1, GenerationResult)", + "got": "isinstance(result2, GenerationResult)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.972972972972973 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "isinstance(result_low, GenerationResult)", + "got": "isinstance(result_low, GenerationResult)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "\"Hi\" in prompt or \"hello\" in prompt.lower()", + "got": "\"Hi\" in prompt", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.49122807017543857 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "isinstance(result.content, CompletionContent)", + "got": "isinstance(result.content, CompletionContent)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "\"This is a direct response without thinking tags.\"", + "got": "\"This is a direct response without thinking tags.\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "\"This is a thinking process.\\nAnalyzing the request.\"", + "got": "\"This is a thinking process.\\nAnalyzing the request.\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "not hasattr(model, \"details\") or model.details is None", + "got": "model.id", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.22580645161290322 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "Exception)", + "got": "ValueError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "ValueError)", + "got": "ValueError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "NotFoundError)", + "got": "NotFoundError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "RuntimeError, match=\"Model loading failed\")", + "got": "RuntimeError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4642857142857143 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "tools)", + "got": "tools)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "result)", + "got": "result)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "result_mixed)", + "got": "result_mixed)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "args[\"encoding\"])", + "got": "args[\"encoding\"])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "result[0].arguments[\"value\"])", + "got": "result[0].arguments[\"value\"])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "tools)", + "got": "tools)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "result)", + "got": "result)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "tool_call.name)", + "got": "tool_call)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "tool_call.arguments)", + "got": "tool_call.arguments)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "1)", + "got": "2)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "5)", + "got": "5)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "2)", + "got": "2)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "7)", + "got": "7)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "42)", + "got": "42)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "{})", + "got": "{})", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "10)", + "got": "10)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "20)", + "got": "20)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "-5)", + "got": "-5)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "-10)", + "got": "-10)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "0.7)", + "got": "0.7)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "True)", + "got": "True)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "\"42\")", + "got": "\"42\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "False)", + "got": "False)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "19.99)", + "got": "19.99)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "\"send_email\")", + "got": "\"send_email\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "\"example.py\")", + "got": "\"example.py\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "\"get_weather\")", + "got": "\"get_weather\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "\"hello world\")", + "got": "\"hello world\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "\"not_a_number\")", + "got": "\"not_a_number\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "\"get_forecast\")", + "got": "\"get_forecast\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "\"[invalid json\")", + "got": "\"[invalid json\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "\"browser.search\")", + "got": "\"browser.search\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "\"get_current_weather\")", + "got": "\"get_current_weather\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "[\"apple\", \"banana\", \"cherry\"])", + "got": "[\"apple\", \"banana\", \"cherry\"])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "{\"theme\": \"dark\", \"fontSize\": 14})", + "got": "{\"theme\": \"dark\", \"fontSize\": 14})", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "[\"file1.py\", \"file2.py\", \"file3.py\"])", + "got": "[\"file1.py\", \"file2.py\", \"file3.py\"])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "int)", + "got": "int)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "str)", + "got": "str)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "bool)", + "got": "bool)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "list)", + "got": "list)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "dict)", + "got": "dict)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "float)", + "got": "float)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "result[1].id)", + "got": "result[1].id)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "args[\"enabled\"])", + "got": "args[\"enabled\"])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "args[\"case_sensitive\"])", + "got": "args[\"case_sensitive\"])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "args[\"regex\"])", + "got": "args[\"regex\"])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "madroidmaq/mlx-omni-server", + "expected": "[10, \"10\"])", + "got": "[\"10\", \"10.0\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8461538461538461 + }, + { + "repo": "maldoinc/wireup", + "expected": "0", + "got": "0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "1", + "got": "1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "5", + "got": "\"foo\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "4", + "got": "\"foo\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "s2", + "got": "s2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "{}", + "got": "{\"scoped\": True}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "maldoinc/wireup", + "expected": "[]", + "got": "[\"headerReader\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "maldoinc/wireup", + "expected": "42", + "got": "42", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "11", + "got": "11", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "22", + "got": "22", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "200", + "got": "200", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "401", + "got": "401", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "\"q\"", + "got": "\"q\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "\"f\"", + "got": "\"f\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "\"x\"", + "got": "\"x\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "Foo", + "got": "Foo", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "666", + "got": "666", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "\"1\"", + "got": "\"2\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "maldoinc/wireup", + "expected": "\"42\"", + "got": "\"42\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "path", + "got": "\"Hello World! Debug = True. Your lucky number is 4\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "True", + "got": "True", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "body", + "got": "body", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "res2", + "got": "res2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "done", + "got": "not done", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "maldoinc/wireup", + "expected": "inst", + "got": "inst", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "None", + "got": "None", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "\"foo\"", + "got": "\"foo\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "\"bar\"", + "got": "\"bar\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "\"opt\"", + "got": "\"opt\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "mock2", + "got": "mock2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "outer", + "got": "outer", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "inner", + "got": "inner", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "Thing", + "got": "Thing", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "\"test\"", + "got": "\"test\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "\"boom\"", + "got": "\"boom\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "\"mock\"", + "got": "\"mock\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "fake_b", + "got": "fake_b", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "{None}", + "got": "random_service_factory", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.14285714285714285 + }, + { + "repo": "maldoinc/wireup", + "expected": "result", + "got": "result", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "\"quack\"", + "got": "\"quack\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "scoped2", + "got": "scoped2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "g1_done", + "got": "g1_done", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "g2_done", + "got": "not g2_done", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7777777777777778 + }, + { + "repo": "maldoinc/wireup", + "expected": "\"empty\"", + "got": "\"empty\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "\"value\"", + "got": "\"value\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "self.rng", + "got": "self.rng", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "[\"name\"]", + "got": "[\"name\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "expected", + "got": "expected.strip()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "maldoinc/wireup", + "expected": "mock_obj", + "got": "mock_obj", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "override", + "got": "override", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "\"value1\"", + "got": "\"value1\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "\"value2\"", + "got": "\"value2\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "container", + "got": "container", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "websocket", + "got": "websocket", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "rand1_mock", + "got": "rand1_mock", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "rand2_mock", + "got": "rand2_mock", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "singleton1", + "got": "singleton1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "ids[\"task\"]", + "got": "ids[\"task\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "sys.modules", + "got": "sys.modules", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "cleanup_ran", + "got": "cleanup_ran", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "\"test-test\"", + "got": "\"test-test\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "\"singleton\"", + "got": "\"singleton\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "self.greeter", + "got": "self.greeter", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "cleanup_done", + "got": "cleanup_done", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "new_instance", + "got": "new_instance", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "\"HELLO TEST\"", + "got": "\"HELLO TEST\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "[\"f2\", \"f1\"]", + "got": "[\"f1\", \"f2\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "maldoinc/wireup", + "expected": "{\"number\": 4}", + "got": "{\"number\": 1}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9230769230769231 + }, + { + "repo": "maldoinc/wireup", + "expected": "{\"random\": 4}", + "got": "{\"random\": 4}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "[\"fastapi:4\"]", + "got": "[\"fastapi:4\", \"fastapi:4\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "maldoinc/wireup", + "expected": "\"Hello World\"", + "got": "\"Hello World\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "impl_instance", + "got": "impl_instance", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "\"after_yield\"", + "got": "\"after_yield\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "MyCustomClass", + "got": "\"MyCustomClass\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9285714285714286 + }, + { + "repo": "maldoinc/wireup", + "expected": "{\"number\": 16}", + "got": "{\"random\": 42}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "maldoinc/wireup", + "expected": "{\"test\": True}", + "got": "\"foo\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.21052631578947367 + }, + { + "repo": "maldoinc/wireup", + "expected": "\"foo mocked 1\"", + "got": "\"foo mocked 1\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "falsy_override", + "got": "falsy_override", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "registry.impls", + "got": "registry.services", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6451612903225806 + }, + { + "repo": "maldoinc/wireup", + "expected": "sig.parameters", + "got": "sig.parameters", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "scoped_service2", + "got": "scoped_service_dependency", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7 + }, + { + "repo": "maldoinc/wireup", + "expected": "\"not autowired\"", + "got": "\"Hello, World!\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.26666666666666666 + }, + { + "repo": "maldoinc/wireup", + "expected": "[\"Hello Async\"]", + "got": "[\"Hello Async\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "[\"after_yield\"]", + "got": "[\"before_yield\", \"after_yield\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6521739130434783 + }, + { + "repo": "maldoinc/wireup", + "expected": "rebound_factory", + "got": "rebound_factory", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "qualifier_value", + "got": "qualifier_value", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "Optional[Thing]", + "got": "Optional", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6956521739130435 + }, + { + "repo": "maldoinc/wireup", + "expected": "\"The number: 42\"", + "got": "\"The number: 42\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "{\"name\": \"Test\"}", + "got": "{\"message\": \"Hello Test\"}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6829268292682927 + }, + { + "repo": "maldoinc/wireup", + "expected": "\"/var/cache/etc\"", + "got": "\"/var/cache/etc\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "Something(\"foo\")", + "got": "\"foo\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.47619047619047616 + }, + { + "repo": "maldoinc/wireup", + "expected": "[\"before_yield\"]", + "got": "[\"before_yield\", \"after_yield\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6808510638297872 + }, + { + "repo": "maldoinc/wireup", + "expected": "isinstance(a, A)", + "got": "isinstance(a, A)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "isinstance(b, B)", + "got": "isinstance(b, B)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "\"Hello Starlette\"", + "got": "\"Hello Starlette!\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9714285714285714 + }, + { + "repo": "maldoinc/wireup", + "expected": "\"dependency_impl\"", + "got": "\"dependency_impl\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "\"sqlite://memory\"", + "got": "\"sqlite://memory\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "middleware_called", + "got": "middleware_called", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "\"value1 and None\"", + "got": "\"value1 and None\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "(\"manual\", \"bar\")", + "got": "(\"manual\", \"bar\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "{\"test\": \"mocked\"}", + "got": "{\"test\": \"mocked\"}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "help_result.output", + "got": "help_result.stderr", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7222222222222222 + }, + { + "repo": "maldoinc/wireup", + "expected": "_cleanup_performed", + "got": "not _cleanup_performed", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9 + }, + { + "repo": "maldoinc/wireup", + "expected": "exception_notified", + "got": "exception_notified", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "registry.factories", + "got": "registry.impls", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6875 + }, + { + "repo": "maldoinc/wireup", + "expected": "(MyCustomClass, 0)", + "got": "\"MyCustomClass\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7878787878787878 + }, + { + "repo": "maldoinc/wireup", + "expected": "req_context.request", + "got": "req_context.request", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "{\"lucky_number\": 4}", + "got": "\"random\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "maldoinc/wireup", + "expected": "[\"Hello Starlette\"]", + "got": "[\"Hello Starlette\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "\"value1 and value2\"", + "got": "\"value1 and value2\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "\"App 1: Hello World\"", + "got": "\"Hello World! Debug = True. Your lucky number is 5\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.36619718309859156 + }, + { + "repo": "maldoinc/wireup", + "expected": "\"/var/cache/anon/db\"", + "got": "\"/var/cache/anon/db\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "\"bar_multiple_bases\"", + "got": "\"foo\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.16 + }, + { + "repo": "maldoinc/wireup", + "expected": "isinstance(dep, Dep)", + "got": "dep", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2608695652173913 + }, + { + "repo": "maldoinc/wireup", + "expected": "isinstance(foo, Foo)", + "got": "isinstance(foo, Foo)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "container.get(FooImpl)", + "got": "container.get(FooImpl)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "Something(\"Something\")", + "got": "Something(\"Something\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "str(record[0].message)", + "got": "record[0].message.args[0]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.723404255319149 + }, + { + "repo": "maldoinc/wireup", + "expected": "scoped.get(SomeService)", + "got": "not done", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.25806451612903225 + }, + { + "repo": "maldoinc/wireup", + "expected": "bag._ConfigStore__cache", + "got": "bag._cache", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6060606060606061 + }, + { + "repo": "maldoinc/wireup", + "expected": "isinstance(thing, Thing)", + "got": "cleanup_done", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "maldoinc/wireup", + "expected": "isinstance(bar.foo, Foo)", + "got": "isinstance(bar.foo, Foo)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "{\"hello\": \"Hello Django\"}", + "got": "{\"hello\": \"Hello Django\"}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "scoped.get(ScopedService)", + "got": "scoped.get(ScopedService)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "\"Your lucky number is: 42\"", + "got": "\"Your lucky number is: 42\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "isinstance(instance, Duck)", + "got": "isinstance(instance, Duck)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "scoped1.get(ScopedService)", + "got": "scoped2.get(ScopedService)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9615384615384616 + }, + { + "repo": "maldoinc/wireup", + "expected": "scoped2.get(ScopedService)", + "got": "scoped2.get(ScopedService)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "{\"greeting\": \"Hello world\"}", + "got": "\"Hello, World!\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "maldoinc/wireup", + "expected": "\"Hello World! Debug = True\"", + "got": "\"Hello World! Debug = True. Your lucky number is 4\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6923076923076923 + }, + { + "repo": "maldoinc/wireup", + "expected": "{\"greeting\": \"Hello World\"}", + "got": "\"Hello World! Debug = True\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.48148148148148145 + }, + { + "repo": "maldoinc/wireup", + "expected": "{\"counter\": i, \"random\": 4}", + "got": "{\"number\": i}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "maldoinc/wireup", + "expected": "isinstance(res, EnvService)", + "got": "isinstance(res, EnvService)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "{\"greeting\": \"Hoi, webview\"}", + "got": "{\"greeting\": \"Hoi, world\"}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8148148148148148 + }, + { + "repo": "maldoinc/wireup", + "expected": "{\"greeting\": \"Go away, Bob\"}", + "got": "{\"greeting\": \"Go away, Bob\"}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "isinstance(svc, RealService)", + "got": "isinstance(svc, RealService)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "container.get(RandomService)", + "got": "None", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.1875 + }, + { + "repo": "maldoinc/wireup", + "expected": "scoped.get(TransientService)", + "got": "scoped.get(TransientService)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "container.get(Optional[Foo])", + "got": "None", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.1875 + }, + { + "repo": "maldoinc/wireup", + "expected": "isinstance(bag, ConfigStore)", + "got": "not bag._parameters", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2978723404255319 + }, + { + "repo": "maldoinc/wireup", + "expected": "_sample_function.__globals__", + "got": "globals()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3783783783783784 + }, + { + "repo": "maldoinc/wireup", + "expected": "{\"greeting\": \"Hello webview\"}", + "got": "{\"webview\": \"Hello world\"}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6181818181818182 + }, + { + "repo": "maldoinc/wireup", + "expected": "{\"counter\": 1, \"random\": 100}", + "got": "{\"counter\": 1, \"random\": 100}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "isinstance(instance, FooImpl)", + "got": "isinstance(instance, FooImpl)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "isinstance(res, service_type)", + "got": "isinstance(res, expected_type)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8135593220338984 + }, + { + "repo": "maldoinc/wireup", + "expected": "await run(container.get(Foo))", + "got": "await run(container.get(Foo))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "isinstance(foo, SyncOverride)", + "got": "isinstance(foo, SyncOverride)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "isinstance(sc, ScopedService)", + "got": "isinstance(sc, ScopedService)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "{\"debug\": False, \"test\": True}", + "got": "{\"env\": \"test\"}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5777777777777777 + }, + { + "repo": "maldoinc/wireup", + "expected": "isinstance(res, expected_type)", + "got": "isinstance(res, expected_type)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "isinstance(foo, AsyncOverride)", + "got": "isinstance(foo, AsyncOverride)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "isinstance(svc.dep, ProtoImpl)", + "got": "isinstance(svc.dep, ProtoImpl)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "{\"message\": \"FBV: Hello World\"}", + "got": "\"Hello World!\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5777777777777777 + }, + { + "repo": "maldoinc/wireup", + "expected": "{\"message\": \"CBV: Hello World\"}", + "got": "{\"message\": \"CBV: Hello World\"}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "SomethingElse(\"Something else\")", + "got": "SomethingElse(\"Something else\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "isinstance(container.get(A), A)", + "got": "not isinstance(a, B)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5882352941176471 + }, + { + "repo": "maldoinc/wireup", + "expected": "isinstance(container.get(B), B)", + "got": "isinstance(container.get(B), B)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "isinstance(container.get(C), C)", + "got": "isinstance(container.get(C), C)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "Exception)", + "got": "Exception)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "StopAsyncIteration)", + "got": "StopAsyncIteration)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "ContainerCloseError)", + "got": "ValueError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5806451612903226 + }, + { + "repo": "maldoinc/wireup", + "expected": "UnknownParameterError)", + "got": "UnknownParameterError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "ValueError, match=\"boom!\")", + "got": "ValueError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5945945945945946 + }, + { + "repo": "maldoinc/wireup", + "expected": "UnknownServiceRequestedError)", + "got": "UnknownServiceRequestedError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "PositionalOnlyParameterError)", + "got": "PositionalOnlyParameterError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "InvalidRegistrationTypeError)", + "got": "InvalidRegistrationTypeError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "FactoryReturnTypeIsEmptyError)", + "got": "FactoryReturnTypeIsEmptyError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "DuplicateServiceRegistrationError)", + "got": "DuplicateServiceRegistrationError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "ValueError, match=\"RequestContext name: World\")", + "got": "ValueError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3793103448275862 + }, + { + "repo": "maldoinc/wireup", + "expected": "WireupError, match=\"has an unknown dependency\")", + "got": "WireupError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4067796610169492 + }, + { + "repo": "maldoinc/wireup", + "expected": "InvalidAsTypeError, match=\"Invalid as_type value\")", + "got": "AsTypeMismatchError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.34285714285714286 + }, + { + "repo": "maldoinc/wireup", + "expected": "WireupError, match=\"Multiple Wireup annotations used\")", + "got": "WireupError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "maldoinc/wireup", + "expected": "AsTypeMismatchError, match=\"Cannot register implementation\")", + "got": "AsTypeMismatchError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "maldoinc/wireup", + "expected": "WireupError, match=f\"{t.__name__} in Wireup is only available.\")", + "got": "WireupError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3157894736842105 + }, + { + "repo": "maldoinc/wireup", + "expected": "WireupError, match=\"@inject decorator applied multiple times to\")", + "got": "WireupError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3116883116883117 + }, + { + "repo": "maldoinc/wireup", + "expected": "WireupError, match=\"Sync container cannot perform injection on async targets\")", + "got": "WireupError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.26666666666666666 + }, + { + "repo": "maldoinc/wireup", + "expected": "UnknownParameterError, match=re.escape(\"Unknown config key requested: 'param2'\"))", + "got": "UnknownParameterError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.42718446601941745 + }, + { + "repo": "maldoinc/wireup", + "expected": "WireupError, match=\"is an async dependency and it cannot be created in a synchronous context\")", + "got": "WireupError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.22641509433962265 + }, + { + "repo": "maldoinc/wireup", + "expected": "WireupError, match=f\"Injectable {NotDecorated} is not decorated with @abstract or @injectable\")", + "got": "WireupError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.22429906542056074 + }, + { + "repo": "maldoinc/wireup", + "expected": "UnknownParameterError, match=re.escape(\"Unknown config key requested: 'param1.nested2'. 'nested2' not found in 'param1'\"))", + "got": "UnknownParameterError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3055555555555556 + }, + { + "repo": "maldoinc/wireup", + "expected": "UnknownOverrideRequestedError, match=re.escape(\"Cannot override unknown Type unittest.case.TestCase with qualifier 'foo'.\"))", + "got": "UnknownOverrideRequestedError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.38961038961038963 + }, + { + "repo": "maldoinc/wireup", + "expected": "DuplicateServiceRegistrationError, match=re.escape(f\"Cannot register type Type {Multiple.__module__}.{Multiple.__name__} as it already exists.\"))", + "got": "DuplicateServiceRegistrationError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.37988826815642457 + }, + { + "repo": "maldoinc/wireup", + "expected": "UnknownParameterError, match=re.escape( \"Unknown config key requested: 'param1.nested1.nested1_2'. 'nested1_2' not found in 'param1.nested1'\" ))", + "got": "UnknownParameterError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.26506024096385544 + }, + { + "repo": "maldoinc/wireup", + "expected": "WireupError, match=re.escape( \"Sync container cannot perform injection on async targets. \" \"Create an async container via wireup.create_async_container.\" ))", + "got": "TypeError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.08433734939759036 + }, + { + "repo": "maldoinc/wireup", + "expected": "WireupError, match=re.escape( \"Parameter 'foo' of Type test.unit.test_container_creation.Bar \" \"has an unknown dependency on Type test.unit.test_container_creation.Foo.\" ))", + "got": "WireupError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.13043478260869565 + }, + { + "repo": "maldoinc/wireup", + "expected": "WireupError, match=re.escape( \"Parameter 'not_managed_by_wireup' of Function test.unit.test_inject_from_container._ \" \"depends on an unknown Wireup config key 'invalid'.\" ))", + "got": "WireupError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.12972972972972974 + }, + { + "repo": "maldoinc/wireup", + "expected": "ValueError, match=re.escape( \"Provided config key format is invalid: 'param1.'.\" \" Please provide a non-empty config key, or a valid dot-separated path, with non-empty parts.\" ))", + "got": "ValueError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.1164021164021164 + }, + { + "repo": "maldoinc/wireup", + "expected": "ValueError, match=re.escape( \"Provided config key format is invalid: '.param1'.\" \" Please provide a non-empty config key, or a valid dot-separated path, with non-empty parts.\" ))", + "got": "ValueError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.1164021164021164 + }, + { + "repo": "maldoinc/wireup", + "expected": "ValueError, match=re.escape( \"Provided config key format is invalid: '.param1.'.\" \" Please provide a non-empty config key, or a valid dot-separated path, with non-empty parts.\" ))", + "got": "ValueError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.11578947368421053 + }, + { + "repo": "maldoinc/wireup", + "expected": "WireupError, match=re.escape( \"is an async dependency and it cannot be created in a synchronous context. \" \"Create and use an async container via wireup.create_async_container.\" ))", + "got": "WireupError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.125 + }, + { + "repo": "maldoinc/wireup", + "expected": "WireupError, match=re.escape( \"Parameter '_param' of Type test.unit.services.no_annotations.random.random_service.RandomService \" \"depends on an unknown Wireup config key 'foo'.\" ))", + "got": "WireupError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.12435233160621761 + }, + { + "repo": "maldoinc/wireup", + "expected": "UnknownServiceRequestedError, match=re.escape( f\"Cannot create unknown injectable Type builtins.str with qualifier '{__name__}'. \" \"Make sure it is registered with the container.\" ))", + "got": "WireupError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.09278350515463918 + }, + { + "repo": "maldoinc/wireup", + "expected": "WireupError, match=re.escape( f\"{Foo} is an async dependency and it cannot be created in a synchronous context.\" \" Create and use an async container via wireup.create_async_container.\" ))", + "got": "WireupError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.12060301507537688 + }, + { + "repo": "maldoinc/wireup", + "expected": "None)", + "got": "None)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "container.get(Foo).get_foo())", + "got": "container.get(Foo).name)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7924528301886793 + }, + { + "repo": "maldoinc/wireup", + "expected": "container.get(EnvService).env_name)", + "got": "container.get(EnvService).name)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9393939393939394 + }, + { + "repo": "maldoinc/wireup", + "expected": "container.get(services.Foo).get_foo())", + "got": "container.get(Foo).get_foo())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8656716417910447 + }, + { + "repo": "maldoinc/wireup", + "expected": "container.get(RandomService, qualifier=\"foo\").get_random())", + "got": "container.get(TrulyRandomService, qualifier=\"bar\").get_truly_random())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8682170542635659 + }, + { + "repo": "maldoinc/wireup", + "expected": "\"foo\")", + "got": "\"foo\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "_param_get_annotation(params.parameters[\"_f\"]))", + "got": "_param_get_annotation(params.parameters[\"_f\"]))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "0", + "got": "0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "6", + "got": "6", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "2", + "got": "2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "1", + "got": "0", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "0.0", + "got": "0.0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "None", + "got": "None", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "True", + "got": "True", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "mint", + "got": "mint", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "0.001", + "got": "\"0.000000000000000000\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2962962962962963 + }, + { + "repo": "michaelhly/solana-py", + "expected": "False", + "got": "True", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "michaelhly/solana-py", + "expected": "amount", + "got": "amount", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "0.0005", + "got": "0.005", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "michaelhly/solana-py", + "expected": "0.0003", + "got": "0.0003", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "str(0)", + "got": "str(500)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "michaelhly/solana-py", + "expected": "actual", + "got": "actual", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "params", + "got": "params", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "9999994000", + "got": "AIRDROP_AMOUNT + 1000", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.1935483870967742 + }, + { + "repo": "michaelhly/solana-py", + "expected": "9999988000", + "got": "9999994000", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "michaelhly/solana-py", + "expected": "9999982000", + "got": "9999987000", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9 + }, + { + "repo": "michaelhly/solana-py", + "expected": "10000001000", + "got": "1000", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5333333333333333 + }, + { + "repo": "michaelhly/solana-py", + "expected": "min_signers", + "got": "min_signers", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "js_wire_msg", + "got": "js_wire_msg", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "err.data.logs", + "got": "\"insufficient funds\" in err.message", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.20833333333333334 + }, + { + "repo": "michaelhly/solana-py", + "expected": "AIRDROP_AMOUNT", + "got": "AIRDROP_AMOUNT", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "owned.pubkey()", + "got": "owned.pubkey()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "decoded_params", + "got": "decoded_params", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "multisig_params", + "got": "multisig_params", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "RPC_RESULT_TYPES", + "got": "RPC_RESULT_TYPES", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "TOKEN_PROGRAM_ID", + "got": "TOKEN_PROGRAM_ID", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "resp_data.pubkey", + "got": "resp_data.is_initialized", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.55 + }, + { + "repo": "michaelhly/solana-py", + "expected": "stubbed_receiver", + "got": "stubbed_receiver.pubkey()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7804878048780488 + }, + { + "repo": "michaelhly/solana-py", + "expected": "msg.result.slots", + "got": "msg.result.vote_epoch > 0", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6341463414634146 + }, + { + "repo": "michaelhly/solana-py", + "expected": "params_no_freeze", + "got": "params_no_freeze", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "test_token.pubkey", + "got": "test_token.pubkey", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "expected_decimals", + "got": "expected_decimals", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "str(total_amount)", + "got": "str(total_amount)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "SolanaRpcException", + "got": "ReadTimeout", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.27586206896551724 + }, + { + "repo": "michaelhly/solana-py", + "expected": "security_text.name", + "got": "security_text", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8387096774193549 + }, + { + "repo": "michaelhly/solana-py", + "expected": "params_with_freeze", + "got": "params_with_freeze", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "security_text.policy", + "got": "security_text.policy", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "async_stubbed_receiver", + "got": "async_stubbed_receiver.pubkey()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8301886792452831 + }, + { + "repo": "michaelhly/solana-py", + "expected": "create_resp.value.data", + "got": "create_resp.value.amount", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8695652173913043 + }, + { + "repo": "michaelhly/solana-py", + "expected": "security_text.contacts", + "got": "security_text.contacts", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "expected_amount_delegated", + "got": "expected_amount_delegated", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "expected_token_account_key", + "got": "expected_token_account_key", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "AIRDROP_AMOUNT - amount - 5000", + "got": "9999999999", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "\"http://api.devnet.solana.com\"", + "got": "\"https://api.devnet.solana.com\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9836065573770492 + }, + { + "repo": "michaelhly/solana-py", + "expected": "\"https://api.devnet.solana.com\"", + "got": "\"https://api.devnet.solana.com\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "not decoded_params.new_authority", + "got": "decoded_params", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6086956521739131 + }, + { + "repo": "michaelhly/solana-py", + "expected": "isinstance(msg, LogsNotification)", + "got": "isinstance(msg, LogsNotification)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "isinstance(msg, SlotNotification)", + "got": "isinstance(msg, SlotNotification)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "isinstance(msg, RootNotification)", + "got": "isinstance(msg, RootNotification)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "isinstance(msg, VoteNotification)", + "got": "isinstance(msg, VoteNotification)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "isinstance(msg, BlockNotification)", + "got": "isinstance(msg, BlockNotification)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "isinstance(msg, SubscriptionResult)", + "got": "isinstance(msg, AccountNotification)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.647887323943662 + }, + { + "repo": "michaelhly/solana-py", + "expected": "not decoded_params.freeze_authority", + "got": "not decoded_params.freeze_authority", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "isinstance(msg, AccountNotification)", + "got": "isinstance(msg, AccountNotification)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "isinstance(msg, ProgramNotification)", + "got": "isinstance(msg, ProgramNotification)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "isinstance(msg, SignatureNotification)", + "got": "isinstance(msg, SignatureNotification)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "not isinstance(resp, RPCError.__args__)", + "got": "resp[\"result\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.22641509433962265 + }, + { + "repo": "michaelhly/solana-py", + "expected": "isinstance(msg, SlotUpdateNotification)", + "got": "isinstance(msg, SlotsUpdatesNotification)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.975 + }, + { + "repo": "michaelhly/solana-py", + "expected": "\"Program 11111111111111111111111111111111 invoke [1]\"", + "got": "\"Hello World\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.12121212121212122 + }, + { + "repo": "michaelhly/solana-py", + "expected": "\" raised in \\\"GetEpochInfo\\\" endpoint request\"", + "got": "\"placeholder\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.18604651162790697 + }, + { + "repo": "michaelhly/solana-py", + "expected": "\" raised in \\\"GetEpochInfo\\\" endpoint request\"", + "got": "\"placeholder\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.22988505747126436 + }, + { + "repo": "michaelhly/solana-py", + "expected": "resp)", + "got": "resp)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "resp[0])", + "got": "resp[0])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "resp[1])", + "got": "resp[1])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "tx_resp)", + "got": "tx_resp)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "fee_resp)", + "got": "fee_resp)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "burn_resp)", + "got": "burn_resp)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "mint_resp)", + "got": "mint_resp)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "thaw_resp)", + "got": "thaw_resp)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "info_resp)", + "got": "info_resp)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "revoke_resp)", + "got": "revoke_resp)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "airdrop_resp)", + "got": "airdrop_resp)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "transfer_resp)", + "got": "transfer_resp)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "test_token.mint_to(dest=stubbed_sender_token_account_pk, mint_authority=stubbed_sender, amount=1000, opts=OPTS))", + "got": "test_token.mint_to(stubbed_sender_token_account_pk, expected_amount))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7071823204419889 + }, + { + "repo": "michaelhly/solana-py", + "expected": "test_token.burn( account=stubbed_sender_token_account_pk, owner=stubbed_sender, amount=burn_amount, multi_signers=None, opts=OPTS, ))", + "got": "test_token.burn(source=stubbed_sender_token_account_pk, owner=stubbed_sender, amount=burn_amount))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8138528138528138 + }, + { + "repo": "michaelhly/solana-py", + "expected": "test_token.transfer( source=stubbed_sender_token_account_pk, dest=stubbed_receiver_token_account_pk, owner=stubbed_sender, amount=expected_amount, opts=OPTS, ))", + "got": "test_token.transfer(transfer_authority=stubbed_sender, dest=stubbed_receiver_token_account_pk, amount=500))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6591760299625468 + }, + { + "repo": "michaelhly/solana-py", + "expected": "test_token.burn_checked( account=stubbed_sender_token_account_pk, owner=stubbed_sender, amount=burn_amount, decimals=expected_decimals, multi_signers=None, opts=OPTS, ))", + "got": "test_token.burn_checked( account=stubbed_sender_token_account_pk, owner=stubbed_sender, amount=burn_amount, decimals=expected_decimals, multi_signers=None, opts=OPTS, ))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "test_token.mint_to_checked( dest=stubbed_sender_token_account_pk, mint_authority=stubbed_sender, amount=mint_amount, decimals=expected_decimals, multi_signers=None, opts=OPTS, ))", + "got": "test_token.mint_to_checked( source=stubbed_sender_token_account_pk, mint_authority=stubbed_sender, amount=mint_amount, decimals=expected_decimals, opts=OPTS,))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9112426035502958 + }, + { + "repo": "michaelhly/solana-py", + "expected": "test_token.transfer_checked( source=stubbed_sender_token_account_pk, dest=stubbed_receiver_token_account_pk, owner=stubbed_sender, amount=transfer_amount, decimals=expected_decimals, multi_signers=None, opts=OPTS, ))", + "got": "test_token.transfer_checked( source=stubbed_sender_token_account_pk, dest=stubbed_receiver_token_account_pk, owner=stubbed_sender, amount=transfer_amount, decimals=expected_decimals, multi_signers=None, opts=OPTS, ))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "TypeError)", + "got": "TypeError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "RPCException)", + "got": "RPCException)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "SolanaRpcException)", + "got": "SolanaRpcException)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "NoSecurityTxtFoundError)", + "got": "NoSecurityTxtFoundError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "TransactionExpiredBlockheightExceededError)", + "got": "RPCException)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.21428571428571427 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "1", + "got": "self.isvr._sio._clients", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "0", + "got": "0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "4", + "got": "3", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "3", + "got": "2", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "2", + "got": "2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "r", + "got": "r", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "5", + "got": "5", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "n", + "got": "n", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "{}", + "got": "{}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "[]", + "got": "[1, '2']", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "32", + "got": "32", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "10", + "got": "10", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "60", + "got": "0.01", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "12", + "got": "0.01", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "ep", + "got": "ep", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "sid", + "got": "sid", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "pkt", + "got": "'789/bar,[\"my event\",{\"foo\":\"bar\"}]'", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.05128205128205128 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "mgr", + "got": "mgr", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "foo", + "got": "foo", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "bar", + "got": "bar", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'n'", + "got": "'n'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'1'", + "got": "'2'", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'2'", + "got": "'2'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "123", + "got": "123", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "True", + "got": "True", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "None", + "got": "None", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'ok'", + "got": "'ok'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "json", + "got": "json", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "0.01", + "got": "10", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'url'", + "got": "'url'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "['/']", + "got": "[]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "False", + "got": "False", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'foo'", + "got": "'foo'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'baz'", + "got": "'baz'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'123'", + "got": "True", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'456'", + "got": "'456'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "rooms", + "got": "rooms", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'sid'", + "got": "'sid'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'bar'", + "got": "'bar'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'789'", + "got": "'789'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "p2.id", + "got": "p2.id", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "rdata", + "got": "rdata", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'auth'", + "got": "'auth'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'path'", + "got": "'path'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'/bar'", + "got": "'/bar'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'2123'", + "got": "'2123,'", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9230769230769231 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "b'456'", + "got": "b'789'", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "['foo']", + "got": "'foo'", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "p2.data", + "got": "p2.data", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "connect", + "got": "connect", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "b'1234'", + "got": "b'1234'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "[1, '2']", + "got": "[1, '2']", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'4hello'", + "got": "'hello'", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9333333333333333 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "(1, '2')", + "got": "(1, '2')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'result'", + "got": "'result'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'2/bar,'", + "got": "'2/bar'", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9333333333333333 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'/b-a-r'", + "got": "'/b-a-r'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "('/foo',)", + "got": "('/foo',)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'bar/foo'", + "got": "'/foo'", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'polling'", + "got": "'polling'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "my_logger", + "got": "my_logger", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'environ'", + "got": "'environ'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "namespaces", + "got": "namespaces", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'socketio'", + "got": "'/'", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "('1', 'a')", + "got": "('foo', 'a')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8181818181818182 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "('a', 'b')", + "got": "('baz', 'a', 'b')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7407407407407407 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "{'a': 'b'}", + "got": "{'a': 'b'}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "packet.ACK", + "got": "packet.ACK", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'2[\"foo\"]'", + "got": "'1,foo'", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5882352941176471 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'3[\"foo\"]'", + "got": "'3[\"foo\"]'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "c.connected", + "got": "not c.connected", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8461538461538461 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'websocket'", + "got": "'websocket'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "test_logger", + "got": "test_logger", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "('foo', 321)", + "got": "321", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "{sid: '123'}", + "got": "dict(self.bm.rooms['/foo'][sid])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.13636363636363635 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "participants", + "got": "participants", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "logging.INFO", + "got": "logging.DEBUG", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.64 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "CustomPacket", + "got": "CustomPacket", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "{'/': '123'}", + "got": "{'/': '123'}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "p2.namespace", + "got": "p2.namespace", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "packet.EVENT", + "got": "packet.EVENT", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "pkt.encode()", + "got": "pkt.packet_type == packet.EVENT", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.27906976744186046 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'2[\"f-o-o\"]'", + "got": "'2[\"f-o-o\"]'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "{'/foo': '1'}", + "got": "{'/foo': '1'}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "('/foo', 'a')", + "got": "('/foo', 'a')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "{sid2: '456'}", + "got": "{sid2: 'baz'}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7692307692307693 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "{sid3: '456'}", + "got": "{sid3: '456'}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "{sid4: '456'}", + "got": "{sid4: '123'}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7692307692307693 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "logging.ERROR", + "got": "logging.WARNING", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6428571428571429 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "mock_client()", + "got": "mock_client.return_value", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5945945945945946 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'2123[\"foo\"]'", + "got": "'2/foo,123'", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "[b'789', 123]", + "got": "[b'789', 123]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "\"0123456789-\"", + "got": "\"0123456789-\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "packet.Packet", + "got": "packet.Packet", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "('sid', 'foo')", + "got": "('sid', 'foo')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "{'foo': 'bar'}", + "got": "fake_session", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.15384615384615385 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "[b'Not Found']", + "got": "404", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "int('1' * 100)", + "got": "int('1' * 100)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "{'/bar': '123'}", + "got": "{}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.23529411764705882 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "not c.connected", + "got": "not c.connected", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "{'foo': b'bar'}", + "got": "{'foo': b'bar'}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'2/bar,[\"foo\"]'", + "got": "'2[\"foo\"]'", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "s.async_handlers", + "got": "s.eio.on.return_value", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.21621621621621623 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "packet.BINARY_ACK", + "got": "packet.BINARY_EVENT", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7777777777777778 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "self.bm.callbacks", + "got": "self.bm.rooms", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'42[\"foo\",\"bar\"]'", + "got": "'foo'", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.45454545454545453 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'+++ decoded +++'", + "got": "{'foo': 'bar'}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.1935483870967742 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "packet.DISCONNECT", + "got": "packet.CONNECT", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9032258064516129 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'2/b-a-r,[\"foo\"]'", + "got": "'2/b-a-r,[\"foo\"]'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "p2.data[\"current\"]", + "got": "p2.data[\"current\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "self.pm.rooms['/']", + "got": "self.pm.rooms['/']", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "{'123': 'environ'}", + "got": "'environ'", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "('1', '/foo', 'a')", + "got": "('1', '/foo', 'a')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "('/foo', 'a', 'b')", + "got": "('/foo', 'a', 'b')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'2*** encoded ***'", + "got": "'*** encoded ***'", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9714285714285714 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'2/bar,123[\"foo\"]'", + "got": "'2/bar,123,[\"foo\"]'", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.972972972972973 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "c.callbacks['/foo']", + "got": "c.callbacks['/foo']", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "packet.BINARY_EVENT", + "got": "packet.EVENT", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7741935483870968 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "c.is_asyncio_based()", + "got": "c.is_asyncio_based", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9473684210526315 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "not client1.connected", + "got": "not client1.connected", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "self.bm.rooms['/foo']", + "got": "self.bm.rooms['/foo']", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'42/foo,[\"my event\"]'", + "got": "'42/foo,[\"my event\",null]'", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8936170212765957 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "s.eio.send.call_count", + "got": "s.eio.send.call_count", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "s.eio.send.await_count", + "got": "s.eio.send.await_count", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "['foo', {'bar': 'baz'}]", + "got": "['room', 'foo', {'bar': 'baz'}]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8518518518518519 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "('sid', {'foo': 'bar'})", + "got": "('sid', {'foo': 'bar'})", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "threading.active_count()", + "got": "threading.active_count()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "expected_packet.encode()", + "got": "expected_packet.encode()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "not c.is_asyncio_based()", + "got": "c.is_asyncio_based", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "TypeError)", + "got": "TypeError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "ValueError)", + "got": "ConnectionError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5185185185185185 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "RuntimeError)", + "got": "ValueError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "TimeoutError)", + "got": "TimeoutError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "ConnectionError)", + "got": "ConnectionError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "DisconnectedError)", + "got": "DisconnectedError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "exceptions.TimeoutError)", + "got": "exceptions.TimeoutError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "exceptions.ConnectionError)", + "got": "exceptions.SocketIOError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7692307692307693 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "1)", + "got": "0)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "3)", + "got": "3)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "2)", + "got": "'bar')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.25 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "())", + "got": "[], namespace=None)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.09090909090909091 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'/')", + "got": "'/')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'2')", + "got": "'2')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'1')", + "got": "'1', 'foo')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5333333333333333 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'b')", + "got": "'b')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'c')", + "got": "'c')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "123)", + "got": "'url')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'cb')", + "got": "'foo')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5454545454545454 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "None)", + "got": "None)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "1.23)", + "got": "1.23)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'ev')", + "got": "'ev')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'url')", + "got": "'url')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'bar')", + "got": "'bar')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'baz')", + "got": "'baz')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'foo')", + "got": "'foo')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'sid')", + "got": "'sid')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'123')", + "got": "'123')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'/foo')", + "got": "'/foo')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'/baz')", + "got": "'/baz')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'/bar')", + "got": "'/bar')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'data')", + "got": "'data')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'room')", + "got": "'room')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "b'bar')", + "got": "['YmFy'], namespace=None)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.1875 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'path')", + "got": "'path')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "b'foo')", + "got": "b'foo')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'error')", + "got": "'error')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'2\"foo\"')", + "got": "'foo')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'1/foo,')", + "got": "'2')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'connect')", + "got": "'connect')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'message')", + "got": "'message')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'environ')", + "got": "'environ')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'31000[]')", + "got": "'21000[\"my message\",\"foo\"]')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.47368421052631576 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "[1, 'two'])", + "got": "[1, 'two'])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "(1, 'two'))", + "got": "(1, 'two'))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "('one', 2))", + "got": "('one', 2))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "('bar', 'baz'))", + "got": "('bar', 'baz'))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'0{\"sid\":\"1\"}')", + "got": "'0{\"token\":\"abc\"}')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6470588235294118 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'31000[\"foo\"]')", + "got": "'21000[\"my message\",\"foo\"]')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6511627906976745 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "start_response)", + "got": "start_response)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "self.pm._thread)", + "got": "self.pm._publish)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "{'foo': b'bar'})", + "got": "b'bar', namespace=None)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.358974358974359 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'start_response')", + "got": "'start_response')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "c._handle_reconnect)", + "got": "c._handle_reconnect)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'0/foo,{\"sid\":\"1\"}')", + "got": "'0/foo,')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6206896551724138 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'31000[1,\"2\",true]')", + "got": "'31000[\"a\",\"b\",\"c\"]')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6341463414634146 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'__disconnect_final')", + "got": "'reconnect')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6060606060606061 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "[1, b'two', 'three'])", + "got": "[1, b'bar', 'three'], namespace=None)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6206896551724138 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "(1, b'two', 'three'))", + "got": "(1, b'two', 'three'), namespace=None)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7241379310344828 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'31000[[1,\"2\",true]]')", + "got": "'31000[1,\"2\",true]')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9523809523809523 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "c.reason.CLIENT_DISCONNECT)", + "got": "'foo')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.12121212121212122 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "s.reason.CLIENT_DISCONNECT)", + "got": "'/foo')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.11764705882352941 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "['my message', 'a', b'foo'])", + "got": "b'foo')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'2/foo,[\"my event\",\"my data\"]')", + "got": "[packet.EVENT, ['my event', 'my data']])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5352112676056338 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "[('Content-Type', 'text/plain')])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'4/foo,{\"message\":\"fail_reason\",\"data\":1}')", + "got": "'4/foo,{\"message\":\"fail_reason\"}')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8831168831168831 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "{'method': 'emit', 'value': 'foo', 'host_id': 'x'})", + "got": "'foo')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.21052631578947367 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "{'method': 'emit', 'value': 'bar', 'host_id': 'x'})", + "got": "{'method': 'emit', 'value': 'bar'})", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.813953488372093 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'4/foo,{\"message\":\"Connection rejected by server\"}')", + "got": "'4/foo,{\"message\":\"Connection rejected by server\"}')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "{'method': 'callback', 'value': 'bar', 'host_id': 'x'})", + "got": "{'method': 'callback', 'value': 'bar'})", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8297872340425532 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "{'method': 'close_room', 'value': 'baz', 'host_id': 'x'})", + "got": "{'method': 'close_room', 'value': 'baz'})", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8367346938775511 + }, + { + "repo": "oduwsdl/ipwb", + "expected": "2", + "got": "ipwb_test.count_warc_records(new_warc_path)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "oduwsdl/ipwb", + "expected": "1", + "got": "3", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "oduwsdl/ipwb", + "expected": "404", + "got": "302", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "oduwsdl/ipwb", + "expected": "200", + "got": "301", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "oduwsdl/ipwb", + "expected": "400", + "got": "404", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "oduwsdl/ipwb", + "expected": "True", + "got": "True", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "oduwsdl/ipwb", + "expected": "False", + "got": "False", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "oduwsdl/ipwb", + "expected": "status", + "got": "status", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "oduwsdl/ipwb", + "expected": "location", + "got": "location", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "oduwsdl/ipwb", + "expected": "resp.headers", + "got": "resp.headers", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "oduwsdl/ipwb", + "expected": "resp.content", + "got": "resp.text.encode('utf-8')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.43243243243243246 + }, + { + "repo": "oduwsdl/ipwb", + "expected": "'https://example.com'", + "got": "'https://example.com'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "oduwsdl/ipwb", + "expected": "util.pad_digits14(input)", + "got": "util.pad_digits14(input)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "oduwsdl/ipwb", + "expected": "bool(replay.is_uri(input))", + "got": "ipwb_test.is_uri(input)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6122448979591837 + }, + { + "repo": "oduwsdl/ipwb", + "expected": "'Cannot load index file from ipfs.'", + "got": "'IPFS error: StatusError(original=) '", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "oduwsdl/ipwb", + "expected": "'https://example.com?foo=bar&boo=baz'", + "got": "(", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "oduwsdl/ipwb", + "expected": "check_cdxj_fields(first_non_metadata_entry)", + "got": "check_cdxj_fields(first_non_metadata_entry)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "oduwsdl/ipwb", + "expected": "check_ipwb_json_field_presence(first_entry_last__field)", + "got": "first_entry_last__field[0] == '{'", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5227272727272727 + }, + { + "repo": "oduwsdl/ipwb", + "expected": "onlyOneMemento and \\ cond_first_memento and \\ cond_last_memento", + "got": "onlyOneMemento", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "oduwsdl/ipwb", + "expected": "get_web_archive_index(SAMPLE_INDEX).startswith( '!context [\"https://tools.ietf.org/html/rfc7089\"]' )", + "got": "get_web_archive_index('http://localhost:8080')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5068493150684932 + }, + { + "repo": "oduwsdl/ipwb", + "expected": "cond_first_memento and \\ cond_last_next_memento and \\ cond_first_prev_memento and \\ cond_last_memento", + "got": "cond_first_memento and cond_last_next_memento and cond_first_prev_memento and cond_last_memento", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9693877551020408 + }, + { + "repo": "oduwsdl/ipwb", + "expected": "get_web_archive_index( 'QmReQCtRpmEhdWZVLhoE3e8bqreD8G3avGpVfcLD7r4K6W' ).startswith('!context [\"https://tools.ietf.org/html/rfc7089\"]')", + "got": "get_web_archive_index('ipfs://QmW7ZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZzZz", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.1497584541062802 + }, + { + "repo": "oduwsdl/ipwb", + "expected": "get_web_archive_index( 'ipfs://QmReQCtRpmEhdWZVLhoE3e8bqreD8G3avGpVfcLD7r4K6W' ).startswith('!context [\"https://tools.ietf.org/html/rfc7089\"]')", + "got": "get_web_archive_index('https://ipfs.io/ipfs/QmReQCtRpmEhdWZVLhoE3e8bqreD8G3avGpVfcLD7r4K6W')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6553191489361702 + }, + { + "repo": "oduwsdl/ipwb", + "expected": "get_web_archive_index( 'https://raw.githubusercontent.com/oduwsdl/ipwb/master/samples/' + 'indexes/salam-home.cdxj' ).startswith('!context [\"https://tools.ietf.org/html/rfc7089\"]')", + "got": "get_web_archive_index('https://tools.ietf.org/archive/index.html')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.42276422764227645 + }, + { + "repo": "oduwsdl/ipwb", + "expected": "cond_m1m1_first_memento and cond_m1m2_next_memento and cond_m1m3_last_memento and cond_m2m1_first_prev_memento and cond_m2m2_memento and cond_m2m3_last_next_memento and cond_m3m1_first_memento and cond_m3m2_prev_memento and cond_m3m3_last_memento", + "got": "cond_m1m1_first_memento and \\", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.20363636363636364 + }, + { + "repo": "oduwsdl/ipwb", + "expected": "cond_m1m1_first_memento and cond_m1m2_next_memento and", + "got": "(cond_m1m1_first_memento and", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6585365853658537 + }, + { + "repo": "oduwsdl/ipwb", + "expected": "cond_m1m1_first_memento and cond_m1m2_next_memento and", + "got": "(cond_m1m1_first_memento and", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6585365853658537 + }, + { + "repo": "oduwsdl/ipwb", + "expected": "ValueError)", + "got": "ValueError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "oduwsdl/ipwb", + "expected": "BackendError)", + "got": "BackendError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "oduwsdl/ipwb", + "expected": "Exception, match='foo')", + "got": "Exception)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6060606060606061 + }, + { + "repo": "oduwsdl/ipwb", + "expected": "Exception, match=expected_error)", + "got": "Exception)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.47619047619047616 + }, + { + "repo": "oduwsdl/ipwb", + "expected": "multiaddr_exceptions.StringParseError)", + "got": "multiaddr_exceptions.InvalidScheme)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.684931506849315 + }, + { + "repo": "oduwsdl/ipwb", + "expected": "Exception, match='Daemon is not running at')", + "got": "ConnectionError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.23333333333333334 + }, + { + "repo": "oduwsdl/ipwb", + "expected": "Exception, match='IPFS is likely not installed')", + "got": "Exception, match='Cannot connect to IPFS daemon')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5360824742268041 + }, + { + "repo": "oduwsdl/ipwb", + "expected": "'boo')", + "got": "'boo')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "1", + "got": "2", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "2", + "got": "2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "0", + "got": "0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "3", + "got": "3", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "4", + "got": "4", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "{}", + "got": "{}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "200", + "got": "200", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "res", + "got": "True", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "url", + "got": "url", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "None", + "got": "\"{}\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "chunk.id", + "got": "chunk.id", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "question", + "got": "question", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "\"chunk1\"", + "got": "\"chunk1\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "\"chunk2\"", + "got": "\"chunk2\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "'mongodb'", + "got": "\"mongodb\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7777777777777778 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "chunk_ids", + "got": "True", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.15384615384615385 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "is_success", + "got": "is_success", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "len(chunks)", + "got": "N_TEST_CHUNKS", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "caplog.text", + "got": "caplog.text", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "len(queries)", + "got": "len(queries)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "\"Fred Smith\"", + "got": "\"Fred Smith\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "expected_result", + "got": "expected_result", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "len(document_ids)", + "got": "len(document_ids)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "len(query_results)", + "got": "len(query_results[0].results)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7659574468085106 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "EMBEDDING_DIMENSION", + "got": "1", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "created_at.data_type", + "got": "created_at.type", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "expected_status_code", + "got": "expected_status_code", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "expected_num_results", + "got": "expected_num_results", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "document_id.data_type", + "got": "document_id.data_type", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "query_results[0].query", + "got": "query_results[0].query", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "query_results[1].query", + "got": "query_results[1].query", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "f\"{field} eq '{value}'\"", + "got": "f\"\\\"{field}\\\" eq '{needs_escaping_value}'\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7076923076923077 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "delete.json()['success']", + "got": "not delete.json()[\"success\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8461538461538461 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "new_store.col.num_entities", + "got": "new_store.col.num_entities", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "first_document_chunk.score", + "got": "first_document_chunk.score", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "list(document_chunks.keys())", + "got": "True", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.125 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "len(query_results[0].results)", + "got": "len(query_results)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7659574468085106 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "query_results[0].results[0].id", + "got": "query_results[0].results[0].id", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "collection_info.payload_schema", + "got": "collection_info.payload_schema", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "query_results[0].results[i].id", + "got": "query_results[0].results[i].source", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.875 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "list(document_chunk_one.keys())", + "got": "True", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.11428571428571428 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "list(document_chunk_two.keys())", + "got": "list(document_chunk_two.keys())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "query_results[0].results[i].text", + "got": "query_results[0].documents[i].text", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8484848484848485 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "zilliz_datastore.col.has_index()", + "got": "await zilliz_datastore.is_available()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "len(document_chunks[\"first-doc\"])", + "got": "1", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "milvus_datastore.col.num_entities", + "got": "len(milvus_datastore.col.indexes)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7272727272727273 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "\"Aenean euismod bibendum laoreet\"", + "got": "\"Aenean euismod bibendum laoreet\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "len(document_chunks[\"second-doc\"])", + "got": "len(document_chunks[\"second-doc\"])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "client.admin.command('ping')['ok']", + "got": "client", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "{\"question\", \"answer\", \"category\"}", + "got": "[\"question\", \"answer\", \"category\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9411764705882353 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "index[\"index_param\"][\"index_type\"]", + "got": "index[\"index_type\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7169811320754716 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "\"doc:deleteDoc1:chunk:deleteChunk1\"", + "got": "\"deleteChunk1\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "int(os.environ['EMBEDDING_DIMENSION'])", + "got": "3", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "azuresearch_mgmt_client.list_index_names()", + "got": "azuresearch_mgmt_client.list_index_names()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "[doc_id for doc_id in initial_document_chunks]", + "got": "[\"first-doc-4\", \"first-doc-5\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2894736842105263 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "await store.delete([\"test_id_1\", \"test_id_2+\"])", + "got": "not await store.upsert([Document(id=\"test_id_3\", text=\"test text\")])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6260869565217392 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "client.count(collection_name=\"documents\").count", + "got": "len(await client.list_points())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.38461538461538464 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "list( initial_document_chunks.keys() )", + "got": "list(initial_document_chunks.keys())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.972972972972973 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "{ \"$and\": [ {\"metadata.source\": \"email\"} ] }", + "got": "{\"$and\": [{\"metadata.source\": \"email\"}]}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9523809523809523 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "sum( len(v) for v in document_chunks.values() )", + "got": "len(document_chunks)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5970149253731343 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "{ \"$and\": [ {\"created_at\": {\"$gte\": to_unix_timestamp(date)}} ] }", + "got": "{\"start_date\": date}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.35294117647058826 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "{ \"$and\": [ {\"created_at\": {\"$lte\": to_unix_timestamp(date)}} ] }", + "got": "{\"$and\": [{\"created_at\": {\"$lte\": to_unix_timestamp(date)}}]}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9682539682539683 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "len(document_chunks[\"first-doc\"]) + len( document_chunks[\"second-doc\"] )", + "got": "len(document_chunks)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.43478260869565216 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "all( [ result.id in [chunk.id for chunk in chunks] for result in query_results[0].results ] )", + "got": "all(r.document_id == doc_id for r in query_results[0].results)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6193548387096774 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "all( [ result.id in [chunk.id for chunk in document_chunks[\"first-doc\"]] for result in query_results[0].results ] )", + "got": "all(", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.06722689075630252 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "all( [ result.id in [chunk.id for chunk in document_chunks[\"second-doc\"]] for result in query_results[0].results ] )", + "got": "all(", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.06666666666666667 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "[ f\"doc:{doc_id}:chunk:{chunk.id}\" for doc_id, chunk_list in initial_document_chunks.items() for chunk in chunk_list ]", + "got": "[\"first-doc-4\", \"first-doc-5\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.13513513513513514 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "ValueError)", + "got": "ValueError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "ValueError, match=\"WEAVIATE_API_KEY environment variable is not set\")", + "got": "KeyError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.1794871794871795 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "predicate)", + "got": "predicate)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "lambda: res == chunk_ids)", + "got": "collection_size_callback_factory(mongodb_datastore.client[mongodb_datastore.database_name][mongodb_datastore.collection_name], 3))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.09032258064516129 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "collection_size_callback_factory(collection, 3))", + "got": "collection_size_callback_factory(collection, 3))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "collection_size_callback_factory(collection, 2))", + "got": "lambda: not await collection.find_one({\"_id\": first_id}))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3238095238095238 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "collection_size_callback_factory(collection, 0))", + "got": "collection_size_callback_factory(collection, 0))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "oughtinc/ice", + "expected": "20", + "got": "0", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "oughtinc/ice", + "expected": "{}", + "got": "{", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "oughtinc/ice", + "expected": "[]", + "got": "[]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "oughtinc/ice", + "expected": "[1]", + "got": "[1]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "oughtinc/ice", + "expected": "None", + "got": "None", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "oughtinc/ice", + "expected": "0.99", + "got": "0.9", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "oughtinc/ice", + "expected": "thing", + "got": "thing", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "oughtinc/ice", + "expected": "\"Foo\"", + "got": "\"Foo\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "oughtinc/ice", + "expected": "3 / 4", + "got": "0.0", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "oughtinc/ice", + "expected": "1 / 3", + "got": "3 / 5", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6 + }, + { + "repo": "oughtinc/ice", + "expected": "[\"0\"]", + "got": "[\"0\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "oughtinc/ice", + "expected": "[\"1\"]", + "got": "[\"(1, 2, 3)\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5555555555555556 + }, + { + "repo": "oughtinc/ice", + "expected": "[\"b\"]", + "got": "[\"b\", \"d\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "oughtinc/ice", + "expected": "\"key\"", + "got": "\"key\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "oughtinc/ice", + "expected": "things", + "got": "things", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "oughtinc/ice", + "expected": "(0, 0)", + "got": "\"foo\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "oughtinc/ice", + "expected": "(0, 1)", + "got": "(1, 0)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "oughtinc/ice", + "expected": "(0, 2)", + "got": "(0, len(long))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "oughtinc/ice", + "expected": "events", + "got": "events", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "oughtinc/ice", + "expected": "expected", + "got": "expected", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "oughtinc/ice", + "expected": "[1, 2, 3]", + "got": "[1, 2, 3]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "oughtinc/ice", + "expected": "[0, 1, 2]", + "got": "{\"range\": {\"start\": 0, \"end\": 10}}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.18604651162790697 + }, + { + "repo": "oughtinc/ice", + "expected": "expected_result", + "got": "expected_result", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "oughtinc/ice", + "expected": "scores_repeated", + "got": "scores_repeated", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "oughtinc/ice", + "expected": "{\"a\": 1, \"2\": 3}", + "got": "{\"a\": 1, \"2\": 3}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "oughtinc/ice", + "expected": "{\"name\": \"world\"}", + "got": "\"hello world\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6 + }, + { + "repo": "oughtinc/ice", + "expected": "f\"{special} three\"", + "got": "\"1 2 3\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.24 + }, + { + "repo": "oughtinc/ice", + "expected": "{\"stuff\": {\"a\": 1}}", + "got": "{\"stuff\": {\"a\": 1}}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "oughtinc/ice", + "expected": "{\"things\": [1, 2, 3]}", + "got": "\"{'things': [1, 2, 3]}\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8636363636363636 + }, + { + "repo": "oughtinc/ice", + "expected": "complex_format_string", + "got": "complex_format_string", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "oughtinc/ice", + "expected": "\"pizza and ice cream\"", + "got": "\"pizza and ice cream\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "oughtinc/ice", + "expected": "not paragraph.is_empty()", + "got": "paragraph", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5454545454545454 + }, + { + "repo": "oughtinc/ice", + "expected": "{\"f\": {\"name\": \"world\"}}", + "got": "{\"name\": \"world!\"}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8095238095238095 + }, + { + "repo": "oughtinc/ice", + "expected": "calculated_reduced_scores", + "got": "calculated_reduced_scores", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "oughtinc/ice", + "expected": "gold_paragraphs_df.columns", + "got": "id_to_paragraph", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5365853658536586 + }, + { + "repo": "oughtinc/ice", + "expected": "\"1. 1...2. 2...3. 3 three\"", + "got": "\"1...2...3 three\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7906976744186046 + }, + { + "repo": "oughtinc/ice", + "expected": "isinstance(xform, ValueTransform)", + "got": "isinstance(xform, str)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7636363636363637 + }, + { + "repo": "oughtinc/ice", + "expected": "len(resp) == 1 and len(resp[0]) == 1", + "got": "resp[0].dict()[\"nubia\"][\"score\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.23529411764705882 + }, + { + "repo": "oughtinc/ice", + "expected": "isinstance(xform, DependentTransform)", + "got": "xform(case)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "oughtinc/ice", + "expected": "\"1. Paragraph Uno.\\n\\n2. Paragraph Dos\"", + "got": "\"\\n\\n\".join(value)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.21052631578947367 + }, + { + "repo": "oughtinc/ice", + "expected": "(\"pizza and fries\", \"hotdogs and fries\")", + "got": "(\"pizza and fries\", \"hotdogs and fries\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "oughtinc/ice", + "expected": "np.isclose(calculated_reduced_scores[k], v)", + "got": "calculated_reduced_scores[k] == v", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7894736842105263 + }, + { + "repo": "oughtinc/ice", + "expected": "\"abcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabc...cabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabc\"", + "got": "\"abc\" * 100", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.08849557522123894 + }, + { + "repo": "oughtinc/ice", + "expected": "\"Disclosure forms provided by the authors are available with the full text of this article at NEJM.org.\"", + "got": "\"However, the study was not able to determine the causal relationship between mass distribution of the antibiotic agent and mortality.\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.37656903765690375 + }, + { + "repo": "oughtinc/ice", + "expected": "\"The proportion of children whose census status was recorded as moved or unknown did not differ significantly between the groups (P=0.71 and P=0.36, respectively).\"", + "got": "\"Disclosure forms provided by the authors are available with the full text of this article at NEJM.org.\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.21641791044776118 + }, + { + "repo": "oughtinc/ice", + "expected": "\"In this cluster-randomized trial, we assigned communities in Malawi, Niger, and Tanzania to four twice-yearly mass distributions of either oral azithromycin (approxi- mately 20 mg per kilogram of body weight) or placebo.\"", + "got": "\"We hypothesized that mass distribution of a broad-spectrum antibiotic agent to preschool children would reduce mortality in areas of sub-Saharan Africa that are currently far from meeting the Sustainable Development Goals of the United Nations.\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.1452991452991453 + }, + { + "repo": "oughtinc/ice", + "expected": "\"We hypothesized that mass distribution of a broad-spectrum antibiotic agent to preschool children would reduce mortality in areas of sub-Saharan Africa that are currently far from meeting the Sustainable Development Goals of the United Nations.\"", + "got": "\"Introduction\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.015384615384615385 + }, + { + "repo": "oughtinc/ice", + "expected": "{ \"a\": \"b\", }", + "got": "{}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.26666666666666666 + }, + { + "repo": "oughtinc/ice", + "expected": "{ \"a\": { \"b\": { \"c\": 3, } } }", + "got": "{\"a\": {\"b\": {\"c\": 3}}}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8627450980392157 + }, + { + "repo": "oughtinc/ice", + "expected": "{ \"a\": { \"b\": { \"e\": 4, }, } }", + "got": "{\"a\": {\"b\": {\"e\": 4}}}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8461538461538461 + }, + { + "repo": "oughtinc/ice", + "expected": "{ \"class_name\": \"function\", \"name\": \"foo\", }", + "got": "\"function\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.37037037037037035 + }, + { + "repo": "oughtinc/ice", + "expected": "{ 0: 0, 1: 1, 2: 2, 3: 3, 4: 4, 5: 5, 6: 6, 7: 7, 8: 8, 9: 9, }", + "got": "d", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "oughtinc/ice", + "expected": "( \"We can start in Egypt and keep going to France.\", \"We can start partying.\", )", + "got": "\"We can start in Egypt and keep going {there}. We can start {here} and keep going {there}.\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7134502923976608 + }, + { + "repo": "oughtinc/ice", + "expected": "( \"We can start partying.\", \"We can start in Egypt and keep going to France.\", )", + "got": "(\"We can start partying.\",)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5046728971962616 + }, + { + "repo": "oughtinc/ice", + "expected": "{ \"things\": [ { \"stuff\": { \"foo\": { \"things\": [ \"a\", \"b\", ] }, \"bar\": 9, } }, \"c\", ] }", + "got": "{\"things\": [{\"stuff\": {\"foo\": [\"a\", \"b\"]}}, \"c\"]}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.725925925925926 + }, + { + "repo": "oughtinc/ice", + "expected": "{ \"__fstring__\": [ \"hello \", {\"source\": \"name\", \"value\": \"world\", \"formatted\": \"world\"}, ] }", + "got": "\"hello world\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.24761904761904763 + }, + { + "repo": "oughtinc/ice", + "expected": "{ \"__fstring__\": [ \"hello \", {\"formatted\": \"3\", \"source\": \"1 + 2\", \"value\": 3}, \" world\", ] }", + "got": "\"hello 3 world\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2777777777777778 + }, + { + "repo": "oughtinc/ice", + "expected": "{ \"a\": { \"b\": [1, 2, 3], \"c\": { \"d\": \"abcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabc...cabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabc\", \"e\": 1.2346, }, } }", + "got": "{(\"f\", None): None}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.1005586592178771 + }, + { + "repo": "oughtinc/ice", + "expected": "KeyError)", + "got": "TypeError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7368421052631579 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "1", + "got": "1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "3", + "got": "3", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "0", + "got": "0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "2", + "got": "2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "5", + "got": "1", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "{}", + "got": "{}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "[]", + "got": "[\"\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "100", + "got": "100", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "130", + "got": "-1", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "True", + "got": "True", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "None", + "got": "None", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "9999", + "got": "9999", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "False", + "got": "False", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"npx\"", + "got": "\"npx\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"uvx\"", + "got": "\"curl\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5454545454545454 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"bunx\"", + "got": "\"bunx\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"mcpm\"", + "got": "\"test\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"node\"", + "got": "\"node\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "[0, 1]", + "got": "(0, 1)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "config", + "got": "config", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"tool1\"", + "got": "\"tool1\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "success", + "got": "success", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "stashed", + "got": "stashed", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"ERROR\"", + "got": "\"INFO\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "servers", + "got": "servers", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"stdio\"", + "got": "\"stdio\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"docker\"", + "got": "\"docker\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"python\"", + "got": "\"python -m test_server\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5161290322580645 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "profiles", + "got": "profiles", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"global\"", + "got": "\"test_server\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.19047619047619047 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "expected", + "got": "expected", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"qwen-cli\"", + "got": "\"test_key\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"Qwen CLI\"", + "got": "\"Qwen CLI\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "caplog.text", + "got": "caplog.text", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "custom_path", + "got": "custom_path", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "server_list", + "got": "server_list", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "backup_names", + "got": "backup_names", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"test-value\"", + "got": "\"test-value\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "saved_config", + "got": "saved_config", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"sample-key\"", + "got": "\"sample-key\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "result.output", + "got": "result.stderr", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "[\"server.js\"]", + "got": "[\"server.js\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "expected_path", + "got": "expected_path", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"api_profile\"", + "got": "\"api_profile\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"test-server\"", + "got": "\"test-server\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"sk-test-123\"", + "got": "\"sk-test-123\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "vscode_format", + "got": "vscode_format", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"shared-server\"", + "got": "\"shared-server\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"sample-server\"", + "got": "\"sample-server\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "windsurf_format", + "got": "windsurf_format", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "middleware_types", + "got": "middleware_types", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"TOOL_INVOCATION\"", + "got": "\"TOOL_INVOCATION\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"test-sse-server\"", + "got": "\"test-sse-server\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "config.mcpServers", + "got": "config.mcpServers", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "Path(legacy_path)", + "got": "legacy_path", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7857142857142857 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"sk-complete-123\"", + "got": "\"sk-complete-123\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "config[\"servers\"]", + "got": "config[\"servers\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "supported_clients", + "got": "supported_clients", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "{\"mcpServers\": {}}", + "got": "{}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"computercontroller\"", + "got": "\"computercontroller\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "[\"-m\", \"test_server\"]", + "got": "[\"-m\", \"test_server\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "result.output.lower()", + "got": "result.output", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7647058823529411 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "profile_servers[0].name", + "got": "\"profiles\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.48484848484848486 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "[\"arg1\", \"arg2\", \"arg3\"]", + "got": "[\"arg1\", \"arg2\", \"arg3\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "detector.has_v1_config()", + "got": "detector.detect_v1_config()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8235294117647058 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "features[\"share_status\"]", + "got": "not features[\"auth_config\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5490196078431373 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "isinstance(servers, dict)", + "got": "not servers", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "features[\"active_target\"]", + "got": "features[\"auth_config\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"Profile for API testing\"", + "got": "\"Profile for API testing\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"Alibaba's Qwen CLI tool\"", + "got": "\"Qwen CLI is a command-line interface for Qwen, a large language model.\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.26804123711340205 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "isinstance(servers, list)", + "got": "not empty_vscode_manager.get_server(\"test-server\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.37333333333333335 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "[\"single arg with spaces\"]", + "got": "[\"single arg with spaces\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "not any(features.values())", + "got": "not features[\"active_target\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.509090909090909 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "profile_servers[0].command", + "got": "profile_servers[0].command", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "os.path.exists(legacy_path)", + "got": "os.path.exists(legacy_path)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "os.path.exists(temp_db_path)", + "got": "os.path.exists(temp_db_path)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"https://api.example.com/mcp\"", + "got": "\"https://api.example.com/mcp\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "(backup_dir / \"README.md\").exists()", + "got": "(not migrator.detector.config_file.exists())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3291139240506329 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "goose_manager.is_client_installed()", + "got": "goose_manager.is_client_installed()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "{\"Authorization\": \"Bearer token123\"}", + "got": "{\"Authorization\": \"Bearer token123\"}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "mock_client_manager.add_server.called", + "got": "mock_client_manager.add_server.call_count == 1", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8433734939759037 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"https://github.com/QwenLM/qwen-code\"", + "got": "\"https://github.com/QwenLM/qwen-code\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "windsurf_manager.is_client_installed()", + "got": "windsurf_manager.is_client_installed()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "not goose_manager.is_client_installed()", + "got": "not goose_manager.is_client_installed()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "global_server.has_profile_tag(\"web-dev\")", + "got": "global_server.profile_tags", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7575757575757576 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "not windsurf_manager.is_client_installed()", + "got": "not windsurf_manager.is_client_installed()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "not os.path.exists(legacy_path + \".backup\")", + "got": "not os.path.exists(metadata_path)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7105263157894737 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "profile_manager.new_profile(\"test-profile\")", + "got": "not profile_manager.profile_exists(\"test_profile\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8172043010752689 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "[cmd.name for cmd in main.commands.values()]", + "got": "main.commands", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.45614035087719296 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "global_server.has_profile_tag(\"test-profile\")", + "got": "not global_server.has_profile_tag(\"test-profile\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9574468085106383 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "not profile_manager.new_profile(\"test-profile\")", + "got": "not profile_manager.new_profile(\"test-profile\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "global_config_manager.server_exists(\"test-server\")", + "got": "global_config_manager.server_exists(\"test-server\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "os.path.expanduser(\"~/.config/Code/User/mcp.json\")", + "got": "os.path.expanduser(\"~/.config/Code/User/mcp.json\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "not global_config_manager.server_exists(\"test-server\")", + "got": "not global_config_manager.get_server(\"test-server\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8952380952380953 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "os.path.expanduser(\"~/Library/Application Support/Code/User/mcp.json\")", + "got": "\"/Users/username/Library/Application Support/Code/User/settings.json\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7338129496402878 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"Invalid value for '--type'\" in result.output or \"invalid\" in result.output.lower()", + "got": "\"Invalid server type 'invalid'\" in result.output", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5343511450381679 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"Invalid environment variable format\" in result.output or \"Invalid key-value pair\" in result.output", + "got": "\"Invalid environment variable format\" in result.output", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7058823529411765 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"Interactive editing not available\" in result.output or \"This command requires a terminal\" in result.output or \"Create New Server Configuration\" in result.output", + "got": "\"Please enter the server name\" in result.output", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2980769230769231 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"Interactive editing not available\" in result.output or \"falling back to non-interactive mode\" in result.output or \"Use --name and --servers options\" in result.output", + "got": "\"Interactive mode\" in result.output", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3482587064676617 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "None)", + "got": "None)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"npx\")", + "got": "\"npx\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"run\")", + "got": "\"test-value\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.42105263157894735 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"bunx\")", + "got": "\"bunx\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"test\")", + "got": "\"test\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"qwen\")", + "got": "\"qwen\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"github\")", + "got": "\"github\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"server2\")", + "got": "server2)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"Windsurf\")", + "got": "\"test\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"api-server\")", + "got": "\"api-server\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"test-server\")", + "got": "\"test-server\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"nonexistent\")", + "got": "\"nonexistent\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"test-profile\")", + "got": "\"test-profile\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"mcpm_old-server\")", + "got": "\"old-server\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8387096774193549 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"mcpm_existing-server\")", + "got": "\"mcpm_existing-server\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"comprehensive-server\")", + "got": "\"comprehensive-server\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "[\"open\", \"/tmp/test_servers.json\"])", + "got": "[\"open\", \"/tmp/test_servers.json\"])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "ValueError, match=\"No supported servers to proxy\")", + "got": "ValueError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.36065573770491804 + }, + { + "repo": "patroni/patroni", + "expected": "ret)", + "got": "ret)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "pg_perm.orig_umask)", + "got": "pg_perm.ORIG_UMASK)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5263157894736842 + }, + { + "repo": "patroni/patroni", + "expected": "self.etcd.last_seen)", + "got": "self.etcd.last_seen())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9523809523809523 + }, + { + "repo": "patroni/patroni", + "expected": "self.client.machines)", + "got": "self.client._machines_cache)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "patroni/patroni", + "expected": "c._call(None, Mock()))", + "got": "c._call('foo'))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6486486486486487 + }, + { + "repo": "patroni/patroni", + "expected": "etcd3._cluster_version)", + "got": "etcd3.authenticate())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.45454545454545453 + }, + { + "repo": "patroni/patroni", + "expected": "self.ha.reinitialize())", + "got": "self.ha.reinitialize())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "self.handler.create_connection(()))", + "got": "self.handler.create_connection())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9705882352941176 + }, + { + "repo": "patroni/patroni", + "expected": "self.ha.evaluate_scheduled_restart())", + "got": "self.ha.evaluate_scheduled_restart())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "self.k.update_leader(cluster, '123'))", + "got": "self.k.update_leader(cluster, '123', failsafe={'foo': 'bar'}))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7474747474747475 + }, + { + "repo": "patroni/patroni", + "expected": "PostmasterProcess.from_pidfile('', ''))", + "got": "PostmasterProcess.from_pidfile('', ''))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "Unknown)", + "got": "BadRequestError, self.client._handle_server_response, response)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.08450704225352113 + }, + { + "repo": "patroni/patroni", + "expected": "OSError)", + "got": "OSError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "ApiNotOk)", + "got": "ApiNotOk)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "DCSError)", + "got": "DCSError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "Exception)", + "got": "OSError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "patroni/patroni", + "expected": "EtcdError)", + "got": "ValueError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "patroni/patroni", + "expected": "UserEmpty)", + "got": "AuthFailed, self.client.deleteprefix, 'foo')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.14814814814814814 + }, + { + "repo": "patroni/patroni", + "expected": "TypeError)", + "got": "AttributeError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.56 + }, + { + "repo": "patroni/patroni", + "expected": "RaftError)", + "got": "RaftError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "ValueError)", + "got": "ValueError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "SystemExit)", + "got": "SystemExit)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "Etcd3Error)", + "got": "SleepException)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "patroni/patroni", + "expected": "ConsulError)", + "got": "ValueError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6086956521739131 + }, + { + "repo": "patroni/patroni", + "expected": "Unavailable)", + "got": "Exception)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.18181818181818182 + }, + { + "repo": "patroni/patroni", + "expected": "task.result)", + "got": "self.b.post_bootstrap({}, task))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.22727272727272727 + }, + { + "repo": "patroni/patroni", + "expected": "device.open)", + "got": "device.open)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "ret.accepted)", + "got": "ret)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.47058823529411764 + }, + { + "repo": "patroni/patroni", + "expected": "self.p.nosync)", + "got": "self.p.nosync)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "self.p.stop())", + "got": "self.p.is_primary())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7058823529411765 + }, + { + "repo": "patroni/patroni", + "expected": "wd.activate())", + "got": "wd.is_running)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.35714285714285715 + }, + { + "repo": "patroni/patroni", + "expected": "self.p.start())", + "got": "self.p.start())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "\"test-3\" in ca)", + "got": "ca['database'] in ('test-3', 'test.3'))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2962962962962963 + }, + { + "repo": "patroni/patroni", + "expected": "self.p.nostream)", + "got": "self.p.nostream)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "self.p.reload())", + "got": "self.p.reload())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "raft.initialize())", + "got": "raft._sync_obj.set(raft.members_path + 'test', '{\"version\":\"2.0.0\"}'))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.22727272727272727 + }, + { + "repo": "patroni/patroni", + "expected": "self.r.can_rewind)", + "got": "self.r.can_rewind())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9473684210526315 + }, + { + "repo": "patroni/patroni", + "expected": "proc.signal_kill())", + "got": "proc.signal_kill())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "self.p.is_primary())", + "got": "self.p.is_primary())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "self.p.is_healthy())", + "got": "self.p.is_healthy())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "proc.is_single_user)", + "got": "proc.pid)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4827586206896552 + }, + { + "repo": "patroni/patroni", + "expected": "self.so.retry(test))", + "got": "self.so.retry(test))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "self.r.pg_rewind(r))", + "got": "self.r.pg_rewind(r))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "watchdog.activate())", + "got": "watchdog.activate())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "self.zk.watch(1, 1))", + "got": "self.zk._watcher(WatchedEvent('sync', '', '')))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5373134328358209 + }, + { + "repo": "patroni/patroni", + "expected": "task._event.is_set())", + "got": "task.done)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5161290322580645 + }, + { + "repo": "patroni/patroni", + "expected": "'False' in str(rows))", + "got": "rows)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.38461538461538464 + }, + { + "repo": "patroni/patroni", + "expected": "'Reason: ' in str(e))", + "got": "e.status == 409)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.21621621621621623 + }, + { + "repo": "patroni/patroni", + "expected": "self.p.noloadbalance)", + "got": "self.p.noloadbalance)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "self.p.promote(0, task))", + "got": "self.p.promote(task))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9333333333333333 + }, + { + "repo": "patroni/patroni", + "expected": "retry._cur_delay < 4000)", + "got": "retry._delay > 10)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7619047619047619 + }, + { + "repo": "patroni/patroni", + "expected": "self.etcd.watch(None, 1))", + "got": "self.etcd.set_ttl(0))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5652173913043478 + }, + { + "repo": "patroni/patroni", + "expected": "_main())", + "got": "_main())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "machines)", + "got": "machines)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "mpp.group)", + "got": "mpp.get_handler_impl(self.p))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.41025641025641024 + }, + { + "repo": "patroni/patroni", + "expected": "wd.disable())", + "got": "wd.disable())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "cluster.leader)", + "got": "cluster)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6956521739130435 + }, + { + "repo": "patroni/patroni", + "expected": "self.p.start())", + "got": "self.p.start())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "wd.keepalive())", + "got": "wd.keepalive())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "check_psycopg())", + "got": "check_psycopg())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "self.p.follow(m))", + "got": "self.p.follow(m))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "ce.call(callback))", + "got": "ce.execute(callback))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7692307692307693 + }, + { + "repo": "patroni/patroni", + "expected": "self.so.get('foo'))", + "got": "self.so._expire('foo'))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8095238095238095 + }, + { + "repo": "patroni/patroni", + "expected": "self.p.checkpoint())", + "got": "self.p.checkpoint({'user': 'postgres'}))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "patroni/patroni", + "expected": "self.p.replicatefrom)", + "got": "self.p.replicatefrom)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "r.resolve_async('', 0))", + "got": "r.run())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.45161290322580644 + }, + { + "repo": "patroni/patroni", + "expected": "self.ha.post_recover())", + "got": "self.ha.post_recover())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "5)", + "got": "2)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "patroni/patroni", + "expected": "0)", + "got": "0)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "1)", + "got": "1)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "6)", + "got": "2)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "patroni/patroni", + "expected": "2)", + "got": "0)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "patroni/patroni", + "expected": "3)", + "got": "False)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.25 + }, + { + "repo": "patroni/patroni", + "expected": "'')", + "got": "'')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "21)", + "got": "20)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "patroni/patroni", + "expected": "[])", + "got": "[])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "30)", + "got": "30)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "40)", + "got": "40)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "50)", + "got": "1)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "patroni/patroni", + "expected": "{})", + "got": "None)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.25 + }, + { + "repo": "patroni/patroni", + "expected": "202)", + "got": "0)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "patroni/patroni", + "expected": "'1')", + "got": "'123')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "patroni/patroni", + "expected": "[0])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "patroni/patroni", + "expected": "9.4)", + "got": "9)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "patroni/patroni", + "expected": "0.0)", + "got": "None)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "patroni/patroni", + "expected": "500)", + "got": "'500')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "patroni/patroni", + "expected": "'42')", + "got": "\"42\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6 + }, + { + "repo": "patroni/patroni", + "expected": "None)", + "got": "self.ha.cluster.leader)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.14285714285714285 + }, + { + "repo": "patroni/patroni", + "expected": "'{}')", + "got": "'{\"kind\": \"Service\"}')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.37037037037037035 + }, + { + "repo": "patroni/patroni", + "expected": "tags)", + "got": "{})", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.25 + }, + { + "repo": "patroni/patroni", + "expected": "'on')", + "got": "'on')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "True)", + "got": "False)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "patroni/patroni", + "expected": "'10')", + "got": "'10')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "'17')", + "got": "'17')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "0o123)", + "got": "'0123')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7692307692307693 + }, + { + "repo": "patroni/patroni", + "expected": "value)", + "got": "value)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "'/v3')", + "got": "'/v3')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "'foo')", + "got": "'foo')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "99999)", + "got": "'10.1')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.15384615384615385 + }, + { + "repo": "patroni/patroni", + "expected": "90400)", + "got": "9)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "patroni/patroni", + "expected": "False)", + "got": "True)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "patroni/patroni", + "expected": "float)", + "got": "float)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "'off')", + "got": "'auto')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "patroni/patroni", + "expected": "\"test\")", + "got": "\"test\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "'/tmp')", + "got": "'localhost')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.42105263157894735 + }, + { + "repo": "patroni/patroni", + "expected": "'10.1')", + "got": "'10.0')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "patroni/patroni", + "expected": "(1, 2))", + "got": "('postgresql0', 'b'))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.35714285714285715 + }, + { + "repo": "patroni/patroni", + "expected": "'fake')", + "got": "'running pg_rewind from follower')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.1951219512195122 + }, + { + "repo": "patroni/patroni", + "expected": "'Null')", + "got": "Exception)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.11764705882352941 + }, + { + "repo": "patroni/patroni", + "expected": "'INFO')", + "got": "'INFO')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "140000)", + "got": "\"unknown\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.11764705882352941 + }, + { + "repo": "patroni/patroni", + "expected": "140001)", + "got": "140000)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "patroni/patroni", + "expected": "dict())", + "got": "{})", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2 + }, + { + "repo": "patroni/patroni", + "expected": "'17.0')", + "got": "'17beta3')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5882352941176471 + }, + { + "repo": "patroni/patroni", + "expected": "'10.0')", + "got": "'10.0')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "60 - 5)", + "got": "60)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6 + }, + { + "repo": "patroni/patroni", + "expected": "'9.6.5')", + "got": "'9.6.5')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "(10, 1))", + "got": "(10, 10))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9411764705882353 + }, + { + "repo": "patroni/patroni", + "expected": "'citus')", + "got": "False)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "patroni/patroni", + "expected": "'Citus')", + "got": "'Citus')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "'ERROR')", + "got": "'ERROR')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "[\"bar\"])", + "got": "['foo'])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.375 + }, + { + "repo": "patroni/patroni", + "expected": "'value')", + "got": "'value')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "expected)", + "got": "expected)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "'COMMIT')", + "got": "'COMMIT')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "'leader')", + "got": "'alpha')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.47058823529411764 + }, + { + "repo": "patroni/patroni", + "expected": "'test_4')", + "got": "'bar')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "patroni/patroni", + "expected": "'blabla')", + "got": "'blabla')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "'logical')", + "got": "'standard')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.38095238095238093 + }, + { + "repo": "patroni/patroni", + "expected": "'WARNING')", + "got": "'ERROR')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "patroni/patroni", + "expected": "'patroni')", + "got": "'patroni')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "'/v3alpha')", + "got": "'/patroni/')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5217391304347826 + }, + { + "repo": "patroni/patroni", + "expected": "'Bearer a')", + "got": "'b')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "patroni/patroni", + "expected": "\"proc 123\")", + "got": "\"proc 123\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "{\"ttl\": 30})", + "got": "{\"postgresql\": {\"use_pg_rewind\": None}, \"ttl\": 30})", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.38095238095238093 + }, + { + "repo": "patroni/patroni", + "expected": "(True, True))", + "got": "'is_in_recovery=true')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.22857142857142856 + }, + { + "repo": "patroni/patroni", + "expected": "(None, None))", + "got": "(None, None))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "['--string'])", + "got": "('string',))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.72 + }, + { + "repo": "patroni/patroni", + "expected": "'promote msg')", + "got": "'cannot be a real primary in standby cluster')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3 + }, + { + "repo": "patroni/patroni", + "expected": "'https://a:1')", + "got": "'a')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "patroni/patroni", + "expected": "{'foo': 'bar'})", + "got": "{})", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "patroni/patroni", + "expected": "'Bearer token')", + "got": "'Bearer token')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "(False, False))", + "got": "(True, True))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "patroni/patroni", + "expected": "1)", + "got": "1)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "0)", + "got": "False)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.25 + }, + { + "repo": "patroni/patroni", + "expected": "503)", + "got": "400)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "patroni/patroni", + "expected": "200)", + "got": "503)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "patroni/patroni", + "expected": "123)", + "got": "'/tmp/test.conf')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.09523809523809523 + }, + { + "repo": "patroni/patroni", + "expected": "body)", + "got": "body)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "None)", + "got": "False)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "patroni/patroni", + "expected": "True)", + "got": "reset)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5454545454545454 + }, + { + "repo": "patroni/patroni", + "expected": "args)", + "got": "args)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "i[1])", + "got": "i[1])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "dict())", + "got": "[('ls', 'logical', 1, 499, 'b', 'a', 5, 100, 500)])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.13793103448275862 + }, + { + "repo": "patroni/patroni", + "expected": "\"utf-8\")", + "got": "\"utf-8\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "'PAGER')", + "got": "'PAGER')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "response)", + "got": "response)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "'foobar')", + "got": "'b')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "patroni/patroni", + "expected": "'blabla')", + "got": "'blabla')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "ct.cancel())", + "got": "ct.is_running())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "patroni/patroni", + "expected": "task.result)", + "got": "self.b.post_bootstrap({}, task))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.22727272727272727 + }, + { + "repo": "patroni/patroni", + "expected": "device.open)", + "got": "watchdog.is_running)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.25 + }, + { + "repo": "patroni/patroni", + "expected": "ret.accepted)", + "got": "ret.accepted)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "self.p.nosync)", + "got": "self.p.nosync)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "self.p.stop())", + "got": "self.p.stop(on_safepoint=mock_callback))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5185185185185185 + }, + { + "repo": "patroni/patroni", + "expected": "self.p.start())", + "got": "self.p.start())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "ret.in_recovery)", + "got": "ret)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "patroni/patroni", + "expected": "self.p.nostream)", + "got": "self.p.nostream)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "self.p.restart())", + "got": "self.p.restart())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "self.r.can_rewind)", + "got": "self.r.can_rewind)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "proc.signal_kill())", + "got": "proc.signal_kill())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "self.ha.is_leader())", + "got": "self.ha.is_leader())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "self.p.is_primary())", + "got": "self.p.is_primary())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "self.p.is_healthy())", + "got": "self.p.is_healthy())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "self.so.retry(test))", + "got": "self.so.retry(test))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "self.r.pg_rewind(r))", + "got": "self.r.pg_rewind(r))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "watchdog.is_running)", + "got": "watchdog.is_running)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "conn.aws_available())", + "got": "conn.on_role_change(\"primary\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4230769230769231 + }, + { + "repo": "patroni/patroni", + "expected": "self.b.bootstrap({}))", + "got": "self.b.bootstrap({'foo': 'bar'}))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7777777777777778 + }, + { + "repo": "patroni/patroni", + "expected": "self.c.take_leader())", + "got": "self.c.take_leader())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "self.impl.is_healthy)", + "got": "self.impl.is_healthy)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "self.zk.initialize())", + "got": "self.zk.initialize())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "self.so.delete('foo'))", + "got": "self.so.delete('foo'))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "c._call(None, Mock()))", + "got": "c._call(None, Mock()))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "mock_deregister.called)", + "got": "mock_deregister.called)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "self.etcd.initialize())", + "got": "self.etcd.initialize())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "Etcd)", + "got": "AbstractDCS)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.35294117647058826 + }, + { + "repo": "patroni/patroni", + "expected": "Null)", + "got": "Null)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "Bool)", + "got": "ValidatorBool)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5263157894736842 + }, + { + "repo": "patroni/patroni", + "expected": "Raft)", + "got": "Raft)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "Etcd3)", + "got": "Etcd3)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "cmd)", + "got": "cmd)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "errors)", + "got": "errors)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "target_log)", + "got": "target_log)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "result.output)", + "got": "result.output)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "1", + "got": "1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "0", + "got": "1", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "patroni/patroni", + "expected": "cmd)", + "got": "cmd)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "fr'^\\[.*\\] INFO {test_message}$')", + "got": "'^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}.\\d{6}Z$')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2 + }, + { + "repo": "patroni/patroni", + "expected": "r'\\[\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\]')", + "got": "fr'^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9176470588235294 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "0", + "got": "False", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "1", + "got": "1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "3", + "got": "3", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "v", + "got": "v", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "4", + "got": "5", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "5", + "got": "5", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "2", + "got": "2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "42", + "got": "42", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "''", + "got": "''", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "{}", + "got": "'{}'", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "10", + "got": "10", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "20", + "got": "20", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "now", + "got": "now", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "'1'", + "got": "'1'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "'\"'", + "got": "'\"'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "'3'", + "got": "{'N': '3.141'}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.35294117647058826 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "inp", + "got": "inp", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "'S'", + "got": "STRING", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "100", + "got": "100", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "True", + "got": "True", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "None", + "got": "None", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "'{}'", + "got": "'{}'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "item", + "got": "item", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "\"", + "got": "\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "data", + "got": "data", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "value", + "got": "value", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "3.141", + "got": "3.141", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "'foo'", + "got": "'foo'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "False", + "got": "False", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "'\\\\t'", + "got": "'\\\\t'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "'bar'", + "got": "'bar'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "list2", + "got": "list2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "991.0", + "got": "991", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.75 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "992.0", + "got": "992.0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "{1, 2}", + "got": "{1, 2}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "STRING", + "got": "'utc_datetime'", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "BINARY", + "got": "BINARY", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "b'foo'", + "got": "b'foo'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "NUMBER", + "got": "NUMBER", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "values", + "got": "values", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "params", + "got": "params", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "'HASH'", + "got": "'HASH'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "'hiya'", + "got": "'hiya'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "action", + "got": "action", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "'id-0'", + "got": "items[10]['user_id']", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "'id-1'", + "got": "'id-1'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "1100.0", + "got": "100.0", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "m.data", + "got": "m.data", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "b'test'", + "got": "b'test'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "u\"test\"", + "got": "u\"test\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "self.dt", + "got": "'2047-01-06T08:21:30.000000+00:00'", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.04878048780487805 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "'3.141'", + "got": "'3.141'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "BOOLEAN", + "got": "BOOLEAN", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "\"Hello\"", + "got": "\"Hello\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "m.value", + "got": "isinstance(m, DynamicMapDiscriminatorTestModel)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.07407407407407407 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "'hello'", + "got": "'hello'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "\"", + "got": "\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "\"", + "got": "\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "expected", + "got": "expected", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "\"", + "got": "\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "\"my_ttl\"", + "got": "'ttl'", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "attribute", + "got": "attribute", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "'dyn_foo'", + "got": "'dyn_foo'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "'foo.bar'", + "got": "'foo.bar'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "action_eq", + "got": "action_eq", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "\"", + "got": "\"foo = 'bar'\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.14285714285714285 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "\"", + "got": "\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "\"", + "got": "\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "\"", + "got": "\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "BINARY_SET", + "got": "SET", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "NUMBER_SET", + "got": "NUMBER_SET", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "STRING_SET", + "got": "STRING_SET", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "\"", + "got": "\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "\"", + "got": "\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "\"", + "got": "\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "\"", + "got": "\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "'nestedbar'", + "got": "'nestedbar'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "sorted(inp)", + "got": "sorted(inp)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "conn.client", + "got": "conn.host", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "'new_value'", + "got": "'new_value'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "\"'foo.bar'\"", + "got": "'foo.bar'", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "\"REMOVE", + "got": "\"SET", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "User.get(1)", + "got": "User.get(1)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "User.get(2)", + "got": "User.get(2)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "'1559692860'", + "got": "'1559692800'", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9166666666666666 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "grocery_list", + "got": "grocery_list", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "{'Limit': 3}", + "got": "{'Limit': 3}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "\"", + "got": "\"attribute_not_exists(cls) OR attribute_exists(cls)\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.03773584905660377 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "'string val'", + "got": "'string val'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "'foo.bar[0]'", + "got": "'foo'", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5882352941176471 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "\"", + "got": "\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "condition_eq", + "got": "condition_eq", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "\"", + "got": "\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "GET_ITEM_DATA", + "got": "GET_ITEM_DATA[ITEM_KEY]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7222222222222222 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "TestAttribute", + "got": "TestAttribute", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "action_not_eq", + "got": "action_not_eq", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "{'foo': '", + "got": "{'bar': '", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "{'foo', 'bar'}", + "got": "{'foo', 'bar'}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "serialized_raw", + "got": "serialized_attr_from_map", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.631578947368421 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "'dyn_map_attr'", + "got": "'dyn_in_map_a'", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7857142857142857 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "\"Test message\"", + "got": "\"Test message\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "\"'foo.bar'[0]\"", + "got": "\"'foo.bar[0]'.\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.896551724137931 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "\"", + "got": "\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "\"", + "got": "\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "\"", + "got": "\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "\"", + "got": "\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "legacy_encoding", + "got": "legacy_encoding", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "item.validate()", + "got": "item.default_number_attr", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.41025641025641024 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "\"'foo.bar'.baz\"", + "got": "\"'foo.bar.baz'\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9333333333333333 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "\"(NOT", + "got": "\"NOT (", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7272727272727273 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "\"", + "got": "\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "\"", + "got": "\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "pre_recorded[0]", + "got": "pre_recorded[0]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "sorted(expected)", + "got": "sorted(expected)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "{b'foo', b'bar'}", + "got": "set(serialized)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.06451612903225806 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "person_attribute", + "got": "person_attribute", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "condition_not_eq", + "got": "condition_not_eq", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "\"", + "got": "\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "KeyError)", + "got": "AttributeError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5833333333333334 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "GetError)", + "got": "ItemError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7368421052631579 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "PutError)", + "got": "PutError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "ScanError)", + "got": "BotoCoreError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "TypeError)", + "got": "ValueError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "ValueError)", + "got": "AttributeError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "TableError)", + "got": "TableError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "RuntimeError)", + "got": "AttributeError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6428571428571429 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "DoesNotExist)", + "got": "TransactWriteError) as exc_info:", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "AttributeError)", + "got": "AttributeError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "TableDoesNotExist)", + "got": "ValueError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3448275862068966 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "User.DoesNotExist)", + "got": "DoesNotExist)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8387096774193549 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "InvalidStateError)", + "got": "InvalidStateError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "VerboseClientError)", + "got": "ClientError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7741935483870968 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "TransactWriteError)", + "got": "ResourceNotInRegionException)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "NotImplementedError)", + "got": "NotImplementedError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "Exception, match='bleep-bloop')", + "got": "Exception)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4878048780487805 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "ValueError, match='timezone-aware')", + "got": "ValueError, match='must be timezone-aware')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8974358974358975 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "ValueError, match='legacy_encoding')", + "got": "TypeError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.30434782608695654 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "str)", + "got": "str)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "Any)", + "got": "Any)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "int)", + "got": "int)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "float)", + "got": "NumberAttribute)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.18181818181818182 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "MyModel)", + "got": "MyModel)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "params[key])", + "got": "params[key])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "list[MyMap])", + "got": "List[MyMap])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9166666666666666 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "SecondModel)", + "got": "SecondModel)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "TransactGet)", + "got": "TransactGet)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "TransactWrite)", + "got": "TransactWrite)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "dict[str, Any])", + "got": "int)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3157894736842105 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "NumberAttribute)", + "got": "NumberAttribute)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "UnicodeAttribute)", + "got": "str)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "PutError)", + "got": "PutError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "TypeError)", + "got": "IndexError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "Exception)", + "got": "AttributeError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.32 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "ValueError)", + "got": "TypeError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "3)", + "got": "8)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "2)", + "got": "2)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "5)", + "got": "25)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "1)", + "got": "2)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "4)", + "got": "4)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "v)", + "got": "v)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "25)", + "got": "5)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "42)", + "got": "'callable_field')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.1 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "12)", + "got": "8)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "10)", + "got": "10)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "30)", + "got": "30)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "50)", + "got": "50)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "ALL)", + "got": "AllProjection.projection_type)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.11764705882352941 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "123)", + "got": "123)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "1042)", + "got": "142)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "6161)", + "got": "6161)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "\"bob\")", + "got": "\"bob\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "88030)", + "got": "88030)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "params)", + "got": "params)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "actual)", + "got": "actual)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "INCLUDE)", + "got": "INCLUDE)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "'Dodge')", + "got": "'Volkswagen')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "'Justin')", + "got": "'Justin')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "'Beetle')", + "got": "'Jetta')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5882352941176471 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "'INCLUDE')", + "got": "'SEARCH')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.42105263157894735 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "'Garrett')", + "got": "'Garrett')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "actual[k])", + "got": "actual[k])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "'Volkswagen')", + "got": "'Ford')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "GET_ITEM_DATA)", + "got": "GET_ITEM_DATA[ITEM_KEY])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7368421052631579 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "map_serialized)", + "got": "map_serialized)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "item_keys_copy)", + "got": "item_keys_copy)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "'access_key_id')", + "got": "'access_key_id')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "'session_token')", + "got": "'session_token')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "attribute['foo'])", + "got": "123)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.09523809523809523 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "actual['listy'][2])", + "got": "actual.get('listy')[2])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8095238095238095 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "item.is_human)", + "got": "item.is_human)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "complex_thing.person)", + "got": "complex_thing.person)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "office_employee.person)", + "got": "office_employee.exists())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7916666666666666 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "queried)", + "got": "queried)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "scanned_items)", + "got": "scanned_items)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "[ { 'KeyType': 'RANGE', 'AttributeName': 'user_id' }, { 'KeyType': 'HASH', 'AttributeName': 'user_name' }, ])", + "got": "[{'AttributeName': 'user_name', 'KeyType': 'HASH'}])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.422360248447205 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "item)", + "got": "item)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "item.is_human)", + "got": "item.is_admin)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7857142857142857 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "item.picture)", + "got": "item.picture)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "batch.commit())", + "got": "batch.commit())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pynamodb/PynamoDB", + "expected": "item.person.age)", + "got": "item.person.lname)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8823529411764706 + }, + { + "repo": "pypa/pipx", + "expected": "2", + "got": "1", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "pypa/pipx", + "expected": "1", + "got": "0", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "pypa/pipx", + "expected": "0", + "got": "0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "\"\"", + "got": "\"\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "{}", + "got": "[\"pylint\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "pypa/pipx", + "expected": "\"2\"", + "got": "\"2\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "None", + "got": "None", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "\"py\"", + "got": "\"py\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "True", + "got": "False", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "pypa/pipx", + "expected": "False", + "got": "False", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "str(e)", + "got": "e.value.args[0]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.09523809523809523 + }, + { + "repo": "pypa/pipx", + "expected": "exposed", + "got": "exposed", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "expected", + "got": "expected", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "\"python\"", + "got": "\"python\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "test_str", + "got": "test_str", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "[\"black\"]", + "got": "[\"dill\", \"isort\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.38461538461538464 + }, + { + "repo": "pypa/pipx", + "expected": "{\"black\"}", + "got": "{\"pycowsay\"}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.47619047619047616 + }, + { + "repo": "pypa/pipx", + "expected": "{pkg_name}", + "got": "{pkg_name}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "SystemExit", + "got": "assert_exit", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.47619047619047616 + }, + { + "repo": "pypa/pipx", + "expected": "caplog.text", + "got": "caplog.text", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "python_path", + "got": "python_path", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "assert_exit", + "got": "assert_exit", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "captured.err", + "got": "captured.out", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.75 + }, + { + "repo": "pypa/pipx", + "expected": "captured.out", + "got": "captured.out", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "str(env_dir)", + "got": "env_dir", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7368421052631579 + }, + { + "repo": "pypa/pipx", + "expected": "log_contents", + "got": "log_contents", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "home / \"test\"", + "got": "home / \"test\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "\"WindowsApps\"", + "got": "\"3.8\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "pypa/pipx", + "expected": "needs_upgrade", + "got": "needs_upgrade", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "sys.executable", + "got": "\"python.exe\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.38461538461538464 + }, + { + "repo": "pypa/pipx", + "expected": "expected_string", + "got": "expected_string", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "package_version", + "got": "package_version", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "expected_output", + "got": "expected_output", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "out.read_text()", + "got": "caplog.text", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.38461538461538464 + }, + { + "repo": "pypa/pipx", + "expected": "package_name_out", + "got": "package_name_out", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "package_spec_out", + "got": "package_spec_out", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "tag_path.exists()", + "got": "tag_path.exists()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "app_path.exists()", + "got": "not app_path.exists()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8947368421052632 + }, + { + "repo": "pypa/pipx", + "expected": "pip_log_file_match", + "got": "pip_log_file_match", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "captured.out.strip()", + "got": "captured.out", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.75 + }, + { + "repo": "pypa/pipx", + "expected": "ref_metadata_replaced", + "got": "ref_metadata_replaced", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "run_pipx_cli([\"list\"])", + "got": "not run_pipx_cli([\"list\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9166666666666666 + }, + { + "repo": "pypa/pipx", + "expected": "package_or_url_correct", + "got": "package_or_url_correct", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "expected_app_with_args", + "got": "expected_app_with_args", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "man_page_path.exists()", + "got": "man_page_path.exists()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "capsys.readouterr().err", + "got": "capsys.readouterr().err", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "capsys.readouterr().out", + "got": "capsys.read()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7222222222222222 + }, + { + "repo": "pypa/pipx", + "expected": "isort_app_path.exists()", + "got": "not isort_app_path.exists()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.92 + }, + { + "repo": "pypa/pipx", + "expected": "executable_path.exists()", + "got": "not file_or_symlink(executable_path)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5333333333333333 + }, + { + "repo": "pypa/pipx", + "expected": "pylint_app_path.exists()", + "got": "pylint_app_path.exists()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "venv_python_path.is_file()", + "got": "not os.path.exists(venv_python_path)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5483870967741935 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"list\"])", + "got": "not os.path.exists(constants.PIPX_HOME)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "pypa/pipx", + "expected": "{pkg for pkg, _ in packages}", + "got": "{p for _, p in packages}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8461538461538461 + }, + { + "repo": "pypa/pipx", + "expected": "unwrap_log_text(caplog.text)", + "got": "caplog.text", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5641025641025641 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"--help\"])", + "got": "not capsys.readouterr().out", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2545454545454545 + }, + { + "repo": "pypa/pipx", + "expected": "run_pipx_cli([\"pin\", \"abc\"])", + "got": "not run_pipx_cli([\"pin\", \"not_installed_package\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6923076923076923 + }, + { + "repo": "pypa/pipx", + "expected": "not file_or_symlink(app_path)", + "got": "not app_path.exists()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.52 + }, + { + "repo": "pypa/pipx", + "expected": "run_pipx_cli([\"upgrade-all\"])", + "got": "not run_pipx_cli([\"upgrade-all\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9354838709677419 + }, + { + "repo": "pypa/pipx", + "expected": "run_pipx_cli([\"unpin\", \"abc\"])", + "got": "not run_pipx_cli([\"unpin\", \"nox\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.84375 + }, + { + "repo": "pypa/pipx", + "expected": "expected_string[:chars_to_test]", + "got": "expected_string", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6521739130434783 + }, + { + "repo": "pypa/pipx", + "expected": "captured.err.replace(\"\\n\", \" \")", + "got": "captured.err", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5581395348837209 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"pin\", \"nox\"])", + "got": "\"Warning: No such package\" in caplog.text", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.1643835616438356 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"completions\"])", + "got": "not capsys.isredirected()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.27586206896551724 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"environment\"])", + "got": "not capsys.is_terminal()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2807017543859649 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"upgrade-all\"])", + "got": "not run_pipx_cli([\"install\", \"pycowsay\", \"--upgrade\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "pypa/pipx", + "expected": "python_path.endswith(\"python.exe\")", + "got": "not f\"{major}.{minor}.exe\" in python_path", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.29333333333333333 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"pin\", \"black\"])", + "got": "not run_pipx_cli([\"list\", \"--pinned\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.75 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"list\", \"-qqq\"])", + "got": "not main.limit_verbosity(\"foo\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.36923076923076925 + }, + { + "repo": "pypa/pipx", + "expected": "getattr(pipx_metadata2, attribute)", + "got": "getattr(pipx_metadata2, attribute)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "not file_or_symlink(man_page_path)", + "got": "not man_page_path.exists()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"unpin\", \"nox\"])", + "got": "not run_pipx_cli([\"unpin\", \"nox\"])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"list\", \"-vvvv\"])", + "got": "not run_pipx_cli([\"list\", \"-vvv\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9855072463768116 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"reinstall-all\"])", + "got": "not run_pipx_cli([\"reinstall-all\"])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"uninstall-all\"])", + "got": "not run_pipx_cli([\"uninstall\", \"--all\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9333333333333333 + }, + { + "repo": "pypa/pipx", + "expected": "Path(pip_log_file_match[1]).exists()", + "got": "(Path(pip_log_file_match.group(1)).exists())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.85 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"list\", \"--json\"])", + "got": "not run_pipx_cli([\"install\", \"pycowsay\", \"--all\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7209302325581395 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"pin\", \"black@1\"])", + "got": "not run_pipx_cli([\"pin\", PKG[\"black\"][\"spec\"], \"--suffix\", \"@1\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7128712871287128 + }, + { + "repo": "pypa/pipx", + "expected": "not file_or_symlink(executable_path)", + "got": "not file_or_symlink(executable_path)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"unpin\", \"black\"])", + "got": "not run_pipx_cli([\"unpin\", \"nox\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8857142857142857 + }, + { + "repo": "pypa/pipx", + "expected": "run_pipx_cli([\"upgrade\", f\"{name}\"])", + "got": "not run_pipx_cli([\"upgrade\", name])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8732394366197183 + }, + { + "repo": "pypa/pipx", + "expected": "(paths.ctx.bin_dir / name_a).exists()", + "got": "not run_pipx_cli([\"install\", PKG[name][\"spec\"], \"--no-suffix\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.24 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"list\", \"--short\"])", + "got": "not run_pipx_cli([\"list\", \"--short\"])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "shared_libs.shared_libs.needs_upgrade", + "got": "not run_pipx_cli([\"list\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"pin\", \"pycowsay\"])", + "got": "not run_pipx_cli([\"pin\", \"pycowsay\"])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "run_pipx_cli([\"upgrade\", \"pycowsay\"])", + "got": "\"installed package pycowsay\" in captured.out", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.32098765432098764 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"list\", \"--global\"])", + "got": "not run_pipx_cli([\"list\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8125 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"install\", \"black\"])", + "got": "not run_pipx_cli([\"install\", \"pycowsay\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8607594936708861 + }, + { + "repo": "pypa/pipx", + "expected": "run_pipx_cli([\"reinstall\", str(path)])", + "got": "not run_pipx_cli([\"install\", \"pylint\", \"--path\", str(path)])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7346938775510204 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"upgrade\", \"pylint\"])", + "got": "not run_pipx_cli([\"upgrade\", \"pylint\"])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"install\", pkg_spec])", + "got": "not run_pipx_cli([\"install\", pkg_spec])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"upgrade\", \"black@1\"])", + "got": "\"Upgrading pinned package black\" in caplog.text", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2988505747126437 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"upgrade\", f\"{name}\"])", + "got": "run_pipx_cli([\"upgrade\", f\"{pkg_spec}\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.825 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"install\", \"pycowsay\"])", + "got": "not caplog.has_warning()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.27692307692307694 + }, + { + "repo": "pypa/pipx", + "expected": "find_python_interpreter(interpreter_path)", + "got": "_find_python_interpreter_by_path(interpreter_path)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9010989010989011 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"interpreter\", \"list\"])", + "got": "not standalone_python.exists()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.36619718309859156 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"uninstall\", \"pycowsay\"])", + "got": "not file_or_symlink(paths.venv_interpreter)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.32558139534883723 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"interpreter\", \"upgrade\"])", + "got": "not run_pipx_cli([\"interpreter\", \"upgrade\"])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"install\", f\"{pkg_spec}\"])", + "got": "not run_pipx_cli([\"install\", name])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.810126582278481 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"environment\", \"--global\"])", + "got": "not run_pipx_cli([\"environment\", \"--global\"])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"install\", \"pylint==3.0.4\"])", + "got": "not run_pipx_cli([\"install\", \"pycowsay\", \"--specifier\", \">=0.1\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7027027027027027 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"install\", \"nox\", \"--force\"])", + "got": "not run_pipx_cli([\"install\", \"https://github.com/wntrblm/nox/archive/2022.1.7.zip\", \"--force\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6619718309859155 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"runpip\", \"pycowsay\", \"list\"])", + "got": "not run_pipx_cli([\"install\", \"pycowsay\", \"--no-pipx\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7843137254901961 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"upgrade\", f\"{name}{suffix}\"])", + "got": "not run_pipx_cli([\"install\", name])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6987951807228916 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"uninstall\", f\"{name}{suffix}\"])", + "got": "not run_pipx_cli([\"uninstall\", f\"{name}{suffix}\"])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "(paths.ctx.man_dir / \"man6\" / \"pycowsay.6\").exists()", + "got": "(Path(\"man6\") / \"pycowsay.6\").exists()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7777777777777778 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"install\", PKG[\"pylint\"][\"spec\"]])", + "got": "not run_pipx_cli([\"install\", \"pycowsay\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7526881720430108 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"install\", \"--global\", \"pycowsay\"])", + "got": "not run_pipx_cli([\"inject\", \"pycowsay\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7956989247311828 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"upgrade\", \"pycowsay\", \"--install\"])", + "got": "not run_pipx_cli([\"install\", PKG[\"pylint\"][\"spec\"]])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5471698113207547 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"uninstall\", \"--global\", \"pycowsay\"])", + "got": "not run_pipx_cli([\"uninstall\", \"--global\", \"pycowsay\"])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"install\", \"pycowsay\", *install_args])", + "got": "not run_pipx_cli([\"install\", \"pycowsay\"] + install_args)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9464285714285714 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"install\", name, f\"--suffix={suffix}\"])", + "got": "not run_pipx_cli([\"install\", f\"{name}{suffix}\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8380952380952381 + }, + { + "repo": "pypa/pipx", + "expected": "run_pipx_cli([\"inject\", \"pycowsay\", PKG[\"black\"][\"spec\"]])", + "got": "not run_pipx_cli([\"inject\", \"pycowsay\", PKG[\"black\"][\"spec\"]])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9666666666666667 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"runpip\", \"--global\", \"pycowsay\", \"list\"])", + "got": "not run_pipx_cli([\"runpip\", \"--global\", \"pycowsay\", \"list\"])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"upgrade\", \"--editable\", \"empty_project\"])", + "got": "not run_pipx_cli([\"upgrade\", \"--editable\", empty_project_path_as_string])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8721804511278195 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"install\", \"--preinstall\", \"black\", \"nox\"])", + "got": "not run_pipx_cli([\"install\", \"pycowsay\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7058823529411765 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"install\", \"repeatme-0.1-py3-none-any.whl\"])", + "got": "(root / \"local_extras\").exists()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.1702127659574468 + }, + { + "repo": "pypa/pipx", + "expected": "(pipx_venvs_dir / \"pycowsay\" / \"pipx_metadata.json\").is_file()", + "got": "Path(pipx_venvs_dir / \"pycowsay\").exists()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6538461538461539 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"install\", \"pycowsay\", f\"--suffix={suffix}\"])", + "got": "not run_pipx_cli([\"install\", \"pycowsay\" + suffix])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8495575221238938 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"reinstall-all\", \"--python\", sys.executable])", + "got": "not run_pipx_cli([\"install\", \"pycowsay\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6923076923076923 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"uninject\", \"pycowsay\", \"black\", \"--verbose\"])", + "got": "not run_pipx_cli([\"uninject\", \"pycowsay\", \"black\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8869565217391304 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"install\", \"--preinstall\", \"black==22.8.0\", \"nox\"])", + "got": "not run_pipx_cli([\"install\", \"--preinstall\", \"colorama==0.3.0\", \"nox\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8857142857142857 + }, + { + "repo": "pypa/pipx", + "expected": "f\"{major}.{minor}\" in python_path or f\"{major}{minor}\" in python_path", + "got": "python_path.endswith(\"python.exe\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.36893203883495146 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"reinstall\", \"--python\", sys.executable, \"pylint\"])", + "got": "not run_pipx_cli([\"reinstall\", \"--python\", sys.executable, \"pylint\"])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "run_pipx_cli([\"reinstall\", \"--python\", sys.executable, \"nonexistent\"])", + "got": "not run_pipx_cli([\"reinstall\", \"nonexistent\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7241379310344828 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"install\", PKG[\"jupyter\"][\"spec\"], \"--include-deps\"])", + "got": "os.path.exists(paths.pipx_bin / \"jupyter\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.35398230088495575 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"reinstall\", \"--python\", sys.executable, \"pycowsay\"])", + "got": "not run_pipx_cli([\"install\", \"pycowsay\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7321428571428571 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"install\", \"--editable\", empty_project_path_as_string])", + "got": "not run_pipx_cli([\"install\", \"empty_project\", \"--editable\", empty_project_path_as_string])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8957055214723927 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"inject\", \"--global\", \"pycowsay\", PKG[\"black\"][\"spec\"]])", + "got": "not run_pipx_cli([\"inject\", \"pycowsay\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7017543859649122 + }, + { + "repo": "pypa/pipx", + "expected": "run_pipx_cli([\"inject\", \"pycowsay\", PKG[\"black\"][\"spec\"], \"--include-deps\"])", + "got": "not run_pipx_cli([\"inject\", f\"pycowsay{suffix}\", PKG[\"black\"][\"spec\"]])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7891156462585034 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"install\", \"nox[tox_to_nox]==2023.4.22\", \"--include-deps\"])", + "got": "run_pipx_cli([\"install\", PKG[\"pygdbmi\"][\"spec\"], \"--extra\", \"test\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5241379310344828 + }, + { + "repo": "pypa/pipx", + "expected": "run_pipx_cli([\"install\", \"--python\", python_version, \"--verbose\", \"pycowsay\"])", + "got": "run_pipx_cli([\"install\", \"--python\", python_version, \"--verbose\", \"pycowsay\"])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"reinstall\", \"--python\", sys.executable, f\"pycowsay{suffix}\"])", + "got": "not run_pipx_cli([\"reinstall\", \"--python\", sys.executable, \"pycowsay\", f\"--suffix={suffix}\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9132947976878613 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"install\", \"--python\", python_version, \"--verbose\", \"pycowsay\"])", + "got": "not run_pipx_cli([\"install\", \"--python\", python_version, \"pycowsay\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9139072847682119 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"reinstall\", \"--global\", \"--python\", sys.executable, \"pycowsay\"])", + "got": "not run_pipx_cli([\"reinstall\", \"--global\", \"--python\", sys.executable, \"pycowsay\"])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "python_path == f\"python{major}.{minor}\" or f\"Python\\\\{major}.{minor}\" in python_path", + "got": "python_path.endswith(\"python.exe\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3389830508474576 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"install\", \"--editable\", empty_project_path_as_string, \"--force\"])", + "got": "not run_pipx_cli([\"install\", \"--editable\", empty_project_path_as_string])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9299363057324841 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"inject\", f\"pycowsay{suffix}\", PKG[\"black\"][\"spec\"], \"--include-deps\"])", + "got": "not run_pipx_cli([\"inject\", \"pycowsay\", PKG[\"nox\"][\"spec\"]])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7651006711409396 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"install\", \"--preinstall\", \"chardet\", \"--preinstall\", \"colorama\", \"nox\"])", + "got": "not run_pipx_cli([\"install\", \"--preinstall\", \"black\", \"nox\", \"pytest\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7160493827160493 + }, + { + "repo": "pypa/pipx", + "expected": "run_pipx_cli([\"install\", \"--python\", python_version, \"--fetch-missing-python\", \"pycowsay\"])", + "got": "not run_pipx_cli([\"install\", \"pycowsay\", \"--python\", python_version])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6625 + }, + { + "repo": "pypa/pipx", + "expected": "run_pipx_cli([\"upgrade\", \"--verbose\", str((root / \"testdata\" / \"empty_project\").resolve())])", + "got": "not run_pipx_cli([\"install\", PKG[\"pylint\"][\"spec\"]])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3888888888888889 + }, + { + "repo": "pypa/pipx", + "expected": "run_pipx_cli([\"install\", f\"--pip-args='{constraint_flag}{constraint_file_name}'\", \"pycowsay\"])", + "got": "not run_pipx_cli([\"install\", f\"--pip-args='{constraint_flag}wrong_file.txt'\", \"pycowsay\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8586956521739131 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"install\", str(root / f\"{TEST_DATA_PATH}/local_extras[cow]\"), \"--include-deps\"])", + "got": "not run_pipx_cli([\"install\", \"nox[tox_to_nox]==2023.4.22\", \"--include-deps\", \"--local\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6021505376344086 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli([\"install\", f\"--pip-args='{constraint_flag}{constraint_file_name}'\", package_name])", + "got": "not run_pipx_cli([\"install\", \"pycowsay\", constraint_flag, str(tmp_path)])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6473988439306358 + }, + { + "repo": "pypa/pipx", + "expected": "run_pipx_cli( [ \"install\", \"pycowsay\", \"--verbose\", \"--pip-args='--no-index --find-links=D:\\\\TEST\\\\DIR'\", ] )", + "got": "not run_pipx_cli([\"install\", \"pycowsay\", \"--pip-args='--no-index'\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7231638418079096 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli( [ \"install\", \"--fetch-missing-python\", \"--python\", target_python, PKG[\"pycowsay\"][\"spec\"], ] )", + "got": "not run_pipx_cli([\"install\", PKG[\"pycowsay\"][\"spec\"]])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6506024096385542 + }, + { + "repo": "pypa/pipx", + "expected": "install_package_both_paths( monkeypatch, capsys, caplog, module_globals, pipx_temp_env, package_name, deps=True, )", + "got": "install_package_both_paths(", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3829787234042553 + }, + { + "repo": "pypa/pipx", + "expected": "install_package_both_paths( monkeypatch, capsys, caplog, module_globals, pipx_temp_env, package_name, deps=False, )", + "got": "install_package_both_paths( monkeypatch, capsys, caplog, module_globals, pipx_temp_env, package_name)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9351851851851852 + }, + { + "repo": "pypa/pipx", + "expected": "not run_pipx_cli( [ \"install\", \"--fetch-missing-python\", \"--python\", TARGET_PYTHON_VERSION, PKG[\"pycowsay\"][\"spec\"], ] )", + "got": "not run_pipx_cli([\"interpreter\", \"upgrade\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3780487804878049 + }, + { + "repo": "pypa/pipx", + "expected": "run_pipx_cli( [ \"install\",", + "got": "not run_pipx_cli([\"install\", \"--verbose\", \"--pip-args=--no-index\", \"pycowsay\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.45714285714285713 + }, + { + "repo": "pypa/pipx", + "expected": "subprocess.run( [ sys.executable, \"-m\", \"pipx\", \"run\", \"ipython\", \"-c\", \"import os; print(os.environ.get('PYTHONPATH'))\", ], env=env, capture_output=True, text=True, check=True, ).stdout", + "got": "pipx.main.run_ensure_null_pythonpath(env)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.1762114537444934 + }, + { + "repo": "pypa/pipx", + "expected": "PipxError)", + "got": "InterpreterResolutionError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3783783783783784 + }, + { + "repo": "pypa/pipx", + "expected": "SystemExit)", + "got": "SystemExit)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "ValueError)", + "got": "FileNotFoundError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5517241379310345 + }, + { + "repo": "pypa/pipx", + "expected": "InterpreterResolutionError)", + "got": "InterpreterResolutionError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "ValueError, match=\"raised in test to exit early\")", + "got": "SystemExit)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.23333333333333334 + }, + { + "repo": "pypa/pipx", + "expected": "PipxError, match=r\"^Unable to parse package spec\")", + "got": "PipxError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "pypa/pipx", + "expected": "0)", + "got": "0)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "black_package_ref)", + "got": "black_package_ref)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "pycowsay_package_ref)", + "got": "pycowsay_package_ref)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "0", + "got": "0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "1", + "got": "1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "i", + "got": "i", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "B", + "got": "B", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "d", + "got": "d", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "2", + "got": "2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "v", + "got": "v", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "C", + "got": "C", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "c", + "got": "c", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "3", + "got": "3", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "b", + "got": "b", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "a", + "got": "b", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "D", + "got": "D", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "{}", + "got": "{\"val\": object}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.23529411764705882 + }, + { + "repo": "python-attrs/attrs", + "expected": "42", + "got": "42", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "i2", + "got": "i2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "e1", + "got": "e2", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "python-attrs/attrs", + "expected": "sa", + "got": "sa", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "c2", + "got": "c2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "c1", + "got": "c1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "ma", + "got": "ma", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "()", + "got": "(\"a\",)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "python-attrs/attrs", + "expected": "10", + "got": "10", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "11", + "got": "11", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "C1", + "got": "C1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "C2", + "got": "C2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "i.y", + "got": "C.y", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "python-attrs/attrs", + "expected": "c.a", + "got": "c.a", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "i.x", + "got": "i.x", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "i.a", + "got": "i.a", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "i.b", + "got": "i.b", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "i.c", + "got": "i.c", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "Int", + "got": "Int", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "res", + "got": "res", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "C()", + "got": "C(2, 3)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6 + }, + { + "repo": "python-attrs/attrs", + "expected": "src", + "got": "src", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "\"D\"", + "got": "\"D\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "val", + "got": "val", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "\"a\"", + "got": "\"a\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "\"b\"", + "got": "\"b\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "\"c\"", + "got": "\"c\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "c.\u00df", + "got": "c.\u00df", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "c.\u3141", + "got": "c.\u3141", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "c.x", + "got": "c.x", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "org", + "got": "EvolveCase(2, 3)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.10526315789473684 + }, + { + "repo": "python-attrs/attrs", + "expected": "h.x", + "got": "h.x", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "h.y", + "got": "h.y", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "c2_", + "got": "c2_", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "w()", + "got": "w()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "121", + "got": "121", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "289", + "got": "289", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "\"z\"", + "got": "\"z\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "\"h\"", + "got": "\"h\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "C(2)", + "got": "C(1)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.75 + }, + { + "repo": "python-attrs/attrs", + "expected": "C(1)", + "got": "C(2)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.75 + }, + { + "repo": "python-attrs/attrs", + "expected": "None", + "got": "None", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "c.a1", + "got": "c.a1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "c1.x", + "got": "c1.x", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "c2.x", + "got": "c2.x", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "True", + "got": "True", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "C3()", + "got": "C3()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "meth", + "got": "meth", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "init", + "got": "init", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "\"ab\"", + "got": "\"ab\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "bool", + "got": "bool", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "pf.y", + "got": "pf.y", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "pf.x", + "got": "pf.x", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "va.y", + "got": "va.y", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "p.x1", + "got": "p.x1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "p.x2", + "got": "p.x2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "c2.y", + "got": "c2.y", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "c2.z", + "got": "c2.z", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "(0,)", + "got": "(19, 2, 0)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.42857142857142855 + }, + { + "repo": "python-attrs/attrs", + "expected": "new_c", + "got": "new_c", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "taken", + "got": "taken", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "obj.a", + "got": "obj.a", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "obj.b", + "got": "obj.b", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "False", + "got": "False", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "\"foo\"", + "got": "\"foo\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "\"bar\"", + "got": "\"bar\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "calls", + "got": "calls", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "attrs", + "got": "attrs", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "C().x", + "got": "C.x", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.75 + }, + { + "repo": "python-attrs/attrs", + "expected": "e.msg", + "got": "e.msg", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "found", + "got": "found", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "other", + "got": "other", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "(20,)", + "got": "VersionInfo(19, 2, 1)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.23076923076923078 + }, + { + "repo": "python-attrs/attrs", + "expected": "C(1).x", + "got": "C().x", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "python-attrs/attrs", + "expected": "cls(1)", + "got": "cls(1)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "cls(2)", + "got": "cls(2)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "c(val)", + "got": "c(val)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "has(C)", + "got": "has(C)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "has(D)", + "got": "has(D)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "has(A)", + "got": "has(A)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "result", + "got": "result", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "d.xx()", + "got": "50", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "g_val1", + "got": "inst.val1", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5333333333333333 + }, + { + "repo": "python-attrs/attrs", + "expected": "g_val2", + "got": "g_val2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "g_val3", + "got": "g_val3", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "object", + "got": "object", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "C(2).x", + "got": "C(2).x", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "CI().x", + "got": "self.x", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "python-attrs/attrs", + "expected": "(\"b\",)", + "got": "(\"b\",)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "e.args", + "got": "e.args", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "inst.x", + "got": "inst.x", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "dir(A)", + "got": "A.__dict__", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.25 + }, + { + "repo": "python-attrs/attrs", + "expected": "c(None)", + "got": "c(None)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "C(2, 2)", + "got": "C(1, 2)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "python-attrs/attrs", + "expected": "repr(i)", + "got": "repr(i)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "add_str", + "got": "repr(e)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.14285714285714285 + }, + { + "repo": "python-attrs/attrs", + "expected": "hash(b)", + "got": "hash(b)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "hash(a)", + "got": "hash(b)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "python-attrs/attrs", + "expected": "NOTHING", + "got": "NOTHING", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "cls(*b)", + "got": "cls(*b)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "results", + "got": "results", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "list(d)", + "got": "list(d.keys())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "python-attrs/attrs", + "expected": "val + 1", + "got": "3", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "repr(b)", + "got": "repr(b)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "B(1, 2)", + "got": "B(1, 2)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "matched", + "got": "matched", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "repr(a)", + "got": "repr(b)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "python-attrs/attrs", + "expected": "C1Slots", + "got": "C1Slots", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "repr(v)", + "got": "repr(v)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "(19, 2)", + "got": "(19,)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "python-attrs/attrs", + "expected": "\"__eq__\"", + "got": "\"__eq__\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "\"__ne__\"", + "got": "\"__ne__\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "\"__lt__\"", + "got": "\"__lt__\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "\"__le__\"", + "got": "\"__le__\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "\"__gt__\"", + "got": "\"__gt__\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "\"__ge__\"", + "got": "\"__ge__\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "cls(2.0)", + "got": "cls(2.0)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "cls(1.0)", + "got": "cls(1.0)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "NotEqC()", + "got": "NotEqC(1, 2)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "python-attrs/attrs", + "expected": "i.meth()", + "got": "i.meth()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "frozen.x", + "got": "frozen.x", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "hash(c2)", + "got": "hash(c2)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "REGISTRY", + "got": "REGISTRY", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "hash(sa)", + "got": "hash(sa)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "expected", + "got": "expected", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "o.__eq__", + "got": "o.__eq__", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "o.__ne__", + "got": "o.__ne__", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "o.__le__", + "got": "o.__le__", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "i.called", + "got": "i.called", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "repr(B())", + "got": "repr(B())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "repr(C())", + "got": "repr(C())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "pipe()(o)", + "got": "pipe()(o)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "cls(1, 2)", + "got": "cls(1, 2)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "cls(2, 1)", + "got": "cls(2, 1)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "hash(obj)", + "got": "hash(obj_rt)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "python-attrs/attrs", + "expected": "type(cls)", + "got": "cls.__metaclass__", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.23076923076923078 + }, + { + "repo": "python-attrs/attrs", + "expected": "hash(C())", + "got": "hash(C())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "repr(E())", + "got": "repr(E())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "repr(D())", + "got": "repr(D())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "attribute", + "got": "attribute", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "[\"field\"]", + "got": "[\"field\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "repr(C(1))", + "got": "repr(C(1))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "i._private", + "got": "i.private", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9473684210526315 + }, + { + "repo": "python-attrs/attrs", + "expected": "hash(C(2))", + "got": "hash(C(2))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "not has(A)", + "got": "not has(A)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "a._default", + "got": "a._default", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "repr(C3())", + "got": "repr(C3())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "base_attrs", + "got": "[isinstance(a, _CountingAttr) for a in attrs]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2909090909090909 + }, + { + "repo": "python-attrs/attrs", + "expected": "len(attrs)", + "got": "len(attrs)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "a.metadata", + "got": "a.metadata", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "hash(C(1))", + "got": "hash(C(\"1\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "python-attrs/attrs", + "expected": "hash(D(1))", + "got": "hash(D(1))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "X.__dict__", + "got": "X.__dict__", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "y.metadata", + "got": "y.metadata", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "Ordered(2)", + "got": "Ordered(2)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "NoFields()", + "got": "NoFields(1, 2)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "python-attrs/attrs", + "expected": "[\"x\", \"y\"]", + "got": "[\"x\", \"y\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "C(1, 2, 3)", + "got": "C(1, 2, 3)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "ei.value.x", + "got": "ei.value.x", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "HasXSlot.x", + "got": "OverridesX.__slots__[0]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.30303030303030304 + }, + { + "repo": "python-attrs/attrs", + "expected": "self.BOUND", + "got": "self.BOUND", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "Concrete(1)", + "got": "Concrete(1)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "repr(cycle)", + "got": "repr(cycle)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "not NOTHING", + "got": "not NOTHING", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "has(A[str])", + "got": "not has(object)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "python-attrs/attrs", + "expected": "repr(C.D())", + "got": "repr(C.D())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "E)", + "got": "E)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "cls)", + "got": "TypeError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.14285714285714285 + }, + { + "repo": "python-attrs/attrs", + "expected": "TypeError)", + "got": "TypeError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "NameError)", + "got": "ImportError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6363636363636364 + }, + { + "repo": "python-attrs/attrs", + "expected": "Exception)", + "got": "Exception)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "ValueError)", + "got": "ValueError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "VException)", + "got": "VException)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "MyException)", + "got": "MyException)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "SystemError)", + "got": "SystemError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "AttributeError)", + "got": "UnannotatedAttributeError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7317073170731707 + }, + { + "repo": "python-attrs/attrs", + "expected": "ZeroDivisionError)", + "got": "ZeroDivisionError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "FloatingPointError)", + "got": "FloatingPointError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "FrozenInstanceError)", + "got": "FrozenInstanceError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "NotAnAttrsClassError)", + "got": "ValueError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4375 + }, + { + "repo": "python-attrs/attrs", + "expected": "TypeError, match=msg)", + "got": "TypeError, match=msg)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "FrozenAttributeError)", + "got": "FrozenAttributeError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "DefaultAlreadySetError)", + "got": "DefaultAlreadySetError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "UnannotatedAttributeError)", + "got": "UnannotatedAttributeError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "C)", + "got": "x=int, y=str)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.13333333333333333 + }, + { + "repo": "python-attrs/attrs", + "expected": "A)", + "got": "A)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "1", + "got": "1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "2", + "got": "1", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "5", + "got": "2", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "4", + "got": "4", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "3", + "got": "3", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "0", + "got": "0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "c", + "got": "Constraint(\"linux\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "v", + "got": "v", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "a", + "got": "a", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "m", + "got": "m.union(m.invert())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.1 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "[]", + "got": "{}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "10", + "got": "7", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "14", + "got": "14", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "()", + "got": "()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "ns", + "got": "ns", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "a1", + "got": "a3", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "a3", + "got": "a3", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "a2", + "got": "a2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "p2", + "got": "p2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"\"", + "got": "\"not-yanked\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "{}", + "got": "{}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "m2", + "got": "m2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"*\"", + "got": "\"file:demo\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "True", + "got": "True", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "None", + "got": "None", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "v124", + "got": "VersionRange(v124)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "open", + "got": "open", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "name", + "got": "name", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "path", + "got": "path", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "1234", + "got": "1234", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "data", + "got": "data", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"git\"", + "got": "\"git\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "range", + "got": "range", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "i < j", + "got": "i < j", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "i > j", + "got": "(i > j)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"MIT\"", + "got": "license_by_id(\"MIT\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"2.4\"", + "got": "\"2.1\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "0o644", + "got": "0o755", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "names", + "got": "names", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "setup", + "got": "setup", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"foo\"", + "got": "\"foo\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"1.2\"", + "got": "\"1.2\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "email", + "got": "email", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"qux\"", + "got": "\"qux\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"url\"", + "got": "\"url\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"baz\"", + "got": "\"baz\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "spec2", + "got": "spec2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "spec1", + "got": "spec1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "union", + "got": "union", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "parts", + "got": "parts", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "major", + "got": "major", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "minor", + "got": "minor", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "patch", + "got": "patch", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "output", + "got": "output", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "i <= j", + "got": "(i <= j)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "result", + "got": "result", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "parsed", + "got": "parsed", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "marker", + "got": "marker", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"^1.0\"", + "got": "\"^1.0\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"^2.0\"", + "got": "\"^2.0\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "[\"<2\"]", + "got": "[\"<2\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"~2.7\"", + "got": "\"==2.7\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7692307692307693 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "config", + "got": "config", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "str(m)", + "got": "str(m)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "single", + "got": "multi", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.18181818181818182 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "[\"all\"]", + "got": "\"all\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "version", + "got": "version", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "allowed", + "got": "allowed", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "license", + "got": "license", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "message", + "got": "message", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"0.1.0\"", + "got": "\"0.1.0\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "err_str", + "got": "err_str", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\">=3.6\"", + "got": "\"<=3.5\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7142857142857143 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "{\"bar\"}", + "got": "{\"bar\"}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "{\"baz\"}", + "got": "{\"baz\"}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "[\"bar\"]", + "got": "[\"bar\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "[\"foo\"]", + "got": "[\"foo\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"b;ar;\"", + "got": "\"b\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"1.2.4\"", + "got": "\"1.2.4\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "req.url", + "got": "req.url", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "expected", + "got": "expected", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "optional", + "got": "optional", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "namelist", + "got": "namelist", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "inverted", + "got": "inverted", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "metadata", + "got": "metadata", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"Django\"", + "got": "\"Django\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "ref_hash", + "got": "hash(dependency)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"2.18.0\"", + "got": "\"==2.18.0\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"jinja2\"", + "got": "\"jinja2\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"poetry\"", + "got": "\"poetry\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"django\"", + "got": "\"django\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"14.0.2\"", + "got": "\"14.0.2\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"123456\"", + "got": "\"123456\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"Custom\"", + "got": "\"Custom\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "hash(m2)", + "got": "hash(m2)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "req.name", + "got": "req.name", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "release2", + "got": "release2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "release1", + "got": "release1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"WARNING\"", + "got": "\"ERROR\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.375 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "ref_value", + "got": "ref_value", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"develop\"", + "got": "\"develop\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "expected1", + "got": "expected1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "expected2", + "got": "expected2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "precision", + "got": "precision", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "{\"orjson\"}", + "got": "{\"aiohttp\", \"aiohttp[dev]\"}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.32432432432432434 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "exp_python", + "got": "exp_python", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "exp_marker", + "got": "exp_marker", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "constraint", + "got": "constraint", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "normalized", + "got": "normalized", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "pip.stdout", + "got": "pip.stdout", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "wheel_data", + "got": "wheel_data", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"==2.7.15\"", + "got": "\">=3.6.0,<3.7.0\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "1727883000", + "got": "1727883000", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"requests\"", + "got": "\"requests\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "[\"export\"]", + "got": "[\"export\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"pathlib2\"", + "got": "\"pathlib2\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"John Doe\"", + "got": "\"John Doe\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"optional\"", + "got": "group_name", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"Optional\"", + "got": "\"optional\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "source_url", + "got": "source_url", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "Path(path)", + "got": "path", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "decomposed", + "got": "decomposed", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "parsed.rev", + "got": "parsed.rev", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "parsed.url", + "got": "url", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "m.invert()", + "got": "~m", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "self._parse", + "got": "not isinstance(template, str)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.15 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "expected[0]", + "got": "expected[0]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "expected[1]", + "got": "expected[1]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "v.allows(v)", + "got": "v.allows(\"1.2.3\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7142857142857143 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "description", + "got": "description", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "{\"\": [\"*\"]}", + "got": "{\"\": [\"*\"]}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "{\"\": \"src\"}", + "got": "\"src\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.625 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "[\"example\"]", + "got": "[\"my_package\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.56 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "0o644 << 16", + "got": "0", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "pretty_name", + "got": "pretty_name", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "frozenset()", + "got": "[\"python_version\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.06896551724137931 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"<3.11 >=3\"", + "got": "\"<=3.10\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.631578947368421 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"directory\"", + "got": "\"directory\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "hash(spec2)", + "got": "hash(spec2)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "dependency2", + "got": "dependency2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "parsed.name", + "got": "parsed.name", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "str(e.value)", + "got": "str(e.value)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "expected_any", + "got": "expected_any", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "expected_all", + "got": "expected_all", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "whl.exists()", + "got": "whl.exists()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "entry_points", + "got": "entry_points", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\">=3.6,<3.7\"", + "got": "\">=3.6\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7368421052631579 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "z.namelist()", + "got": "z.namelist()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\">=2.7,<2.8\"", + "got": "\">=2.7,<2.8\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "dep.is_vcs()", + "got": "isinstance(dep, URLDependency)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.23809523809523808 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "dep.is_url()", + "got": "not dep.is_vcs()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6428571428571429 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "package_hash", + "got": "hash(package_clone)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5161290322580645 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "content_type", + "got": "content_type", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "m.is_empty()", + "got": "not m.is_empty()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "v1.allows(v2)", + "got": "v1.allows(v2)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "v2.allows(v1)", + "got": "not v2.allows(v1)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8666666666666667 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "sorted_parsed", + "got": "sorted_parsed", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"Pretty.Name\"", + "got": "\"Pretty.Name\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\">3.6,<=3.10\"", + "got": "\">=3.6,<3.10\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9230769230769231 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "dep.is_file()", + "got": "dep.is_file()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"~=3.4 || <3\"", + "got": "\"~=3.4 || <3\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"pytest-mypy\"", + "got": "\"pytest-mypy\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "package_clone", + "got": "package_clone", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"poetry-core\"", + "got": "\"poetry-core\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"MIT License\"", + "got": "\"MIT License\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "expected_name", + "got": "expected_name", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "record.message", + "got": "record.message", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "expected_error", + "got": "expected_error", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "str(err.value)", + "got": "str(err.value)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "sdist.exists()", + "got": "sdist.exists()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "tar.getnames()", + "got": "tar.getnames()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "(main, extras)", + "got": "main", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "setup.exists()", + "got": "project_setup_py.exists()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.717948717948718 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "[\"module_src\"]", + "got": "[\"src.module_file\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"foo[bar,baz]\"", + "got": "\"foo\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5263157894736842 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "{\"bar\", \"baz\"}", + "got": "{\"bar\", \"baz\"}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "[\"foo\", \"bar\"]", + "got": "(\"foo\", \"bar\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"django-utils\"", + "got": "\"django-utils\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "normalized_url", + "got": "normalized_url", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "Path(unc_path)", + "got": "Path(unc_path)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "union.is_any()", + "got": "union.is_any()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "expected_union", + "got": "expected_union", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "req.constraint", + "got": "req.constraint", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "v.allows_all(v)", + "got": "v.allows_all(v)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "pip_show.stdout", + "got": "pip_show.stdout", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "[\"package_src\"]", + "got": "[\"package_src\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "caplog.messages", + "got": "caplog.text", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "dependency.name", + "got": "\"foo\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "{\"fred\", \"bar\"}", + "got": "(\"fred\", \"bar\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8666666666666667 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"example-wheel\"", + "got": "\"example_wheel\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9333333333333333 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"==3.4.* || <3\"", + "got": "\"3.4.*\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6363636363636364 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "package.version", + "got": "\"==1.2.3\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.08333333333333333 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "exception)", + "got": "RuntimeError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.34782608695652173 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "ValueError)", + "got": "ValueError) as e:", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 0.7857142857142857 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "error_type)", + "got": "error_type)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "RuntimeError)", + "got": "ValueError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "AttributeError)", + "got": "ValueError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "PyProjectError)", + "got": "PyProjectError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "FileNotFoundError)", + "got": "FileNotFoundError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "req)", + "got": "req)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytries/marisa-trie", + "expected": "0", + "got": "0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytries/marisa-trie", + "expected": "3", + "got": "3", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytries/marisa-trie", + "expected": "{}", + "got": "\"not a trie\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "pytries/marisa-trie", + "expected": "[]", + "got": "[b\"b\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "pytries/marisa-trie", + "expected": "key", + "got": "key", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytries/marisa-trie", + "expected": "trie", + "got": "trie", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytries/marisa-trie", + "expected": "None", + "got": "None", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytries/marisa-trie", + "expected": "keys", + "got": "keys", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytries/marisa-trie", + "expected": "trie2", + "got": "trie2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytries/marisa-trie", + "expected": "value", + "got": "value", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytries/marisa-trie", + "expected": "[b\"y\"]", + "got": "[b\"x\", b\"y\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "pytries/marisa-trie", + "expected": "wo_trie", + "got": "wo_trie", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytries/marisa-trie", + "expected": "[value]", + "got": "value", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "pytries/marisa-trie", + "expected": "as_dict", + "got": "as_dict", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytries/marisa-trie", + "expected": "[\"bar\"]", + "got": "[\"bar\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytries/marisa-trie", + "expected": "[b\"bar\"]", + "got": "[b\"bar\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytries/marisa-trie", + "expected": "contents", + "got": "trie", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "pytries/marisa-trie", + "expected": "set(keys)", + "got": "set(keys)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytries/marisa-trie", + "expected": "e.args[0]", + "got": "str(e)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.13333333333333333 + }, + { + "repo": "pytries/marisa-trie", + "expected": "set(data)", + "got": "set(data)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytries/marisa-trie", + "expected": "[b\"a\", b\"x\"]", + "got": "[b\"x\", b\"a\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "pytries/marisa-trie", + "expected": "[\"f\", \"foo\"]", + "got": "[\"f\", \"foo\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytries/marisa-trie", + "expected": "[b\"f\", b\"foo\"]", + "got": "[b\"foo\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7272727272727273 + }, + { + "repo": "pytries/marisa-trie", + "expected": "[\"foo\", \"foo\"]", + "got": "[\"foo\", \"foo\", \"fo\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8235294117647058 + }, + { + "repo": "pytries/marisa-trie", + "expected": "trie.key_id(key)", + "got": "trie.key_id(key)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytries/marisa-trie", + "expected": "pickle.load(buf)", + "got": "pickle.load(buf)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytries/marisa-trie", + "expected": "not str(warnings)", + "got": "not warnings.output", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "pytries/marisa-trie", + "expected": "{\"foo\", \"foobar\"}", + "got": "{\"foo\", \"foobar\"}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytries/marisa-trie", + "expected": "marisa_trie.Trie()", + "got": "marisa_trie.Trie()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytries/marisa-trie", + "expected": "{b\"foo\", b\"foobar\"}", + "got": "set([b\"foo\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.375 + }, + { + "repo": "pytries/marisa-trie", + "expected": "list(trie.iterkeys())", + "got": "[b\"foo\", b\"bar\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.05405405405405406 + }, + { + "repo": "pytries/marisa-trie", + "expected": "list(trie.iteritems())", + "got": "list(trie.iteritems())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytries/marisa-trie", + "expected": "{(\"bar\", trie[\"bar\"])}", + "got": "{(\"bar\", trie[\"bar\"])}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytries/marisa-trie", + "expected": "marisa_trie.BinaryTrie()", + "got": "marisa_trie.BinaryTrie()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytries/marisa-trie", + "expected": "list(trie.iterkeys(prefix))", + "got": "list(trie.iterkeys(prefix))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytries/marisa-trie", + "expected": "list(trie.iteritems(prefix))", + "got": "list(trie.iteritems(prefix))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytries/marisa-trie", + "expected": "marisa_trie.Trie([\"foo\", \"bar\"])", + "got": "marisa_trie.Trie([\"foo\", \"bar\"])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytries/marisa-trie", + "expected": "marisa_trie.Trie([\"foo\", \"boo\"])", + "got": "marisa_trie.Trie([\"foo\", \"baz\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9375 + }, + { + "repo": "pytries/marisa-trie", + "expected": "hasattr(marisa_trie.Trie, method)", + "got": "hasattr(marisa_trie.Trie, method)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytries/marisa-trie", + "expected": "fruit_trie.has_keys_with_prefix(\"\")", + "got": "fruit_trie.has_keys_with_prefix(\"a\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9859154929577465 + }, + { + "repo": "pytries/marisa-trie", + "expected": "fruit_trie.has_keys_with_prefix(\"a\")", + "got": "fruit_trie.has_keys_with_prefix(\"a\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytries/marisa-trie", + "expected": "fruit_trie.has_keys_with_prefix(\"pe\")", + "got": "not fruit_trie.has_keys_with_prefix(\"b\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "pytries/marisa-trie", + "expected": "hasattr(marisa_trie.BytesTrie, method)", + "got": "hasattr(marisa_trie.BytesTrie, method)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytries/marisa-trie", + "expected": "set(zip(keys, (trie[k] for k in keys)))", + "got": "set(zip(keys, range(len(keys))))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.647887323943662 + }, + { + "repo": "pytries/marisa-trie", + "expected": "hasattr(marisa_trie.BinaryTrie, method)", + "got": "hasattr(marisa_trie.BinaryTrie, method)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytries/marisa-trie", + "expected": "not empty_trie.has_keys_with_prefix(\"\")", + "got": "not empty_trie.has_keys_with_prefix(\"foo\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9629629629629629 + }, + { + "repo": "pytries/marisa-trie", + "expected": "marisa_trie.BinaryTrie([b\"foo\", b\"bar\"])", + "got": "marisa_trie.BinaryTrie([b\"foo\", b\"bar\"])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytries/marisa-trie", + "expected": "marisa_trie.BinaryTrie([b\"foo\", b\"boo\"])", + "got": "marisa_trie.BinaryTrie()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.75 + }, + { + "repo": "pytries/marisa-trie", + "expected": "not empty_trie.has_keys_with_prefix(b\"\")", + "got": "not empty_trie.has_keys_with_prefix(b\"\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytries/marisa-trie", + "expected": "lo_trie == lo_trie and wo_trie == wo_trie", + "got": "lo_trie != wo_trie", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.576271186440678 + }, + { + "repo": "pytries/marisa-trie", + "expected": "not set(trie.iter_prefixes_with_ids(\"b\"))", + "got": "set(trie.iter_prefixes_with_ids(\"b\")) == set()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8505747126436781 + }, + { + "repo": "pytries/marisa-trie", + "expected": "not empty_trie.has_keys_with_prefix(\"ab\")", + "got": "not empty_trie.has_keys_with_prefix(\"foo\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9397590361445783 + }, + { + "repo": "pytries/marisa-trie", + "expected": "not empty_trie.has_keys_with_prefix(b\"ab\")", + "got": "not empty_trie.has_keys_with_prefix(b\"foo\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9411764705882353 + }, + { + "repo": "pytries/marisa-trie", + "expected": "{ (\"f\", trie[\"f\"]), (\"foo\", trie[\"foo\"]), }", + "got": "{(\"f\", trie[\"f\"]), (\"foo\", trie[\"foo\"])}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.963855421686747 + }, + { + "repo": "pytries/marisa-trie", + "expected": "{ (\"foo\", trie[\"foo\"]), (\"foobar\", trie[\"foobar\"]), }", + "got": "set(zip(keys[:2], (trie[k] for k in keys[:2])))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.26 + }, + { + "repo": "pytries/marisa-trie", + "expected": "{ (b\"foo\", trie[b\"foo\"]), (b\"foobar\", trie[b\"foobar\"]), }", + "got": "set(zip(keys[:2], (trie[k] for k in keys[:2])))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.25 + }, + { + "repo": "pytries/marisa-trie", + "expected": "KeyError)", + "got": "IndexError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7 + }, + { + "repo": "pytries/marisa-trie", + "expected": "TypeError)", + "got": "TypeError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "s", + "got": "s", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "0", + "got": "20", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "qtile/qtile", + "expected": "1", + "got": "2", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "qtile/qtile", + "expected": "4", + "got": "0", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "qtile/qtile", + "expected": "d", + "got": "dimensions", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.18181818181818182 + }, + { + "repo": "qtile/qtile", + "expected": "9", + "got": "6", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "qtile/qtile", + "expected": "t", + "got": "t", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "2", + "got": "2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "3", + "got": "2", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "qtile/qtile", + "expected": "5", + "got": "5", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "8", + "got": "8", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "x", + "got": "x", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "y", + "got": "y", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "a", + "got": "a", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "e", + "got": "e", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "b", + "got": "b", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "c", + "got": "c", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "f", + "got": "f", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "7", + "got": "7", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "v", + "got": "v", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "10", + "got": "0", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "qtile/qtile", + "expected": "12", + "got": "12", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "32", + "got": "32", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "[]", + "got": "()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "qtile/qtile", + "expected": "16", + "got": "0", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "qtile/qtile", + "expected": "30", + "got": "30", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "20", + "got": "20", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "50", + "got": "50", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "40", + "got": "40", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "90", + "got": "90", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "80", + "got": "100", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "qtile/qtile", + "expected": "\"\"", + "got": "\"\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "25", + "got": "WIDTH // 2", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "qtile/qtile", + "expected": "s2", + "got": "s2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "60", + "got": "b.width", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "qtile/qtile", + "expected": "35", + "got": "30", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "qtile/qtile", + "expected": "70", + "got": "70", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "65", + "got": "55", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "qtile/qtile", + "expected": "11", + "got": "11", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "15", + "got": "20", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "qtile/qtile", + "expected": "24", + "got": "100", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "qtile/qtile", + "expected": "41", + "got": "41", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"a\"", + "got": "\"x/a\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.75 + }, + { + "repo": "qtile/qtile", + "expected": "252", + "got": "790", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "qtile/qtile", + "expected": "800", + "got": "800", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "[0]", + "got": "[0]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "two", + "got": "two", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "one", + "got": "one", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"b\"", + "got": "\"b\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "100", + "got": "100", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"A\"", + "got": "\"A\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"B\"", + "got": "\"B\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "num", + "got": "num", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "798", + "got": "100", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"c\"", + "got": "\"c\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "398", + "got": "398", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "578", + "got": "578", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"/\"", + "got": "\"..\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "qtile/qtile", + "expected": "200", + "got": "200", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "120", + "got": "a.width + b.width", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "qtile/qtile", + "expected": "[b]", + "got": "[b]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "[a]", + "got": "[a]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "c.x", + "got": "a.points == [(0, 0)]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.08695652173913043 + }, + { + "repo": "qtile/qtile", + "expected": "110", + "got": "120", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "qtile/qtile", + "expected": "now", + "got": "now", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "400", + "got": "0", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "qtile/qtile", + "expected": "300", + "got": "300", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "nus", + "got": "nus", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"$\"", + "got": "\"$\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"I\"", + "got": "\"B\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "qtile/qtile", + "expected": "\"O\"", + "got": "\"Paused\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "qtile/qtile", + "expected": "\"X\"", + "got": "\"N\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "qtile/qtile", + "expected": "\"+\"", + "got": "\"-\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "qtile/qtile", + "expected": "\"-\"", + "got": "\"-\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "url", + "got": "url", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"|\"", + "got": "\"|\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"=\"", + "got": "\"-\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "qtile/qtile", + "expected": "\"H\"", + "got": "\"H\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"V\"", + "got": "\"V\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "val", + "got": "val", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"M\"", + "got": "\"Volume: -\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "qtile/qtile", + "expected": "\"1\"", + "got": "\"2\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "qtile/qtile", + "expected": "\"2\"", + "got": "\"3\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "qtile/qtile", + "expected": "\" \"", + "got": "widget_text_on_screen(1)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "qtile/qtile", + "expected": "None", + "got": "None", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "[10]", + "got": "[10]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "v[0]", + "got": "isinstance(v, list)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.08695652173913043 + }, + { + "repo": "qtile/qtile", + "expected": "\"10\"", + "got": "\"10\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"OK\"", + "got": "False", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "qtile/qtile", + "expected": "img2", + "got": "img2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "pat0", + "got": "pat1", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.75 + }, + { + "repo": "qtile/qtile", + "expected": "pat1", + "got": "pat1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "90.0", + "got": "pytest.approx(90.0)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.34782608695652173 + }, + { + "repo": "qtile/qtile", + "expected": "\"aa\"", + "got": "\"aa\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "True", + "got": "True", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "term", + "got": "term", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "info", + "got": "info", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "text", + "got": "text", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "code", + "got": "code", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "root", + "got": "root", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "view", + "got": "view", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "tree", + "got": "tree", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "logs", + "got": "logs", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"25\"", + "got": "\"maintag:Canto: 10maintag:Slashdot: 9maintag:MintyFresh: 6\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.06349206349206349 + }, + { + "repo": "qtile/qtile", + "expected": "\"gb\"", + "got": "\"gb\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"us\"", + "got": "\"us\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\" H\"", + "got": "\"Plasma\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "qtile/qtile", + "expected": "\" V\"", + "got": "\" V\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"HS\"", + "got": "\" H\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.75 + }, + { + "repo": "qtile/qtile", + "expected": "mode", + "got": "mode", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "msgs", + "got": "msgs", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"two\"", + "got": "\"one\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6 + }, + { + "repo": "qtile/qtile", + "expected": "\"one\"", + "got": "\"one\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"500\"", + "got": "\"500\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "value", + "got": "value", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"Tab\"", + "got": "\"\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "qtile/qtile", + "expected": "c.bar", + "got": "3", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "qtile/qtile", + "expected": "group", + "got": "group.name", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "qtile/qtile", + "expected": "False", + "got": "False", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "empty", + "got": "empty", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"max\"", + "got": "\"max\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"foo\"", + "got": "\"groupbox\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5333333333333333 + }, + { + "repo": "qtile/qtile", + "expected": "\"Two\"", + "got": "\"One\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "qtile/qtile", + "expected": "\"One\"", + "got": "\"One\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "count", + "got": "count", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"XDG\"", + "got": "\"xdg-shell\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.25 + }, + { + "repo": "qtile/qtile", + "expected": "frame", + "got": "frame", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"SDC\"", + "got": "\"SDC\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "qtile", + "got": "qtile", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "[\"1\"]", + "got": "[\"1\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "[\"2\"]", + "got": "[\"2\", \"3\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "qtile/qtile", + "expected": "[\"b\"]", + "got": "[\"b\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "valid", + "got": "valid", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"BT \"", + "got": "\"Adapter: \"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.375 + }, + { + "repo": "qtile/qtile", + "expected": "\"N/A\"", + "got": "\"No Updates Available\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2962962962962963 + }, + { + "repo": "qtile/qtile", + "expected": "\"USD\"", + "got": "\"USD\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"5.0\"", + "got": "DEFAULT_TIMEOUT_NORMAL", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"50%\"", + "got": "\"V: 50\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "qtile/qtile", + "expected": "f.keys", + "got": "isinstance(f, confreader.Config)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.05263157894736842 + }, + { + "repo": "qtile/qtile", + "expected": "window", + "got": "window", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"True\"", + "got": "False", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.18181818181818182 + }, + { + "repo": "qtile/qtile", + "expected": "d[\"y\"]", + "got": "10", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"tile\"", + "got": "\"tile\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"INFO\"", + "got": "\"INFO\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "stdout", + "got": "stdout", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"dd-a\"", + "got": "\"dd-a\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "output", + "got": "output", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"test\"", + "got": "\"test\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "result", + "got": "result", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "length", + "got": "length", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "number", + "got": "number", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "HEIGHT", + "got": "HEIGHT", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "(0, 2)", + "got": "\"five\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "qtile/qtile", + "expected": "(0, 1)", + "got": "(1, 0)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "qtile/qtile", + "expected": "(0, 0)", + "got": "(0, 0)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "b.size", + "got": "10", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "qtile/qtile", + "expected": "[c, e]", + "got": "[b, c]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "qtile/qtile", + "expected": "[a, b]", + "got": "[b, a]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "qtile/qtile", + "expected": "a.size", + "got": "70", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "qtile/qtile", + "expected": "d.size", + "got": "approx(10)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "qtile/qtile", + "expected": "c.size", + "got": "20", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "qtile/qtile", + "expected": "e.size", + "got": "30", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"four\"", + "got": "\"one\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5454545454545454 + }, + { + "repo": "qtile/qtile", + "expected": "\"five\"", + "got": "\"three\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "qtile/qtile", + "expected": "\"Full\"", + "got": "\"^ 50% 0:28 15.00 W\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.15384615384615385 + }, + { + "repo": "qtile/qtile", + "expected": "\"Exit\"", + "got": "\"BT Headphones, Speaker\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.13333333333333333 + }, + { + "repo": "qtile/qtile", + "expected": "ACTIVE", + "got": "ACTIVE", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"None\"", + "got": "None", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "qtile/qtile", + "expected": "\"22\u00b0C\"", + "got": "\"22.0\u00b0C\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "qtile/qtile", + "expected": "widget", + "got": "widget.width", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "qtile/qtile", + "expected": "\"left\"", + "got": "\"right\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "qtile/qtile", + "expected": "[tray]", + "got": "[tray]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "[0, 10]", + "got": "[0, 90]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "qtile/qtile", + "expected": "[\"one\"]", + "got": "[\"zero\", \"one\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6363636363636364 + }, + { + "repo": "qtile/qtile", + "expected": "\"hello\"", + "got": "\"hello\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"False\"", + "got": "\"False\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "2)", + "got": "2)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "3)", + "got": "3)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "4)", + "got": "4)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "5)", + "got": "5)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "1)", + "got": "1)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "0)", + "got": "0)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "i)", + "got": "i)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"\")", + "got": "\"d\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "qtile/qtile", + "expected": "85)", + "got": "85)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"d\")", + "got": "\"test\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5454545454545454 + }, + { + "repo": "qtile/qtile", + "expected": "\"e\")", + "got": "\"e\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"a\")", + "got": "\"Test Layout\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "qtile/qtile", + "expected": "\"b\")", + "got": "\"b\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"4\")", + "got": "\"4\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "600)", + "got": "600)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"c\")", + "got": "\"c\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "IPCError)", + "got": "CommandError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6363636363636364 + }, + { + "repo": "qtile/qtile", + "expected": "ValueError)", + "got": "ValueError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "QtileError)", + "got": "QtileError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "SyntaxError)", + "got": "SyntaxError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "SelectError)", + "got": "IndexError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6086956521739131 + }, + { + "repo": "qtile/qtile", + "expected": "CommandError)", + "got": "libqtile.command.client.SelectError, match=\"Item not available in object\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.27586206896551724 + }, + { + "repo": "qtile/qtile", + "expected": "AttributeError)", + "got": "ValueError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "qtile/qtile", + "expected": "0.0)", + "got": "0.0)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "1", + "got": "\"TEST\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "2", + "got": "2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "5", + "got": "5", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "0", + "got": "\"playing\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "3", + "got": "3", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "4", + "got": "6", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "30", + "got": "30", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "10", + "got": "30", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "60", + "got": "60", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "20", + "got": "20", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"\"", + "got": "\"\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "[]", + "got": "[]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "123", + "got": "123", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "loop", + "got": "loop", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "4000", + "got": "4000", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "data", + "got": "data", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "True", + "got": "True", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "path", + "got": "path", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "False", + "got": "False", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "types", + "got": "types", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "loop2", + "got": "loop2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"this\"", + "got": "\"this\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"TEST\"", + "got": "\"TEST\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "[1, 5]", + "got": "[1, 5]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"NAME\"", + "got": "\"NAME\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "result", + "got": "result", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "buttons", + "got": "buttons", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"STATE\"", + "got": "\"STATE\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"value\"", + "got": "\"value\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "running", + "got": "running", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "response", + "got": "response", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "new_loop", + "got": "new_loop", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "str(exc)", + "got": "str(exc)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "end_time", + "got": "end_time", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "activity", + "got": "activity", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "old_loop", + "got": "old_loop", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"Testing\"", + "got": "\"Testing\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "client_id", + "got": "client_id", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"PLAYING\"", + "got": "\"PLAYING\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"DETAILS\"", + "got": "\"DETAILS\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "start_time", + "got": "start_time", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"party123\"", + "got": "\"party123\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "custom_pid", + "got": "custom_pid", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"WATCHING\"", + "got": "\"WATCHING\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"123456789\"", + "got": "\"123456789\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"large_key\"", + "got": "\"large_key\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"small_key\"", + "got": "\"small_key\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "os.getpid()", + "got": "os.getpid()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"LISTENING\"", + "got": "\"LISTENING\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "str(payload)", + "got": "payload", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7368421052631579 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "payload.data", + "got": "payload.data[\"args\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.75 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"In a match\"", + "got": "\"In a match\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "error_handler", + "got": "error_handler", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "async_handler", + "got": "async_handler", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"join_secret\"", + "got": "\"join_secret\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"My Activity\"", + "got": "\"My Activity\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"In progress\"", + "got": "\"In progress\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"Custom Name\"", + "got": "\"Custom Name\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"Large Image\"", + "got": "\"Large Image\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"Small Image\"", + "got": "\"Small Image\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"In Progress\"", + "got": "\"In Progress\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "mock_response", + "got": "{}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"SET_ACTIVITY\"", + "got": "\"SET_ACTIVITY\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"match_secret\"", + "got": "\"match_secret\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"Test Details\"", + "got": "\"Test Details\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"Current State\"", + "got": "\"Current State\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"Playing async\"", + "got": "\"Playing async\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"Playing a game\"", + "got": "\"Playing a game\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"My Custom Name\"", + "got": "\"My Custom Name\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "result[\"level1\"]", + "got": "result", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5454545454545454 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"spectate_secret\"", + "got": "\"spectate_secret\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"Doing something\"", + "got": "\"Doing something\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"Current Details\"", + "got": "\"Current Details\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "status_type.value", + "got": "status_type.value", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"Large Image Text\"", + "got": "\"Large Image Text\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "activity_type.value", + "got": "activity_type.value", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"In a ranked match\"", + "got": "\"In a ranked match\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "payload.data[\"args\"]", + "got": "payload.data", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.75 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "ActivityType.PLAYING", + "got": "ActivityType.PLAYING", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "not loop.is_closed()", + "got": "not loop.is_running()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6829268292682927 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "mock_handshake.called", + "got": "mock_handshake.called", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "ActivityType.WATCHING", + "got": "ActivityType.WATCHING", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"Custom Activity Name\"", + "got": "\"Custom Activity Name\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "payload2.data[\"nonce\"]", + "got": "payload2.data[\"nonce\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"Working on something\"", + "got": "\"Working on something\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "ActivityType.LISTENING", + "got": "ActivityType.LISTENING", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "StatusDisplayType.NAME", + "got": "StatusDisplayType.NAME", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "StatusDisplayType.STATE", + "got": "StatusDisplayType.STATE", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "len(json.dumps(payload))", + "got": "len(json.dumps(payload))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "not new_loop.is_closed()", + "got": "not new_loop.is_closed()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "presence.loop.is_closed()", + "got": "not sys._getframe().f_back", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.27450980392156865 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "StatusDisplayType.DETAILS", + "got": "StatusDisplayType.DETAILS", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"https://example.com/game\"", + "got": "\"https://example.com/game\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"https://example.com/state\"", + "got": "\"https://example.com/state\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"https://example.com/async\"", + "got": "\"https://example.com/async\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "StatusDisplayType.NAME.value", + "got": "\"NAME\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.23529411764705882 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "ActivityType.LISTENING.value", + "got": "\"LISTENING\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "isinstance(str(payload), str)", + "got": "not payload.clear_none", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.39215686274509803 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"https://example.com/details\"", + "got": "\"https://example.com/details\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "client.sock_writer.write.called", + "got": "client.sock_writer.write.called", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"https://example.com/match/123\"", + "got": "\"https://example.com/match/123\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"https://example.com/large.png\"", + "got": "\"https://example.com/large.png\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "payload.data[\"args\"][\"activity\"]", + "got": "payload.data[\"args\"][\"activity\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "isinstance(client.client_id, str)", + "got": "isinstance(client.client_id, str)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "presence.sock_writer.write.called", + "got": "presence.sock_writer.write.called", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"https://example.com/match/12345\"", + "got": "\"https://example.com/match/12345\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "client.create_reader_writer.called", + "got": "client.sock_reader", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5769230769230769 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"https://cdn.example.com/large.png\"", + "got": "\"https://cdn.example.com/large.png\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "isinstance(exc, PyPresenceException)", + "got": "isinstance(exc, PyPresenceException)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"https://example.com/images/large.png\"", + "got": "\"https://example.com/images/large.png\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"https://example.com/images/small.png\"", + "got": "\"https://example.com/images/small.png\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "path.startswith(r\"\\\\?\\pipe\\discord-ipc-\")", + "got": "path.startswith(\"\\\\\\\\.\\\\pipe\\\\\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7397260273972602 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"https://cdn.example.com/images/large.png\"", + "got": "\"https://cdn.example.com/images/large.png\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"https://cdn.example.com/images/small.png\"", + "got": "\"https://cdn.example.com/images/small.png\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "isinstance(loop, asyncio.AbstractEventLoop)", + "got": "isinstance(loop, asyncio.AbstractEventLoop)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "path is None or \"discord-ipc-\" in str(path)", + "got": "not path", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.1568627450980392 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "issubclass(InvalidPipe, PyPresenceException)", + "got": "issubclass(InvalidID, PyPresenceException)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9302325581395349 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "isinstance(loop1, asyncio.AbstractEventLoop)", + "got": "loop1.is_running()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3225806451612903 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "isinstance(loop2, asyncio.AbstractEventLoop)", + "got": "isinstance(loop2, asyncio.AbstractEventLoop)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"assets\" not in activity or activity[\"assets\"]", + "got": "\"assets\" not in activity or activity[\"assets\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "isinstance(new_loop, asyncio.AbstractEventLoop)", + "got": "isinstance(new_loop, asyncio.AbstractEventLoop)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "issubclass(InvalidArgument, PyPresenceException)", + "got": "isinstance(PyPresenceException, BaseException)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5319148936170213 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "issubclass(DiscordNotFound, PyPresenceException)", + "got": "issubclass(PipeClosed, PyPresenceException)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7912087912087912 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "presence.sock_writer._call_connection_lost.called", + "got": "presence.sock_writer._call_connection_lost.called", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"timestamps\" not in activity or activity[\"timestamps\"]", + "got": "\"timestamps\" not in activity", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6829268292682927 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "InvalidID)", + "got": "InvalidID)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "PipeClosed)", + "got": "PipeClosed)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "InvalidPipe)", + "got": "InvalidPipe)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "InvalidArgument)", + "got": "PyPresenceException, match=\"must be a coroutine\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.15384615384615385 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "ResponseTimeout)", + "got": "ResponseTimeout)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "ConnectionTimeout)", + "got": "ConnectionTimeout)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "ServerError, match=\"Test error\")", + "got": "ServerError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5454545454545454 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "AssertionError, match=\"You must connect your client\")", + "got": "PipeClosed)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.09375 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "PyPresenceException, match=\"Error handler must be a function\")", + "got": "InvalidArgument)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.1282051282051282 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "PyPresenceException, match=\"should only accept two arguments\")", + "got": "PyPresenceException)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4878048780487805 + }, + { + "repo": "reactive-python/reactpy-django", + "expected": "0", + "got": "0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reactive-python/reactpy-django", + "expected": "2", + "got": "2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reactive-python/reactpy-django", + "expected": "1", + "got": "1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reactive-python/reactpy-django", + "expected": "36", + "got": "36", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reactive-python/reactpy-django", + "expected": "22", + "got": "22", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reactive-python/reactpy-django", + "expected": "\"\"", + "got": "''", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "reactive-python/reactpy-django", + "expected": "None", + "got": "None", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reactive-python/reactpy-django", + "expected": "\"/router/\"", + "got": "\"Router String\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.56 + }, + { + "repo": "reactive-python/reactpy-django", + "expected": "\"subroute/\"", + "got": "\"/router/subroute/\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7333333333333333 + }, + { + "repo": "reactive-python/reactpy-django", + "expected": "query_exists", + "got": "query_exists", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reactive-python/reactpy-django", + "expected": "initial_uuid", + "got": "initial_uuid", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reactive-python/reactpy-django", + "expected": "correct_ports", + "got": "correct_ports", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reactive-python/reactpy-django", + "expected": "initial_count", + "got": "initial_count", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reactive-python/reactpy-django", + "expected": "\"rgb(0, 0, 255)\"", + "got": "\"rgb(255, 0, 0)\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.75 + }, + { + "repo": "reactive-python/reactpy-django", + "expected": "not query_exists", + "got": "not query_exists", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reactive-python/reactpy-django", + "expected": "initial_count + 1", + "got": "initial_count - 1", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9411764705882353 + }, + { + "repo": "reactive-python/reactpy-django", + "expected": "elem.text_content()", + "got": "elem.text_content()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reactive-python/reactpy-django", + "expected": "'\"my_offline_path\"'", + "got": "\"my_offline_path\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9444444444444444 + }, + { + "repo": "reactive-python/reactpy-django", + "expected": "\"/router/any/\"", + "got": "\"/router/any//\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.88 + }, + { + "repo": "reactive-python/reactpy-django", + "expected": "user_data_div.text_content()", + "got": "user_data_div.inner_text()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7407407407407407 + }, + { + "repo": "reactive-python/reactpy-django", + "expected": "\"/router/path//\"", + "got": "\"/router/path//123/\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9333333333333333 + }, + { + "repo": "reactive-python/reactpy-django", + "expected": "\"/router/slug//\"", + "got": "\"/router/slug//\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reactive-python/reactpy-django", + "expected": "\"/router/uuid//\"", + "got": "\"/router/uuid//\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reactive-python/reactpy-django", + "expected": "\"/router/string//\"", + "got": "\"/router/string//\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9508196721311475 + }, + { + "repo": "reactive-python/reactpy-django", + "expected": "\"/router/unspecified//\"", + "got": "\"unspecified/\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6363636363636364 + }, + { + "repo": "reactive-python/reactpy-django", + "expected": "\"/router/integer//\"", + "got": "\"/router/integer//\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9285714285714286 + }, + { + "repo": "reactive-python/reactpy-django", + "expected": "broken_component.text_content()", + "got": "broken_component.text_content()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reactive-python/reactpy-django", + "expected": "path.get_attribute(\"data-path\")", + "got": "path.text_content()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "reactive-python/reactpy-django", + "expected": "'{% component \"my.component\" %}'", + "got": "'{% component \"my.component\" %}'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reactive-python/reactpy-django", + "expected": "not re.search(COMPONENT_REGEX, \"\")", + "got": "not re.search(COMPONENT_REGEX, \"{% component 'my.component' %}\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6938775510204082 + }, + { + "repo": "reactive-python/reactpy-django", + "expected": "not re.search(COMPONENT_REGEX, \" \")", + "got": "not re.search(COMPONENT_REGEX, \"{% component %}\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "reactive-python/reactpy-django", + "expected": "\"/router/two///\"", + "got": "\"/router/two///\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.987012987012987 + }, + { + "repo": "reactive-python/reactpy-django", + "expected": "TodoItem.objects.filter(text=uuid).exists()", + "got": "TodoItem.objects.filter(text=uuid).exists()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reactive-python/reactpy-django", + "expected": "TimeoutError)", + "got": "TimeoutError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "1", + "got": "2", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "reata/sqllineage", + "expected": "2", + "got": "3", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "reata/sqllineage", + "expected": "\"\"", + "got": "\"\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "reata/sqllineage", + "expected": "[]", + "got": "{}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "reata/sqllineage", + "expected": "str", + "got": "str", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "sql", + "got": "sql", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "bool", + "got": "bool", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "None", + "got": "None", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "expected", + "got": "set(expected)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7619047619047619 + }, + { + "repo": "reata/sqllineage", + "expected": "schemas[i]", + "got": "\"db\" + str(i)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.17391304347826086 + }, + { + "repo": "reata/sqllineage", + "expected": "Table(\"a\")", + "got": "Table(\"a\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "str(runner)", + "got": "runner.has_lineage", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.41379310344827586 + }, + { + "repo": "reata/sqllineage", + "expected": "repr(Schema())", + "got": "repr(Column(\"col\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5454545454545454 + }, + { + "repo": "reata/sqllineage", + "expected": "repr(Path(\"\"))", + "got": "repr(Path(\"a.b\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9032258064516129 + }, + { + "repo": "reata/sqllineage", + "expected": "NetworkXGraphOperator", + "got": "NetworkXGraphOperator", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "RustworkXGraphOperator", + "got": "RustworkXGraphOperator", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "repr(StatementLineageHolder())", + "got": "\"StatementLineageHolder()\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "reata/sqllineage", + "expected": "not isinstance(col.parent, SubQuery)", + "got": "not col.is_subquery", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.509090909090909 + }, + { + "repo": "reata/sqllineage", + "expected": "SqlFluffColumn._extract_source_columns(segment_mock)", + "got": "find_from_expression_element(segment_mock)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.46808510638297873 + }, + { + "repo": "reata/sqllineage", + "expected": "container.status.startswith(str(HTTPStatus.OK.value))", + "got": "200", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "reata/sqllineage", + "expected": "TypeError)", + "got": "NotImplementedError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5333333333333333 + }, + { + "repo": "reata/sqllineage", + "expected": "SystemExit)", + "got": "SystemExit) as e:", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 0.7857142857142857 + }, + { + "repo": "reata/sqllineage", + "expected": "ConfigException)", + "got": "ConfigException)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "SQLLineageException)", + "got": "InvalidSyntaxException)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5581395348837209 + }, + { + "repo": "reata/sqllineage", + "expected": "NotImplementedError)", + "got": "SQLLineageException)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.25 + }, + { + "repo": "reata/sqllineage", + "expected": "InvalidSyntaxException)", + "got": "InvalidSyntaxException)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "MetaDataProviderException)", + "got": "MetaDataProviderException)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "UnsupportedStatementException)", + "got": "UnsupportedStatementException)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "sql)", + "got": "[(ColumnQualifierTuple(\"'2012-12-21'\", \"tab2\"), ColumnQualifierTuple(\"col2\", \"tab1\"))])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.04395604395604396 + }, + { + "repo": "reata/sqllineage", + "expected": "None)", + "got": "None)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "dialect)", + "got": "dialect)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "{\"foo\"})", + "got": "{\"foo\"})", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "{\"tgt\"})", + "got": "{\"tgt\"})", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "{\"tab1\"})", + "got": "{\"tab1\"})", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "{\"tab2\"})", + "got": "{\"tab2\"})", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "{\"tab3\"})", + "got": "{\"tab3\"})", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "{\"host\"})", + "got": "{\"host\"})", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "{\"view1\"})", + "got": "{\"view1\"})", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "{\"tab_1\"})", + "got": "{\"tab_1\"}, dialect=dialect)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5405405405405406 + }, + { + "repo": "reata/sqllineage", + "expected": "\"USE db1\")", + "got": "\"test_use\", \"test_use\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.24242424242424243 + }, + { + "repo": "reata/sqllineage", + "expected": "{\"tests\"})", + "got": "{\"tests\"})", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "{\"tab_b\"})", + "got": "{\"tab_b\"})", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "{\"tab_c\"})", + "got": "{\"tab_c\"})", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "{\"target\"})", + "got": "{\"target\"})", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "\"SELECT 1\")", + "got": "\"SELECT 1 FROM tab1\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6875 + }, + { + "repo": "reata/sqllineage", + "expected": "{\"segment\"})", + "got": "{\"segment\"})", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "{\"my_table\"})", + "got": "\"SELECT 456 FROM my_table WHERE id > 10 LIMIT 5\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3548387096774194 + }, + { + "repo": "reata/sqllineage", + "expected": "{\"apachelog\"})", + "got": "{\"apachelog\"})", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "\"analyze tab\")", + "got": "\"ANALYZE tab1\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4827586206896552 + }, + { + "repo": "reata/sqllineage", + "expected": "{\"films_all\"})", + "got": "{\"films_all\"}, dialect=dialect)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6222222222222222 + }, + { + "repo": "reata/sqllineage", + "expected": "\"SELECT NOW()\")", + "got": "\"SELECT COUNT(*)\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7878787878787878 + }, + { + "repo": "reata/sqllineage", + "expected": "{\"foo\", \"bar\"})", + "got": "{\"foo\", \"bar\"}, dialect=dialect)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6382978723404256 + }, + { + "repo": "reata/sqllineage", + "expected": "{\"schema1.tab1\"})", + "got": "\"schema1.tab1\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9375 + }, + { + "repo": "reata/sqllineage", + "expected": "{\"tab1\", \"tab2\"})", + "got": "{\"tab1\", \"tab2\"})", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "{\"default.tab2\"})", + "got": "{\"tab2\"})", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6923076923076923 + }, + { + "repo": "reata/sqllineage", + "expected": "{\"films_recent\"})", + "got": "dialect)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.48 + }, + { + "repo": "reata/sqllineage", + "expected": "{\"manufacturers\"})", + "got": "{\"manufacturers\", \"get_product_names\"}, dialect=dialect)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4864864864864865 + }, + { + "repo": "reata/sqllineage", + "expected": "\"DESC FUNCTION abs\")", + "got": "\"DESCRIBE FUNCTION simple_udf\", dialect=dialect)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5294117647058824 + }, + { + "repo": "reata/sqllineage", + "expected": "\"refresh table tab1\")", + "got": "\"refresh table tab1\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "\"uncache table tab1\")", + "got": "\"uncache table tab1\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "{\"db1.schema1.tbl1\"})", + "got": "{\"db1.schema1.tbl1\"})", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "\"TRUNCATE TABLE tab1\")", + "got": "\"TRUNCATE TABLE tab1\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "\"SHOW FUNCTIONS trim\")", + "got": "\"show functions\", dialect=dialect)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.21428571428571427 + }, + { + "repo": "reata/sqllineage", + "expected": "{\"tab1\", \"tab2\", \"tab3\"})", + "got": "{\"tab1\", \"tab2\", \"tab3\"})", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "\"DELETE FROM table tab1\")", + "got": "\"DELETE FROM tab1\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8636363636363636 + }, + { + "repo": "reata/sqllineage", + "expected": "\"show create table tab1\")", + "got": "\"show create table tab1\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "\"DROP FUNCTION test_avg\")", + "got": "\"DROP FUNCTION abs\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "reata/sqllineage", + "expected": "{\"dbname.dto.tablename\"})", + "got": "{\"dbname.dto.tablename\"}, dialect=dialect)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.746268656716418 + }, + { + "repo": "reata/sqllineage", + "expected": "\"analyze tab (col1, col2)\")", + "got": "\"analyze tab column col\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8461538461538461 + }, + { + "repo": "reata/sqllineage", + "expected": "{\"Production.ScrapReason\"})", + "got": "{\"Production.ScrapReason\"}, dialect=dialect)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7605633802816901 + }, + { + "repo": "reata/sqllineage", + "expected": "\"DROP TABLE IF EXISTS tab1\")", + "got": "\"DROP db1\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5128205128205128 + }, + { + "repo": "reata/sqllineage", + "expected": "\"DROP VIEW IF EXISTS view1\")", + "got": "\"DROP VIEW IF EXISTS view1\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "\"rename table tab1 to tab2\")", + "got": "dialect)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2777777777777778 + }, + { + "repo": "reata/sqllineage", + "expected": "{\"schema1.tab1\", \"db.wtab1\"})", + "got": "{\"schema1.tab1\", \"db.wtab1\"})", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "\"ADD JAR /tmp/SimpleUdf.jar\")", + "got": "\"add jar jar1\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3181818181818182 + }, + { + "repo": "reata/sqllineage", + "expected": "\"uncache table if exists tab1\")", + "got": "\"uncache table if exists tab1\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "{Path(\"hdfs://path/to/folder\")})", + "got": "dialect=dialect)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.20833333333333334 + }, + { + "repo": "reata/sqllineage", + "expected": "\"SELECT * FROM (VALUES (1, 2))\")", + "got": "\"SELECT * FROM (VALUES (1, 'a'), (2, 'b'))\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8421052631578947 + }, + { + "repo": "reata/sqllineage", + "expected": "{\"tab1\", \"tab2\", \"tab3\", \"tab4\"})", + "got": "{\"tab1\", \"tab2\", \"tab3\", \"tab4\"})", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "\"ALTER TABLE tab1 rename TO tab2\")", + "got": "\"ALTER TABLE tab1 RENAME TO tab2\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8235294117647058 + }, + { + "repo": "reata/sqllineage", + "expected": "\"SELECT * FROM (\\nVALUES (1, 2))\")", + "got": "\"SELECT * FROM (VALUES\\n(1, 2))\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9253731343283582 + }, + { + "repo": "reata/sqllineage", + "expected": "\"SET spark.sql.variable.substitute\")", + "got": "\"SET spark.sql.variable.substitute\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "\"cache table tab1 select * from tab2\")", + "got": "\"cache table tab1\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "reata/sqllineage", + "expected": "{Path(\"examples/src/main/resources/\")})", + "got": "{Path(\"examples/src/main/resources/\")}, {\"examples\"})", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8478260869565217 + }, + { + "repo": "reata/sqllineage", + "expected": "\"SELECT trim(BOTH ' ' FROM ' abc ')\")", + "got": "\"SELECT trim('abc') FROM tab1\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.704225352112676 + }, + { + "repo": "reata/sqllineage", + "expected": "{\"tab1\", \"tab2\", \"tab3\", \"tab4\", \"tab5\"})", + "got": "{\"tab1\", \"tab2\", \"tab3\", \"tab4\", \"tab5\"})", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "\"rename table tab1 to tab2, tab3 to tab4\")", + "got": "\"rename tables tab1 to tab2\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7887323943661971 + }, + { + "repo": "reata/sqllineage", + "expected": "\"SET spark.sql.variable.substitute=false\")", + "got": "\"SET command\", dialect=dialect)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4383561643835616 + }, + { + "repo": "reata/sqllineage", + "expected": "\"WITH tab1 AS (SELECT 1) SELECT * FROM tab1\")", + "got": "\"test_with_select\", \"test_with_select_with_select\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.10416666666666667 + }, + { + "repo": "reata/sqllineage", + "expected": "\"SELECT * FROM (VALUES (1, 2)) AS t(col1, col2)\")", + "got": "\"SELECT * FROM (VALUES (1, 2)) AS dt\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8505747126436781 + }, + { + "repo": "reata/sqllineage", + "expected": "{Path(\"s3://amzn-s3-demo-bucket/tickit/unload/venue_\")})", + "got": "{\"s3://amzn-s3-demo-bucket/tickit/unload/venue_\"}, dialect=dialect)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8130081300813008 + }, + { + "repo": "reata/sqllineage", + "expected": "[(ColumnQualifierTuple(\"*\", \"tab2\"), ColumnQualifierTuple(\"*\", \"tab1\"))])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.07894736842105263 + }, + { + "repo": "reata/sqllineage", + "expected": "[(ColumnQualifierTuple(\"a\", \"tab1\"), ColumnQualifierTuple(\"a\", \"tab2\"))])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.07894736842105263 + }, + { + "repo": "reata/sqllineage", + "expected": "[(ColumnQualifierTuple(\"v\", \"src\"), ColumnQualifierTuple(\"v\", \"target\"))])", + "got": "\"\"\"target.v -> src.v\"\"\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.22448979591836735 + }, + { + "repo": "reata/sqllineage", + "expected": "[(ColumnQualifierTuple(\"k\", \"src\"), ColumnQualifierTuple(\"k\", \"target\"))])", + "got": "[ColumnQualifierTuple(\"k\", \"src\")])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6422018348623854 + }, + { + "repo": "reata/sqllineage", + "expected": "[(ColumnQualifierTuple(\"col1\", \"tab2\"), ColumnQualifierTuple(\"col2\", \"tab1\"))])", + "got": "[(\"col1\", \"tab2\"), (\"col2\", \"tab1\")])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6379310344827587 + }, + { + "repo": "reata/sqllineage", + "expected": "[(ColumnQualifierTuple(\"col1\", \"tab2\"), ColumnQualifierTuple(\"col1\", \"tab1\"))])", + "got": "[(ColumnQualifierTuple(\"col1\", \"tab2\"), ColumnQualifierTuple(\"col1\", \"tab1\"))])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "\"\"\"--comment DROP TABLE IF EXISTS tab1\"\"\")", + "got": "\"DROP TABLE IF EXISTS tab1 COMMENT 'test'\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6588235294117647 + }, + { + "repo": "reata/sqllineage", + "expected": "[ (ColumnQualifierTuple(\"*\", \"tab1\"), ColumnQualifierTuple(\"*\", \"tab3\")), ])", + "got": "[(ColumnQualifierTuple(\"*\", \"tab3\"),)])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6608695652173913 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"*\", \"tab2\"), ColumnQualifierTuple(\"*\", \"tab3\"), ), ])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.07317073170731707 + }, + { + "repo": "reata/sqllineage", + "expected": "\"\"\"CREATE FUNCTION simple_udf AS 'SimpleUdf' USING JAR '/tmp/SimpleUdf.jar'\"\"\")", + "got": "\"CREATE FUNCTION tab1 AS 'com.exasol.udf.MyUDF'\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.484375 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"col1\", \"tab2\"), ColumnQualifierTuple(\"col1\", \"tab1\"), ) ])", + "got": "[(\"col1\", \"col1\")])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.36893203883495146 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"host\", \"tab2\"), ColumnQualifierTuple(\"host\", \"tab1\"), ) ])", + "got": "[(\"host\", \"host\")])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.36893203883495146 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"col2\", \"tab1\"), ColumnQualifierTuple(\"col1\", \"tab2\"), ) ])", + "got": "[(\"col2\", \"tab1\")])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.36893203883495146 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"col1\", \"tab1\"), ColumnQualifierTuple(\"col2\", \"tab2\"), ), ])", + "got": "[(ColumnQualifierTuple(\"col1\", \"tab1\"),)])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6614173228346457 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"col1\", \"tab2\"), ColumnQualifierTuple(\"col1\", \"tab1\"), ), ])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.06818181818181818 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"col1\", \"tab2\"), ColumnQualifierTuple(\"col1\", \"tab3\"), ), ])", + "got": "[(ColumnQualifierTuple(\"col1\", \"tab2\"),)])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6614173228346457 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"col_b\", \"tab_b\"), ColumnQualifierTuple(\"col_a\", \"tab_c\"), ), ])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.06521739130434782 + }, + { + "repo": "reata/sqllineage", + "expected": "\"\"\"CREATE OR REPLACE FUNCTION simple_udf AS 'SimpleUdf' USING JAR '/tmp/SimpleUdf.jar'\"\"\")", + "got": "\"\"\"CREATE OR REPLACE FUNCTION simple_udf AS 'SimpleUdf' USING JAR '/tmp/SimpleUdf.jar'\"\"\", dialect=dialect)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9137055837563451 + }, + { + "repo": "reata/sqllineage", + "expected": "\"\"\"CREATE TEMPORARY FUNCTION simple_temp_udf AS 'SimpleUdf' USING JAR '/tmp/SimpleUdf.jar'\"\"\")", + "got": "\"CREATE TEMPORARY FUNCTION simple_udf AS 'SimpleUdf'\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7297297297297297 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"col1\", \"tab2\"), ColumnQualifierTuple(\"cast(col1 as timestamp)\", \"tab1\"), ) ])", + "got": "\"\"\"tab2.col1 -> tab1.col1\"\"\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2878787878787879 + }, + { + "repo": "reata/sqllineage", + "expected": "\"\"\"CREATE OR REPLACE TEMPORARY FUNCTION simple_temp_udf AS 'SimpleUdf' USING JAR '/tmp/SimpleUdf.jar'\"\"\")", + "got": "\"\"\"CREATE OR REPLACE TEMPORARY FUNCTION simple_temp_udf AS 'SimpleUdf' USING JAR '/tmp/SimpleUdf.jar'\"\"\", dialect=dialect)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9251101321585903 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"ProductID\", \"Production.Product\"), ColumnQualifierTuple(\"SecondColumnHeading\", \"foo\"), ) ])", + "got": "\"FirstColumnHeading\", \"ProductID\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.31788079470198677 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"col1\", \"tab2\"), ColumnQualifierTuple( \"CASE WHEN col1 = 1 THEN 'V1' WHEN col1 = 2 THEN 'V2' END\", \"tab1\" ), ), ])", + "got": "\"\"\"tab2.col1 -> tab1.col1\"\"\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2261904761904762 + }, + { + "repo": "reata/sqllineage", + "expected": "[ (ColumnQualifierTuple(\"*\", \"tab2\"), ColumnQualifierTuple(\"*\", \"tab1\")), (ColumnQualifierTuple(\"*\", \"tab3\"), ColumnQualifierTuple(\"*\", \"tab1\")), ])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.039735099337748346 + }, + { + "repo": "reata/sqllineage", + "expected": "[ (ColumnQualifierTuple(\"b\", \"tab1\"), ColumnQualifierTuple(\"b\", \"tab3\")), (ColumnQualifierTuple(\"d\", \"tab2\"), ColumnQualifierTuple(\"d\", \"tab3\")), ])", + "got": "[(ColumnQualifierTuple(\"b\", \"tab1\"), ColumnQualifierTuple(\"b\", \"tab3\"))])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6606334841628959 + }, + { + "repo": "reata/sqllineage", + "expected": "[ (ColumnQualifierTuple(\"k\", \"src\"), ColumnQualifierTuple(\"k\", \"target\")), (ColumnQualifierTuple(\"v\", \"src\"), ColumnQualifierTuple(\"v\", \"target\")), ])", + "got": "[(ColumnQualifierTuple(\"k\", \"src\"), ColumnQualifierTuple(\"k\", \"target\"))])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6607142857142857 + }, + { + "repo": "reata/sqllineage", + "expected": "[ (ColumnQualifierTuple(\"v\", \"src\"), ColumnQualifierTuple(\"v\", \"target\")), (ColumnQualifierTuple(\"k\", \"src\"), ColumnQualifierTuple(\"k\", \"target\")), ])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0392156862745098 + }, + { + "repo": "reata/sqllineage", + "expected": "[ (ColumnQualifierTuple(\"v\", \"src\"), ColumnQualifierTuple(\"v\", \"target\")), (ColumnQualifierTuple(\"v1\", \"src\"), ColumnQualifierTuple(\"v1\", \"target\")), ])", + "got": "[(\"v\", \"src\"), (\"v1\", \"src\")])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.32967032967032966 + }, + { + "repo": "reata/sqllineage", + "expected": "[ (ColumnQualifierTuple(\"col1\", \"tab2\"), ColumnQualifierTuple(\"rn\", \"tab1\")), (ColumnQualifierTuple(\"col2\", \"tab2\"), ColumnQualifierTuple(\"rn\", \"tab1\")), ])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.03773584905660377 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"f1\", \"tab1\"), ColumnQualifierTuple(\"f1\", \"tab3\"), ), ( ColumnQualifierTuple(\"f1\", \"tab2\"), ColumnQualifierTuple(\"f1\", \"tab3\"), ), ])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.037267080745341616 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"id\", \"tab1\"), ColumnQualifierTuple(\"id\", \"tab3\"), ), ( ColumnQualifierTuple(\"id\", \"tab2\"), ColumnQualifierTuple(\"id\", \"tab3\"), ), ])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.037267080745341616 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"col1\", \"tab2\"), ColumnQualifierTuple(\"col2\", \"tab1\"), ), (ColumnQualifierTuple(\"*\", \"tab2\"), ColumnQualifierTuple(\"cnt\", \"tab1\")), ])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.037037037037037035 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"id\", \"tab1\"), ColumnQualifierTuple(\"id\", \"tab2\"), ), ( ColumnQualifierTuple(\"col1\", \"tab1\"), ColumnQualifierTuple(\"col1\", \"tab2\"), ), ])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.03636363636363636 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"col2\", \"tab4\"), ColumnQualifierTuple(\"col1\", \"tab1\"), ), ( ColumnQualifierTuple(\"col1\", \"tab3\"), ColumnQualifierTuple(\"col1\", \"tab1\"), ), ])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.03550295857988166 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"col1\", \"tab2\"), ColumnQualifierTuple(\"col1\", \"tab1\"), ), ( ColumnQualifierTuple(\"col2\", \"tab2\"), ColumnQualifierTuple(\"col2\", \"tab1\"), ), ])", + "got": "[(\"col1\", \"tab2\"), (\"col2\", \"tab2\")])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3645320197044335 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"col1\", \"tab2\"), ColumnQualifierTuple(\"col1\", \"tab1\"), ), ( ColumnQualifierTuple(\"col3\", \"tab2\"), ColumnQualifierTuple(\"col3\", \"tab1\"), ), ])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.03550295857988166 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"col2\", \"tab2\"), ColumnQualifierTuple(\"col2\", \"tab1\"), ), ( ColumnQualifierTuple(\"col3\", \"tab2\"), ColumnQualifierTuple(\"col3\", \"tab1\"), ), ])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.03550295857988166 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"col1\", \"tab2\"), ColumnQualifierTuple(\"col3\", \"tab1\"), ), ( ColumnQualifierTuple(\"col2\", \"tab2\"), ColumnQualifierTuple(\"col3\", \"tab1\"), ), ])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.03550295857988166 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"col1\", \"tab1\"), ColumnQualifierTuple(\"col1\", \"tab3\"), ), ( ColumnQualifierTuple(\"col1\", \"tab2\"), ColumnQualifierTuple(\"col1\", \"tab3\"), ), ])", + "got": "[(ColumnQualifierTuple(\"col1\", \"tab1\")), (ColumnQualifierTuple(\"col1\", \"tab2\"))])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6558704453441295 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"col1\", \"tab2\"), ColumnQualifierTuple(\"col1\", \"tab1\"), ), ( ColumnQualifierTuple(\"col2\", \"tab3\"), ColumnQualifierTuple(\"col2\", \"tab1\"), ), ])", + "got": "\"\"\"tab2.col1 -> tab1.col1\"\"\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.18461538461538463 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"col1\", \"tab2\"), ColumnQualifierTuple(\"col3\", \"tab1\"), ), ( ColumnQualifierTuple(\"col2\", \"tab3\"), ColumnQualifierTuple(\"col4\", \"tab1\"), ), ])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.03550295857988166 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"col1\", \"tab2\"), ColumnQualifierTuple(\"col2\", \"tab1\"), ), ( ColumnQualifierTuple(\"col1\", \"tab2\"), ColumnQualifierTuple(\"col3\", \"tab1\"), ), ])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.03550295857988166 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"col1\", \"tab2\"), ColumnQualifierTuple(\"col1\", \"tab1\"), ), ( ColumnQualifierTuple(\"col1\", \"tab3\"), ColumnQualifierTuple(\"col1\", \"tab1\"), ), ])", + "got": "[(\"col1\", \"tab2\"), (\"col1\", \"tab3\")])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3645320197044335 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"col1\", \"tab2\"), ColumnQualifierTuple(\"flag\", \"tab1\"), ), ( ColumnQualifierTuple(\"col2\", \"tab2\"), ColumnQualifierTuple(\"flag\", \"tab1\"), ), ])", + "got": "[ColumnQualifierTuple(\"flag\", \"tab1\")])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3804878048780488 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"col1\", \"tab2\"), ColumnQualifierTuple(\"rnum\", \"tab1\"), ), ( ColumnQualifierTuple(\"col2\", \"tab2\"), ColumnQualifierTuple(\"rnum\", \"tab1\"), ), ])", + "got": "[ColumnQualifierTuple(\"col1\", \"tab2\"), ColumnQualifierTuple(\"rnum\", \"tab1\")])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6337448559670782 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"col1\", \"tab1\"), ColumnQualifierTuple(\"col1\", \"tab2\"), ), ( ColumnQualifierTuple(\"col1\", \"tab1\"), ColumnQualifierTuple(\"col2\", \"tab2\"), ), ])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.03550295857988166 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"Name\", \"Employee\"), ColumnQualifierTuple(\"Name\", \"foo\"), ), ( ColumnQualifierTuple(\"DOB\", \"Employee\"), ColumnQualifierTuple(\"DOB\", \"foo\"), ), ])", + "got": "[], dialect=dialect)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.10526315789473684 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"col1\", \"tbl\"), ColumnQualifierTuple(\"col1\", \"my_view\"), ), ( ColumnQualifierTuple(\"col2\", \"tbl\"), ColumnQualifierTuple(\"col2\", \"my_view\"), ), ])", + "got": "[(ColumnQualifierTuple(\"col1\", \"tbl\"), ColumnQualifierTuple(\"col1\", \"my_view\"))])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6454183266932271 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"price\", \"tab1\"), ColumnQualifierTuple(\"price\", \"tab1\"), ), ( ColumnQualifierTuple(\"price\", \"tab2\"), ColumnQualifierTuple(\"price\", \"tab1\"), ), ])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.03468208092485549 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"col1\", \"tab2\"), ColumnQualifierTuple(\"max(col1)\", \"tab1\"), ), ( ColumnQualifierTuple(\"*\", \"tab2\"), ColumnQualifierTuple(\"count(*)\", \"tab1\"), ), ])", + "got": "\"\"\"tab1.col1 -> tab2.col1\"\"\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.1890547263681592 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"col1\", \"bar\"), ColumnQualifierTuple(\"sum_result\", \"foo\"), ), ( ColumnQualifierTuple(\"col2\", \"bar\"), ColumnQualifierTuple(\"sum_result\", \"foo\"), ), ])", + "got": "[()])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.055865921787709494 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"col1\", \"tab2\"), ColumnQualifierTuple(\"max(col1)\", \"tab1\"), ), ( ColumnQualifierTuple(\"col2\", \"tab2\"), ColumnQualifierTuple(\"max(col2)\", \"tab1\"), ), ])", + "got": "[(\"max(col1)\", \"max(col1)\",), (\"max(col2)\", \"max(col2)\",)])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4085106382978723 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"col1\", \"tbl\"), ColumnQualifierTuple(\"random1\", \"my_view\"), ), ( ColumnQualifierTuple(\"col2\", \"tbl\"), ColumnQualifierTuple(\"random2\", \"my_view\"), ), ])", + "got": "\"tbl.col1 tbl.col2\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.15306122448979592 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"col1\", \"tab2\"), ColumnQualifierTuple(\"col1 + col2\", \"tab1\"), ), ( ColumnQualifierTuple(\"col2\", \"tab2\"), ColumnQualifierTuple(\"col1 + col2\", \"tab1\"), ), ])", + "got": "\"\"\"tab2.col1 tab2.col2 tab1.col1+tab2.col2\"\"\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.18584070796460178 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"col1\", \"src_tbl\"), ColumnQualifierTuple(\"random1\", \"tgt_tbl\"), ), ( ColumnQualifierTuple(\"col2\", \"src_tbl\"), ColumnQualifierTuple(\"random2\", \"tgt_tbl\"), ), ])", + "got": "[(\"col1\", \"src_tbl\"), (\"col2\", \"src_tbl\")])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3788546255506608 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"col1\", \"tab2\"), ColumnQualifierTuple(\"(col1 + col2)\", \"tab1\"), ), ( ColumnQualifierTuple(\"col2\", \"tab2\"), ColumnQualifierTuple(\"(col1 + col2)\", \"tab1\"), ), ])", + "got": "[(\"col1\", \"tab2\"), (\"col2\", \"tab2\")])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.334841628959276 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"a\", \"db.tbl_x\"), ColumnQualifierTuple(\"a\", \".test_v\"), ), ( ColumnQualifierTuple(\"b\", \"db.tbl_x\"), ColumnQualifierTuple(\"b\", \".test_v\"), ), ])", + "got": "provider)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.06153846153846154 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"a\", \"db.tbl_x\"), ColumnQualifierTuple(\"a\", \".test_v\"), ), ( ColumnQualifierTuple(\"h\", \"db.tbl_y\"), ColumnQualifierTuple(\"h\", \".test_v\"), ), ])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.031746031746031744 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"id\", \"db.tbl_x\"), ColumnQualifierTuple(\"id\", \".test_v\"), ), ( ColumnQualifierTuple(\"b\", \"db.tbl_x\"), ColumnQualifierTuple(\"b\", \".test_v\"), ), ])", + "got": "provider)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.06091370558375635 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"col1\", \"src_tbl_new\"), ColumnQualifierTuple(\"random1\", \"tgt_tbl\"), ), ( ColumnQualifierTuple(\"col2\", \"src_tbl_new\"), ColumnQualifierTuple(\"random2\", \"tgt_tbl\"), ), ])", + "got": "[(\"col1\", \"src_tbl_new\"), (\"col2\", \"src_tbl_new\")])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.41975308641975306 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"alias1\", None), ColumnQualifierTuple(\"name\", \"public.tgt_tbl1\"), ), ( ColumnQualifierTuple(\"email\", \"public.src_tbl2\"), ColumnQualifierTuple(\"name\", \"public.tgt_tbl1\"), ), ])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.029556650246305417 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"col1\", \"dataset.tab1\"), ColumnQualifierTuple(\"col1\", \"dataset.target\"), ), ( ColumnQualifierTuple(\"col1\", \"dataset.tab2\"), ColumnQualifierTuple(\"col1\", \"dataset.target\"), ), ])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.02926829268292683 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"alias1\", \"public.src_tbl1\"), ColumnQualifierTuple(\"name\", \"public.tgt_tbl1\"), ), ( ColumnQualifierTuple(\"b\", \"public.src_tbl1\"), ColumnQualifierTuple(\"name\", \"public.tgt_tbl1\"), ), ])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.02830188679245283 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"id\", \"public.src_tbl1\"), ColumnQualifierTuple(\"id\", \"public.tgt_tbl1\"), ), ( ColumnQualifierTuple(\"id\", \"public.src_tbl1\"), ColumnQualifierTuple(\"id_original\", \"public.tgt_tbl1\"), ), ])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.028037383177570093 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"name\", \"public.src_tbl1\"), ColumnQualifierTuple(\"name\", \"public.tgt_tbl1\"), ), ( ColumnQualifierTuple(\"email\", \"public.src_tbl2\"), ColumnQualifierTuple(\"email\", \"public.tgt_tbl1\"), ), ])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.027906976744186046 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"name\", \"public.src_tbl1\"), ColumnQualifierTuple(\"name\", \"public.tgt_tbl1\"), ), ( ColumnQualifierTuple(\"address\", \"public.src_tbl1\"), ColumnQualifierTuple(\"email\", \"public.tgt_tbl1\"), ), ])", + "got": "\"\"\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "reata/sqllineage", + "expected": "[ (ColumnQualifierTuple(\"a\", \"tab1\"), ColumnQualifierTuple(\"a\", \"tab2\")), (ColumnQualifierTuple(\"b\", \"tab1\"), ColumnQualifierTuple(\"b_max\", \"tab2\")), (ColumnQualifierTuple(\"b\", \"tab1\"), ColumnQualifierTuple(\"b_cnt\", \"tab2\")), ])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.025974025974025976 + }, + { + "repo": "reata/sqllineage", + "expected": "[ (ColumnQualifierTuple(\"id\", \"tab1\"), ColumnQualifierTuple(\"id\", \"tab3\")), ( ColumnQualifierTuple(\"name\", \"tab1\"), ColumnQualifierTuple(\"name1\", \"tab3\"), ), ( ColumnQualifierTuple(\"name\", \"tab2\"), ColumnQualifierTuple(\"name2\", \"tab3\"), ), ])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.024489795918367346 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"col2\", \"tab4\"), ColumnQualifierTuple(\"col1\", \"tab1\"), ), ( ColumnQualifierTuple(\"col1\", \"tab3\"), ColumnQualifierTuple(\"col1\", \"tab1\"), ), ( ColumnQualifierTuple(\"col1\", \"tab5\"), ColumnQualifierTuple(\"col1\", \"tab1\"), ), ])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.024 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"col1\", \"tab2\"), ColumnQualifierTuple(\"col1\", \"tab1\"), ), ( ColumnQualifierTuple(\"col2\", \"tab2\"), ColumnQualifierTuple(\"col2\", \"tab1\"), ), ( ColumnQualifierTuple(\"col3\", \"tab2\"), ColumnQualifierTuple(\"col2\", \"tab1\"), ), ])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.024 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"col1\", \"tab2\"), ColumnQualifierTuple(\"col1\", \"tab1\"), ), ( ColumnQualifierTuple(\"col2\", \"tab2\"), ColumnQualifierTuple(\"col1\", \"tab1\"), ), ( ColumnQualifierTuple(\"col3\", \"tab2\"), ColumnQualifierTuple(\"col1\", \"tab1\"), ), ])", + "got": "[(ColumnQualifierTuple(\"col1\", \"tab2\"), ColumnQualifierTuple(\"col1\", \"tab1\"))])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.018404907975460124 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"a\", \"db1.table1\"), ColumnQualifierTuple(\"a\", \"db1.foo\"), ), ( ColumnQualifierTuple(\"b\", \"db1.table1\"), ColumnQualifierTuple(\"b\", \"db1.foo\"), ), ( ColumnQualifierTuple(\"c\", \"db1.table1\"), ColumnQualifierTuple(\"c\", \"db1.foo\"), ), ])", + "got": "provider)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"id\", \"db.tbl_y\"), ColumnQualifierTuple(\"id\", \"db.tbl_x2\"), ), ( ColumnQualifierTuple(\"h\", \"db.tbl_y\"), ColumnQualifierTuple(\"a\", \"db.tbl_x2\"), ), ( ColumnQualifierTuple(\"i\", \"db.tbl_y\"), ColumnQualifierTuple(\"b\", \"db.tbl_x2\"), ), ])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.022988505747126436 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"id\", \"db.tbl_x\"), ColumnQualifierTuple(\"id\", \".test_v\"), ), ( ColumnQualifierTuple(\"a\", \"db.tbl_x\"), ColumnQualifierTuple(\"a\", \".test_v\"), ), ( ColumnQualifierTuple(\"b\", \"db.tbl_x\"), ColumnQualifierTuple(\"b\", \".test_v\"), ), ])", + "got": "provider)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.006944444444444444 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"a\", \"db.tbl_x\"), ColumnQualifierTuple(\"a\", \".test_v\"), ), ( ColumnQualifierTuple(\"id\", \"db.tbl_y\"), ColumnQualifierTuple(\"id\", \".test_v\"), ), ( ColumnQualifierTuple(\"h\", \"db.tbl_y\"), ColumnQualifierTuple(\"h\", \".test_v\"), ), ])", + "got": "provider)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.006944444444444444 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"a\", \"db.tbl_x\"), ColumnQualifierTuple(\"a\", \".test_v\"), ), ( ColumnQualifierTuple(\"id\", \"db.tbl_x\"), ColumnQualifierTuple(\"id\", \".test_v\"), ), ( ColumnQualifierTuple(\"h\", \"db.tbl_y\"), ColumnQualifierTuple(\"h\", \".test_v\"), ), ])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.02127659574468085 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"id\", \"public.src_tbl1\"), ColumnQualifierTuple(\"name\", \"public.tgt_tbl1\"), ), ( ColumnQualifierTuple(\"a\", \"public.src_tbl1\"), ColumnQualifierTuple(\"name\", \"public.tgt_tbl1\"), ), ( ColumnQualifierTuple(\"b\", \"public.src_tbl1\"), ColumnQualifierTuple(\"name\", \"public.tgt_tbl1\"), ), ])", + "got": "[(ColumnQualifierTuple(\"name\", \"public.tgt_tbl1\"))])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2801120448179272 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"id\", \"public.src_tbl1\"), ColumnQualifierTuple(\"name\", \"public.tgt_tbl1\"), ), ( ColumnQualifierTuple(\"name\", \"public.src_tbl1\"), ColumnQualifierTuple(\"name\", \"public.tgt_tbl1\"), ), ( ColumnQualifierTuple(\"email\", \"public.src_tbl2\"), ColumnQualifierTuple(\"name\", \"public.tgt_tbl1\"), ), ])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.01904761904761905 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"name\", \"public.src_tbl1\"), ColumnQualifierTuple(\"user_name\", \"public.tgt_tbl1\"), ), ( ColumnQualifierTuple(\"name\", \"public.src_tbl1\"), ColumnQualifierTuple(\"id\", \"public.tgt_tbl1\"), ), ( ColumnQualifierTuple(\"email\", \"public.src_tbl1\"), ColumnQualifierTuple(\"id\", \"public.tgt_tbl1\"), ), ])", + "got": "provider)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.012345679012345678 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"name\", \"public.src_tbl1\"), ColumnQualifierTuple(\"name\", \"public.tgt_tbl1\"), ), ( ColumnQualifierTuple(\"name\", \"public.src_tbl1\"), ColumnQualifierTuple(\"email\", \"public.tgt_tbl1\"), ), ( ColumnQualifierTuple(\"email\", \"public.src_tbl1\"), ColumnQualifierTuple(\"email\", \"public.tgt_tbl1\"), ), ])", + "got": "[], test_sqlparse=False)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.011764705882352941 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"name\", \"public.src_tbl1\"), ColumnQualifierTuple(\"user_name\", \"public.tgt_tbl1\"), ), ( ColumnQualifierTuple(\"user_name\", \"public.src_tbl1\"), ColumnQualifierTuple(\"id\", \"public.tgt_tbl1\"), ), ( ColumnQualifierTuple(\"email\", \"public.src_tbl1\"), ColumnQualifierTuple(\"id\", \"public.tgt_tbl1\"), ), ])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.018575851393188854 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"a\", \"db1.table1\"), ColumnQualifierTuple(\"a\", \"db.tbl\"), ), ( ColumnQualifierTuple(\"b\", \"db1.table1\"), ColumnQualifierTuple(\"x\", \"db.tbl\"), ), ( ColumnQualifierTuple(\"h\", \"db2.table2\"), ColumnQualifierTuple(\"h\", \"db.tbl\"), ), ( ColumnQualifierTuple(\"j\", \"db2.table2\"), ColumnQualifierTuple(\"y\", \"db.tbl\"), ), ])", + "got": "provider)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"a\", \"db1.table1\"), ColumnQualifierTuple(\"x\", \"db.tbl\"), ), ( ColumnQualifierTuple(\"b\", \"db1.table1\"), ColumnQualifierTuple(\"b\", \"db.tbl\"), ), ( ColumnQualifierTuple(\"pk\", \"db3.table3\"), ColumnQualifierTuple(\"y\", \"db.tbl\"), ), ( ColumnQualifierTuple(\"q\", \"db3.table3\"), ColumnQualifierTuple(\"q\", \"db.tbl\"), ), ])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.01764705882352941 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"col0\", \"tab2\"), ColumnQualifierTuple(\"col0\", \"tab1\"), ), ( ColumnQualifierTuple(\"col1\", \"tab2\"), ColumnQualifierTuple(\"rnum\", \"tab1\"), ), ( ColumnQualifierTuple(\"col2\", \"tab2\"), ColumnQualifierTuple(\"rnum\", \"tab1\"), ), ( ColumnQualifierTuple(\"col3\", \"tab2\"), ColumnQualifierTuple(\"rnum\", \"tab1\"), ), ( ColumnQualifierTuple(\"col4\", \"tab2\"), ColumnQualifierTuple(\"col4\", \"tab1\"), ), ])", + "got": "[(\"col0\", \"tab2\"), (\"col4\", \"tab2\")])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.16591928251121077 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"id\", \"db1.table1\"), ColumnQualifierTuple(\"id\", \"db.tbl\"), ), ( ColumnQualifierTuple(\"a\", \"db1.table1\"), ColumnQualifierTuple(\"x\", \"db.tbl\"), ), ( ColumnQualifierTuple(\"b\", \"db1.table1\"), ColumnQualifierTuple(\"b\", \"db.tbl\"), ), ( ColumnQualifierTuple(\"h\", \"db2.table2\"), ColumnQualifierTuple(\"h\", \"db.tbl\"), ), ( ColumnQualifierTuple(\"i\", \"db2.table2\"), ColumnQualifierTuple(\"y\", \"db.tbl\"), ), ])", + "got": "provider)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"col1\", \"tab2\"), ColumnQualifierTuple(\"a.col1 + a.col2 + a.col3 + a.col4\", \"tab1\"), ), ( ColumnQualifierTuple(\"col2\", \"tab2\"), ColumnQualifierTuple(\"a.col1 + a.col2 + a.col3 + a.col4\", \"tab1\"), ), ( ColumnQualifierTuple(\"col3\", \"tab2\"), ColumnQualifierTuple(\"a.col1 + a.col2 + a.col3 + a.col4\", \"tab1\"), ), ( ColumnQualifierTuple(\"col4\", \"tab2\"), ColumnQualifierTuple(\"a.col1 + a.col2 + a.col3 + a.col4\", \"tab1\"), ), ])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.013422818791946308 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"a\", \"db.tbl_x\"), ColumnQualifierTuple(\"a\", \".test_v\"), ), ( ColumnQualifierTuple(\"b\", \"db.tbl_x\"), ColumnQualifierTuple(\"b\", \".test_v\"), ), ( ColumnQualifierTuple(\"id\", \"db.tbl_x\"), ColumnQualifierTuple(\"id\", \".test_v\"), ), ( ColumnQualifierTuple(\"h\", \"db.tbl_y\"), ColumnQualifierTuple(\"h\", \".test_v\"), ), ( ColumnQualifierTuple(\"i\", \"db.tbl_y\"), ColumnQualifierTuple(\"i\", \".test_v\"), ), ])", + "got": "provider)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.00425531914893617 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"a\", \"db.tbl_x\"), ColumnQualifierTuple(\"a\", \"db.tbl\"), ), ( ColumnQualifierTuple(\"b\", \"db.tbl_x\"), ColumnQualifierTuple(\"b\", \"db.tbl\"), ), ( ColumnQualifierTuple(\"h\", \"db.tbl_y\"), ColumnQualifierTuple(\"h\", \"db.tbl\"), ), ( ColumnQualifierTuple(\"pk\", \"db.tbl_z\"), ColumnQualifierTuple(\"pk\", \"db.tbl\"), ), ( ColumnQualifierTuple(\"s\", \"db.tbl_z\"), ColumnQualifierTuple(\"s\", \"db.tbl\"), ), ( ColumnQualifierTuple(\"t\", \"db.tbl_z\"), ColumnQualifierTuple(\"t\", \"db.tbl\"), ), ])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.012121212121212121 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"a\", \"db1.table1\"), ColumnQualifierTuple(\"a\", \"db.tbl\"), ), ( ColumnQualifierTuple(\"b\", \"db1.table1\"), ColumnQualifierTuple(\"x\", \"db.tbl\"), ), ( ColumnQualifierTuple(\"h\", \"db2.table2\"), ColumnQualifierTuple(\"y\", \"db.tbl\"), ), ( ColumnQualifierTuple(\"i\", \"db2.table2\"), ColumnQualifierTuple(\"i\", \"db.tbl\"), ), ( ColumnQualifierTuple(\"p\", \"db3.table3\"), ColumnQualifierTuple(\"p\", \"db.tbl\"), ), ( ColumnQualifierTuple(\"q\", \"db3.table3\"), ColumnQualifierTuple(\"z\", \"db.tbl\"), ), ])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.011881188118811881 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"a\", \"db1.table1\"), ColumnQualifierTuple(\"a\", \"db.tbl\"), ), ( ColumnQualifierTuple(\"b\", \"db1.table1\"), ColumnQualifierTuple(\"x\", \"db.tbl\"), ), ( ColumnQualifierTuple(\"h\", \"db2.table2\"), ColumnQualifierTuple(\"h\", \"db.tbl\"), ), ( ColumnQualifierTuple(\"i\", \"db2.table2\"), ColumnQualifierTuple(\"y\", \"db.tbl\"), ), ( ColumnQualifierTuple(\"p\", \"db3.table3\"), ColumnQualifierTuple(\"p\", \"db.tbl\"), ), ( ColumnQualifierTuple(\"pk\", \"db3.table3\"), ColumnQualifierTuple(\"z\", \"db.tbl\"), ), ])", + "got": "provider)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"id\", \"db1.table1\"), ColumnQualifierTuple(\"id\", \"db.tbl\"), ), ( ColumnQualifierTuple(\"a\", \"db1.table1\"), ColumnQualifierTuple(\"x\", \"db.tbl\"), ), ( ColumnQualifierTuple(\"h\", \"db2.table2\"), ColumnQualifierTuple(\"y\", \"db.tbl\"), ), ( ColumnQualifierTuple(\"i\", \"db2.table2\"), ColumnQualifierTuple(\"i\", \"db.tbl\"), ), ( ColumnQualifierTuple(\"p\", \"db3.table3\"), ColumnQualifierTuple(\"p\", \"db.tbl\"), ), ( ColumnQualifierTuple(\"q\", \"db3.table3\"), ColumnQualifierTuple(\"z\", \"db.tbl\"), ), ])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.011834319526627219 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"a\", \"public.src_tbl1\"), ColumnQualifierTuple(\"id\", \"public.tgt_tbl1\"), ), ( ColumnQualifierTuple(\"b\", \"public.src_tbl1\"), ColumnQualifierTuple(\"id\", \"public.tgt_tbl1\"), ), ( ColumnQualifierTuple(\"c\", \"public.src_tbl1\"), ColumnQualifierTuple(\"id\", \"public.tgt_tbl1\"), ), ( ColumnQualifierTuple(\"id\", \"public.src_tbl1\"), ColumnQualifierTuple(\"id\", \"public.tgt_tbl1\"), ), ( ColumnQualifierTuple(\"id\", \"public.src_tbl1\"), ColumnQualifierTuple(\"id_original\", \"public.tgt_tbl1\"), ), ])", + "got": "[], test_sqlparse=False)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.007561436672967864 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"a\", \"db.tbl_x\"), ColumnQualifierTuple(\"a\", \".test_v\"), ), ( ColumnQualifierTuple(\"b\", \"db.tbl_x\"), ColumnQualifierTuple(\"b\", \".test_v\"), ), ( ColumnQualifierTuple(\"id\", \"db.tbl_x\"), ColumnQualifierTuple(\"id\", \".test_v\"), ), ( ColumnQualifierTuple(\"pk\", \"db.tbl_z\"), ColumnQualifierTuple(\"pk\", \".test_v\"), ), ( ColumnQualifierTuple(\"s\", \"db.tbl_z\"), ColumnQualifierTuple(\"s\", \".test_v\"), ), ( ColumnQualifierTuple(\"t\", \"db.tbl_z\"), ColumnQualifierTuple(\"t\", \".test_v\"), ), ])", + "got": "provider)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.003552397868561279 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"id\", \"db.tbl_x\"), ColumnQualifierTuple(\"id\", \".test_v\"), ), ( ColumnQualifierTuple(\"a\", \"db.tbl_x\"), ColumnQualifierTuple(\"a\", \".test_v\"), ), ( ColumnQualifierTuple(\"b\", \"db.tbl_x\"), ColumnQualifierTuple(\"b\", \".test_v\"), ), ( ColumnQualifierTuple(\"id\", \"db.tbl_y\"), ColumnQualifierTuple(\"id\", \".test_v\"), ), ( ColumnQualifierTuple(\"h\", \"db.tbl_y\"), ColumnQualifierTuple(\"a\", \".test_v\"), ), ( ColumnQualifierTuple(\"i\", \"db.tbl_y\"), ColumnQualifierTuple(\"b\", \".test_v\"), ), ])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.010771992818671455 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"id\", \"db.tbl_x\"), ColumnQualifierTuple(\"id\", \".test_v\"), ), ( ColumnQualifierTuple(\"a\", \"db.tbl_x\"), ColumnQualifierTuple(\"a\", \".test_v\"), ), ( ColumnQualifierTuple(\"b\", \"db.tbl_x\"), ColumnQualifierTuple(\"b\", \".test_v\"), ), ( ColumnQualifierTuple(\"id\", \"db.tbl_x1\"), ColumnQualifierTuple(\"id\", \".test_v\"), ), ( ColumnQualifierTuple(\"a\", \"db.tbl_x1\"), ColumnQualifierTuple(\"a\", \".test_v\"), ), ( ColumnQualifierTuple(\"b\", \"db.tbl_x1\"), ColumnQualifierTuple(\"b\", \".test_v\"), ), ])", + "got": "[(\"a\", \".test_v\")])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.08547008547008547 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"a\", \"db1.table1\"), ColumnQualifierTuple(\"a\", \"db.tbl\"), ), ( ColumnQualifierTuple(\"b\", \"db1.table1\"), ColumnQualifierTuple(\"x\", \"db.tbl\"), ), ( ColumnQualifierTuple(\"h\", \"db2.table2\"), ColumnQualifierTuple(\"h\", \"db.tbl\"), ), ( ColumnQualifierTuple(\"i\", \"db2.table2\"), ColumnQualifierTuple(\"y\", \"db.tbl\"), ), ( ColumnQualifierTuple(\"p\", \"db3.table3\"), ColumnQualifierTuple(\"p\", \"db.tbl\"), ), ( ColumnQualifierTuple(\"q\", \"db3.table3\"), ColumnQualifierTuple(\"q\", \"db.tbl\"), ), ( ColumnQualifierTuple(\"pk\", \"db3.table3\"), ColumnQualifierTuple(\"z\", \"db.tbl\"), ), ])", + "got": "provider)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "2", + "got": "2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "0", + "got": "0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "1", + "got": "1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "m", + "got": "m", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "6", + "got": "6", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "4", + "got": "4", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "3", + "got": "3", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "x", + "got": "x", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"\"", + "got": "\"\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "[]", + "got": "[]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "ol", + "got": "ol", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "h1", + "got": "h1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "16", + "got": "16", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "200", + "got": "200", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "302", + "got": "403", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "redimp/otterwiki", + "expected": "403", + "got": "302", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "redimp/otterwiki", + "expected": "str", + "got": "str", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"/\"", + "got": "\"/-/index\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "redimp/otterwiki", + "expected": "404", + "got": "404", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "400", + "got": "404", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "redimp/otterwiki", + "expected": "500", + "got": "500", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "401", + "got": "401", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "pre", + "got": "pre", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"\ud83d\ude0a\"", + "got": "\"\ud83d\ude0a\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "html", + "got": "html", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "None", + "got": "None", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "diff", + "got": "diff", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "True", + "got": "True", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "ftoc", + "got": "ftoc", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "soup", + "got": "soup.find(\"meta\", property=\"og:title\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.19047619047619047 + }, + { + "repo": "redimp/otterwiki", + "expected": "body", + "got": "body", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "slug", + "got": "\"/{}\".format(slug)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"de\"", + "got": "\"de\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"42\"", + "got": "42", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "redimp/otterwiki", + "expected": "text", + "got": "text", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "code", + "got": "code.find('span', {'class': 'token keyword'})", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.16326530612244897 + }, + { + "repo": "redimp/otterwiki", + "expected": "type", + "got": "type", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "files", + "got": "files", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "bytes", + "got": "bytes", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "[\"a\"]", + "got": "[\".\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "redimp/otterwiki", + "expected": "[\"c\"]", + "got": "[\"b/c\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "redimp/otterwiki", + "expected": "False", + "got": "False", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "html0", + "got": "html0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "html1", + "got": "html1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "title", + "got": "title", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "0o600", + "got": "0o400", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"abc\"", + "got": "\"abc\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"a-b\"", + "got": "\"a-b\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"a/b\"", + "got": "\"a/b\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "p16_2", + "got": "p16_2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "[\"ok\"]", + "got": "[]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"Home\"", + "got": "\"home\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"home\"", + "got": "\"home.md\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "redimp/otterwiki", + "expected": "result", + "got": "result", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "a_html", + "got": "a_html", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "content", + "got": "content", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "message", + "got": "message", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"/Home\"", + "got": "\"/home.md\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7058823529411765 + }, + { + "repo": "redimp/otterwiki", + "expected": "req_ctx", + "got": "not create_app_raw_filenames.config[\"RETAIN_PAGE_NAME_CASE\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.08955223880597014 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"/home\"", + "got": "\"/home\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"allow\"", + "got": "\"allow\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "pagename", + "got": "\"/{}\".format(pagename)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5333333333333333 + }, + { + "repo": "redimp/otterwiki", + "expected": "filename", + "got": "filename in storage.list()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 0.47058823529411764 + }, + { + "repo": "redimp/otterwiki", + "expected": "content1", + "got": "content1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "content2", + "got": "content2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "pre.text", + "got": "pre.text", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "img_html", + "got": "img_html", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"1.0KiB\"", + "got": "\"1.0 KB\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.875 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"1.0MiB\"", + "got": "\"1.0MiB\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"4.0GiB\"", + "got": "\"4.0GiB\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"2 secs\"", + "got": "\"2.000000 seconds\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "redimp/otterwiki", + "expected": "author[0]", + "got": "\"Example Author\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.48 + }, + { + "repo": "redimp/otterwiki", + "expected": "metadata2", + "got": "metadata2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "revision1", + "got": "revision1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"home.md\"", + "got": "\"Home.md\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"Home.md\"", + "got": "\"Home.md\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"hOme.md\"", + "got": "\"hOme.md\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "soup.text", + "got": "soup.text", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"3 weeks\"", + "got": "\"21 days\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "redimp/otterwiki", + "expected": "SystemExit", + "got": "SystemExit", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "content_rb", + "got": "content_rb", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "content[0]", + "got": "content[0]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "[\"c\", \"d\"]", + "got": "[\"c\", \"d\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "create_app", + "got": "req_ctx", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.35294117647058826 + }, + { + "repo": "redimp/otterwiki", + "expected": "[\"a\", \"b\"]", + "got": "(\"a\", \"b\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"Test User\"", + "got": "\"John Doe\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.38095238095238093 + }, + { + "repo": "redimp/otterwiki", + "expected": "rv.location", + "got": "rv.headers[\"Location\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6060606060606061 + }, + { + "repo": "redimp/otterwiki", + "expected": "directories", + "got": "directories", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "html.lower()", + "got": "html", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "redimp/otterwiki", + "expected": "preview_soup", + "got": "whitespace.sub(\"\", str(preview_soup))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4897959183673469 + }, + { + "repo": "redimp/otterwiki", + "expected": "preview_html", + "got": "preview_html", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "onclick_attr", + "got": "onclick_attr", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "admin_emails", + "got": "admin_emails", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"draft saved\"", + "got": "\"success\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "redimp/otterwiki", + "expected": "[test_string]", + "got": "[test_string]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "response_text", + "got": "response_text", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "'

test

'", + "got": "'

test

'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"another/test\"", + "got": "\"Another/Test\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"Another/Test\"", + "got": "\"Another/Test\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "rv.status_code", + "got": "rv.status_code", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "get_header(md)", + "got": "get_header(md)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"Hello, World!\"", + "got": "\"Hello, World!\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"indexpathtest\"", + "got": "\"IndexPathTest\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"", + "got": "\"javascript:alert(document.cookie)\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.05555555555555555 + }, + { + "repo": "redimp/otterwiki", + "expected": "rv.data.decode()", + "got": "rv.data.decode()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"<script>\"", + "got": "\"\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "redimp/otterwiki", + "expected": "expected_content", + "got": "expected_content", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "test_description", + "got": "test_description", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "[\"a\", \"b \", \"c\"]", + "got": "[\"a\", \"b /c\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8275862068965517 + }, + { + "repo": "redimp/otterwiki", + "expected": "response.location", + "got": "response.headers[\"Location\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7111111111111111 + }, + { + "repo": "redimp/otterwiki", + "expected": "str(preview_soup)", + "got": "str(preview_soup)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"aaa bbb ddd ccc\"", + "got": "\"aaa bbb ddd ccc\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "'italic'", + "got": "'italic'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "sidebar_shortcuts", + "got": "sidebar_shortcuts", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"Subspace/Example\"", + "got": "\"Example\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "redimp/otterwiki", + "expected": "soup and soup.text", + "got": "soup", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "redimp/otterwiki", + "expected": "html.count(\"
  • bold'", + "got": "'bold'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "create_app_raw_filenames", + "got": "\"example\" == get_pagename(\"example.md\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "redimp/otterwiki", + "expected": "storage.exists(filename)", + "got": "not storage.exists(filename)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9230769230769231 + }, + { + "repo": "redimp/otterwiki", + "expected": "test_data['private_key']", + "got": "test_data['private_key']", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "deserialize(serialize(s))", + "got": "deserialize(serialize(s))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "get_pagename(\"two words\")", + "got": "get_pagename(\"two words.md\", full=True)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.78125 + }, + { + "repo": "redimp/otterwiki", + "expected": "get_pagename(\"Two words\")", + "got": "get_pagename(\"two words.md\", full=True)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.75 + }, + { + "repo": "redimp/otterwiki", + "expected": "call_args[1]['text_body']", + "got": "call_args[1]['body']", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "redimp/otterwiki", + "expected": "not flask.session.modified", + "got": "toast(test_string)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "redimp/otterwiki", + "expected": "not storage.exists(filename)", + "got": "not storage.exists(filename)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "response.data.decode().lower()", + "got": "response.data.decode()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8461538461538461 + }, + { + "repo": "redimp/otterwiki", + "expected": "isinstance(admin_emails, list)", + "got": "isinstance(admin_emails, list)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "get_page_directoryname(\"Home\")", + "got": "get_page_directoryname(\"\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9285714285714286 + }, + { + "repo": "redimp/otterwiki", + "expected": "data and data['preview_content']", + "got": "\"HTML\" in data['preview_content']", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.49122807017543857 + }, + { + "repo": "redimp/otterwiki", + "expected": "'application/xml; charset=utf-8'", + "got": "'text/xml'", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "redimp/otterwiki", + "expected": "get_page_directoryname(\"Sub/Dir\")", + "got": "get_page_directoryname(\"Sub\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9354838709677419 + }, + { + "repo": "redimp/otterwiki", + "expected": "os.path.exists(tmpdir.join(\"aa\"))", + "got": "tmpdir.join(path_a).exists()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5245901639344263 + }, + { + "repo": "redimp/otterwiki", + "expected": "get_page_directoryname(\"/Sub/Dir\")", + "got": "get_page_directoryname(\"Sub\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9206349206349206 + }, + { + "repo": "redimp/otterwiki", + "expected": "create_app.storage.exists(filename)", + "got": "True", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.10256410256410256 + }, + { + "repo": "redimp/otterwiki", + "expected": "check_password_hash(hash, password)", + "got": "check_password_hash(hash, password)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "os.path.exists(tmpdir.join(path_b))", + "got": "os.path.exists(tmpdir.join(\"aa\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8823529411764706 + }, + { + "repo": "redimp/otterwiki", + "expected": "os.path.exists(tmpdir.join(path_c))", + "got": "os.path.exists(tmpdir.join(path_c))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "storage.log(fail_on_git_error=False)", + "got": "storage.log(\"non-existent.md\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5454545454545454 + }, + { + "repo": "redimp/otterwiki", + "expected": "'<embed src=&", + "got": "'<embed src=&", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"<script>alert(1)</script>\"", + "got": "\"<script>alert(1)</script>\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "'home.md' in files or 'Home.md' in files", + "got": "\"Home\" in files", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.43636363636363634 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"Missing\" in html or \"../Missing\" in html", + "got": "\"../Home\" in html", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3448275862068966 + }, + { + "repo": "redimp/otterwiki", + "expected": "check_password_hash_backport(pwhash, passwd)", + "got": "check_password_hash_backport(pwhash, passwd)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "'PPPPP

    '", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.13333333333333333 + }, + { + "repo": "redimp/otterwiki", + "expected": "'{http://www.sitemaps.org/schemas/sitemap/0.9}urlset'", + "got": "'sitemapindex'", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3283582089552239 + }, + { + "repo": "redimp/otterwiki", + "expected": "'<div onload=&", + "got": "\"<div onload=&", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9411764705882353 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"", + "got": "\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\"", + "got": "\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca49f6a74d6d2a3a\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8396946564885496 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b\"", + "got": "\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.22727272727272727 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"c0b00e171401dfa2c70f2524fa977d66ead451ec9837543556fc66087b211646\"", + "got": "\"b000000000000000000000000000000000000000000000000000000000000000\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.12121212121212122 + }, + { + "repo": "redimp/otterwiki", + "expected": "[ 'Directory', 'Directory/Content', 'Example', 'Random page', ]", + "got": "sorted(pages)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.23684210526315788 + }, + { + "repo": "redimp/otterwiki", + "expected": "storage.store( filename, content=content, author=author, message=message )", + "got": "storage.store(filename, content=content)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7017543859649122 + }, + { + "repo": "redimp/otterwiki", + "expected": "storage.store( filename, content=content1, author=author, message=message1 )", + "got": "storage.store(filename, content=content1)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7008547008547008 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"test_subpage_links\" in html.lower() or \"test subpage links\" in html.lower()", + "got": "\"test_subpage_links\" in html.lower()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6428571428571429 + }, + { + "repo": "redimp/otterwiki", + "expected": "storage.store( filename1, content=content, author=author, message=\"operation 1\" )", + "got": "storage.store( filename1, content=content, author=author, message=\"Test commit\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8944099378881988 + }, + { + "repo": "redimp/otterwiki", + "expected": "storage.store( filename2, content=content, author=author, message=\"operation 2\" )", + "got": "storage.store( filename2, content=content, author=author, message=\"operation 2\" )", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"test_link_title_style\" in html.lower() or \"test link title style\" in html.lower()", + "got": "\"test_link_title_style\" in html.lower()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6446280991735537 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"test_case_sensitivity\" in html.lower() or \"test case sensitivity\" in html.lower()", + "got": "\"test_page_case\" not in html.lower()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4576271186440678 + }, + { + "repo": "redimp/otterwiki", + "expected": "storage.store( filename, content=content, author=author, message=message, mode=\"wb\" )", + "got": "storage.store(filename, content)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5470085470085471 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"test_links_with_anchors\" in html.lower() or \"test links with anchors\" in html.lower()", + "got": "\"test_links_with_anchors\" in html.lower()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6456692913385826 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"Header only\" not in html or pagepath not in html.split(\"Header only\")[0].split(\"<\")[-1]", + "got": "\"Page is empty\" not in html", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3130434782608696 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"test_broken_links_default\" in html.lower() or \"test broken links default\" in html.lower()", + "got": "\"Broken links\" in html", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.375 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"test_page_name_title_style\" in html.lower() or \"test page name title style\" in html.lower()", + "got": "\"test_page_name_title_style\" in html.lower()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6470588235294118 + }, + { + "repo": "redimp/otterwiki", + "expected": "create_app.storage.store( f\"{pagename}.md\", content=content, author=(\"John\", \"john@doe.com\"), message=f\"added {pagename}\", )", + "got": "save_shortcut(test_client, pagename, content)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3431952662721893 + }, + { + "repo": "redimp/otterwiki", + "expected": "SystemExit)", + "got": "Exception)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.38095238095238093 + }, + { + "repo": "redimp/otterwiki", + "expected": "ValueError)", + "got": "ValueError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "SerializeError)", + "got": "SerializeError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "gitstorage.StorageError)", + "got": "gitstorage.StorageNotFound)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7843137254901961 + }, + { + "repo": "redimp/otterwiki", + "expected": "gitstorage.StorageNotFound)", + "got": "gitstorage.GitError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6382978723404256 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"\")", + "got": "test_data['private_key'])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.07142857142857142 + }, + { + "repo": "redimp/otterwiki", + "expected": "test_data['private_key'])", + "got": "\"Test commit for auto-push\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.22641509433962265 + }, + { + "repo": "rubik/radon", + "expected": "2", + "got": "2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "3", + "got": "3", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "4", + "got": "4", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "0", + "got": "2", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "rubik/radon", + "expected": "1", + "got": "3", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "rubik/radon", + "expected": "b", + "got": "b", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "19", + "got": "19", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "-1", + "got": "10", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "rubik/radon", + "expected": "[]", + "got": "[]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "63", + "got": "100", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "rubik/radon", + "expected": "52", + "got": "10", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "rubik/radon", + "expected": "'B'", + "got": "'B'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "'Y'", + "got": "'Y'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "_md", + "got": "_md", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "'{}'", + "got": "'Config()'", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "rubik/radon", + "expected": "True", + "got": "fobj.name.endswith(__file__)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0625 + }, + { + "repo": "rubik/radon", + "expected": "diff", + "got": "diff", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "['-']", + "got": "['-']", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "False", + "got": "False", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "names", + "got": "names", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "result", + "got": "result", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "expected", + "got": "expected", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "v.blocks", + "got": "len(v.functions)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "rubik/radon", + "expected": "h.as_md()", + "got": "h.as_md()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "filenames", + "got": "filenames", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "c.__dict__", + "got": "not c.has('b')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "rubik/radon", + "expected": "[('-', 42)]", + "got": "[{'file-0': 0 ** 2}, {'file-1': 1 ** 2}, {'file-2': 2 ** 2}]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.22535211267605634 + }, + { + "repo": "rubik/radon", + "expected": "dict_result", + "got": "dict_result", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "'{\\'a\\': 2}'", + "got": "'{a: 2}'", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "rubik/radon", + "expected": "cli.Config()", + "got": "cli.Config()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "expected_str", + "got": "expected_str", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "expected_rank", + "got": "expected_rank", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "len(expected)", + "got": "3", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "rubik/radon", + "expected": "closure_names", + "got": "closure_names", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "expected_name", + "got": "expected_name", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "cli.Config(a=2)", + "got": "cli.Config(a=2)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "cli.Config(b=2)", + "got": "cli.Config(a=3)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8666666666666667 + }, + { + "repo": "rubik/radon", + "expected": "cli.Config(b=3)", + "got": "cli.Config(b=3)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "d2x_mock.called", + "got": "d2x_mock.called", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "expected_sorted", + "got": "expected_sorted", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "expected_result", + "got": "expected_result", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "expected_letter", + "got": "expected_letter", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "{'a': 2, 'b': 3}", + "got": "{'a': 2, 'b': 3}", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "fobj.read.called", + "got": "not fobj.read.called", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "rubik/radon", + "expected": "expected_average", + "got": "expected_average", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "number_of_blocks", + "got": "number_of_blocks", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "total_complexity", + "got": "total_complexity", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "Module(*expected)", + "got": "expected", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.64 + }, + { + "repo": "rubik/radon", + "expected": "methods_complexity", + "got": "methods_complexity", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "visitor.complexity", + "got": "visitor.complexity == total_complexity", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 0.6428571428571429 + }, + { + "repo": "rubik/radon", + "expected": "classes_complexity", + "got": "classes_complexity", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "dict(sample_results)", + "got": "sample_results", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8235294117647058 + }, + { + "repo": "rubik/radon", + "expected": "colors.color_enabled()", + "got": "colors.enabled", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7777777777777778 + }, + { + "repo": "rubik/radon", + "expected": "_is_python_file(target)", + "got": "any(target.endswith(f) for f in filenames)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.27692307692307694 + }, + { + "repo": "rubik/radon", + "expected": "dict(sample_results[1:])", + "got": "dict(sample_results[1:])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "expected_number_of_lines", + "got": "expected_number_of_lines", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "{'mi': 23.5, 'rank': 'A'}", + "got": "23.5", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.27586206896551724 + }, + { + "repo": "rubik/radon", + "expected": "not any(['error' in out])", + "got": "isinstance(out, str)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.26666666666666666 + }, + { + "repo": "rubik/radon", + "expected": "not colors.color_enabled()", + "got": "not colors.color_enabled()", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "[('-', {'error': 'mystr'})]", + "got": "[('-', TypeError('mystr'))]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7777777777777778 + }, + { + "repo": "rubik/radon", + "expected": "isinstance(h.results, list)", + "got": "all(isinstance(result, dict) for result in h.results)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.525 + }, + { + "repo": "rubik/radon", + "expected": "cli.Config(b=2, c=[], d=None)", + "got": "{'a': 'a', 'b': 'b', 'c': 'c'}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.13559322033898305 + }, + { + "repo": "rubik/radon", + "expected": "'{\"filename\": {\"complexity\": 2}}'", + "got": "json.dumps({'filename': {'complexity': 2}})", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7105263157894737 + }, + { + "repo": "rubik/radon", + "expected": "isinstance(h.run(), collections_abc.Iterator)", + "got": "isinstance(h.run(), list)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7142857142857143 + }, + { + "repo": "rubik/radon", + "expected": "isinstance(results, collections_abc.Iterator)", + "got": "isinstance(results, collections_abc.Mapping)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8539325842696629 + }, + { + "repo": "rubik/radon", + "expected": "[{'file-0': 0}, {'file-1': 1}, {'file-2': 4}]", + "got": "[('0', 0), ('1', 1), ('2', 4)]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.37333333333333335 + }, + { + "repo": "rubik/radon", + "expected": "a == e or int(a * 10 ** 3) == int(e * 10 ** 3)", + "got": "round(a - e) == 0", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.38095238095238093 + }, + { + "repo": "rubik/radon", + "expected": "( res, 120, 6, )", + "got": "(res, 120, 6)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.896551724137931 + }, + { + "repo": "rubik/radon", + "expected": "( res[:-2], 47, 4, )", + "got": "(", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.09523809523809523 + }, + { + "repo": "rubik/radon", + "expected": "( res[2:], 109, 4, )", + "got": "(", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.09523809523809523 + }, + { + "repo": "rubik/radon", + "expected": "result.blank + result.sloc + result.single_comments + result.multi", + "got": "Location(0, 0)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.1 + }, + { + "repo": "rubik/radon", + "expected": "not any( ['error' in kw for msg, args, kw in harvester.to_terminal()] )", + "got": "harvester.encoding", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.29213483146067415 + }, + { + "repo": "rubik/radon", + "expected": "( visitor.operators, visitor.operands, visitor.distinct_operators, visitor.distinct_operands, )", + "got": "visitor.counts", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.23853211009174313 + }, + { + "repo": "rubik/radon", + "expected": "{ 'loc': 103, 'lloc': 123, 'sloc': 98, 'comments': 8, 'multi': 19, 'blank': 5, 'single_comments': 3, }", + "got": "{ 'type': 'module', 'name': 'test_module.py', 'lineno': 103, 'col_offset': 123, 'endline': 19, 'complexity': 5, 'rank': 'A', 'imports': [], 'functions': [], 'classes': [], 'methods': [], 'closures': [], 'complexity': 3, 'rank': 'B', 'imports': [], 'functions': [], 'classes': [], 'methods': [], 'closures': [], 'complexity': 98, 'rank': 'A', 'imports': [], 'functions': [], 'classes': [], 'methods': [], 'closures': [], 'complexity': 8, 'rank': 'B', 'imports': [], 'functions': [], 'classes': [], 'methods': [], 'closures': [], 'complexity': 19, 'rank': 'B', 'imports': [], 'functions': [], 'classes': [], 'methods': [], 'closures': [], 'complexity': 123, 'rank': 'B', 'imports': [], 'functions': [], 'classes': [], 'methods': [], 'closures': [], 'complexity': 103", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.13163972286374134 + }, + { + "repo": "rubik/radon", + "expected": "[ ('a', ('mystr',), {'error': True}), ('{0} - {1}{2}{3}{4}', ('c', '<|B|>', 'B', ' (15.00)', '__R__'), {}), ('{0} - {1}{2}{3}{4}', ('d', '<|C|>', 'C', ' (0.00)', '__R__'), {}), ]", + "got": "[('a', ('mystr',), {'error': True}), ('b', (), {}), ('{0}: {1}', ('MI', 25), {'indent': 1}), ('{0}: {1}', ('Rank', 'A'), {'indent': 1}), ('c', (), {}), ('{0}: {1}', ('MI', 15), {'indent': 1}), ('{0}: {1}', ('Rank', 'B'), {'indent': 1}), ('d', (), {}), ('{0}: {1}', ('MI', 0), {'indent': 1}), ('{0}: {1}', ('Rank', 'C'), {'indent': 1}), ]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.32233009708737864 + }, + { + "repo": "rubik/radon", + "expected": "[ ('a', ('mystr',), {'error': True}), ('b', (), {}), (['res'], (), {'indent': 1}), ('\\n{0} blocks (classes, functions, methods) analyzed.', (3,), {}), ( 'Average complexity: {0}{1} ({2}){3}', ('<|A|>', 'A', 3, '__R__'), {}, ), ]", + "got": "['res', 'res']", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.05785123966942149 + }, + { + "repo": "rubik/radon", + "expected": "''' 6 name B filename 12 16 8 Classname B filename 17 29 7 Classname.name B filename 19 26 4 aux A filename 13 17 10 name B filename 12 16 '''.replace( '\\n', '' ).replace( ' ', '' )", + "got": "XML_OUTPUT", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "rubik/radon", + "expected": "[ ('a', ('mystr',), {'error': True}), ('b', (), {}), ('{0}: {1}', ('LOC', 24), {'indent': 1}), ('{0}: {1}', ('LLOC', 27), {'indent': 1}), ('{0}: {1}', ('SLOC', 15), {'indent': 1}), ('{0}: {1}', ('Comments', 3), {'indent': 1}), ('{0}: {1}', ('Single comments', 3), {'indent': 1}), ('{0}: {1}', ('Multi', 3), {'indent': 1}), ('{0}: {1}', ('Blank', 9), {'indent': 1}), ('- Comment Stats', (), {'indent': 1}), ('(C % L): {0:.0%}', (0.125,), {'indent': 2}), ('(C % S): {0:.0%}', (0.2,), {'indent': 2}), ('(C + M % L): {0:.0%}', (0.25,), {'indent': 2}), ('c', (), {}), ('{0}: {1}', ('LOC', 24), {'indent': 1}), ('{0}: {1}', ('LLOC', 27), {'indent': 1}), ('{0}: {1}', ('SLOC', 15), {'indent': 1}), ('{0}: {1}', ('Comments', 3), {'indent': 1}), ('{0}: {1}', ('Single comments', 13), {'indent': 1}), ('{0}: {1}', ('Multi', 3), {'indent': 1}), ('{0}: {1}', ('Blank', 9), {'indent': 1}), ('- Comment Stats', (), {'indent': 1}), ('(C % L): {0:.0%}', (0.125,), {'indent': 2}), ('(C % S): {0:.0%}', (0.2,), {'indent': 2}), ('(C + M % L): {0:.0%}', (0.25,), {'indent': 2}), ('e', (), {}), ('{0}: {1}', ('LOC', 0), {'indent': 1}), ('{0}: {1}', ('LLOC', 0), {'indent': 1}), ('{0}: {1}', ('SLOC', 0), {'indent': 1}), ('{0}: {1}', ('Comments', 0), {'indent': 1}), ('{0}: {1}', ('Single comments', 12), {'indent': 1}), ('{0}: {1}', ('Multi', 0), {'indent': 1}), ('{0}: {1}', ('Blank', 0), {'indent': 1}), ('- Comment Stats', (), {'indent': 1}), ('(C % L): {0:.0%}', (0.0,), {'indent': 2}), ('(C % S): {0:.0%}', (0.0,), {'indent': 2}), ('(C + M % L): {0:.0%}', (0.0,), {'indent': 2}), ('** Total **', (), {}), ('{0}: {1}', ('LOC', 48), {'indent': 1}), ('{0}: {1}', ('LLOC', 54), {'indent': 1}), ('{0}: {1}', ('SLOC', 30), {'indent': 1}), ('{0}: {1}', ('Comments', 6), {'indent': 1}), ('{0}: {1}', ('Single comments', 28), {'indent': 1}), ('{0}: {1}', ('Multi', 6), {'indent': 1}), ('{0}: {1}', ('Blank', 18), {'indent': 1}), ('- Comment Stats', (), {'indent': 1}), ('(C % L): {0:.0%}', (0.125,), {'indent': 2}), ('(C % S): {0:.0%}', (0.2,), {'indent': 2}), ('(C + M % L): {0:.0%}', (0.25,), {'indent': 2}), ]", + "got": "[('a', ()), ('b', ()), ('c', ()), ('e', ()),]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.019699812382739212 + }, + { + "repo": "rubik/radon", + "expected": "expected)", + "got": "SyntaxError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.19047619047619047 + }, + { + "repo": "rubik/radon", + "expected": "ValueError)", + "got": "ValueError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "expected_rank)", + "got": "expected_rank)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "AttributeError)", + "got": "TypeError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.56 + }, + { + "repo": "rubik/radon", + "expected": "NotImplementedError)", + "got": "NotImplementedError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "[])", + "got": "[])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "'mystr')", + "got": "'mystr')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "'msg\\n')", + "got": "'\\n')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7692307692307693 + }, + { + "repo": "rubik/radon", + "expected": "mi_config.multi)", + "got": "mi_config.show_complexity)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "rubik/radon", + "expected": "'randomfile.py')", + "got": "'randomfile.py')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "base_config.ignore)", + "got": "base_config.ignore)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "mocker.sentinel.one)", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.08695652173913043 + }, + { + "repo": "rubik/radon", + "expected": "mocker.sentinel.two)", + "got": "mocker.sentinel.two)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "cc_config.total_average)", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.07407407407407407 + }, + { + "repo": "rubik/radon", + "expected": "mocker.sentinel.harvester)", + "got": "'')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.06896551724137931 + }, + { + "repo": "rubik/radon", + "expected": "to_dicts_mock.return_value)", + "got": "{'a': {'rank': 'A'}})", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.08333333333333333 + }, + { + "repo": "rubik/radon", + "expected": "'@<|||>ERROR__R__: mystr\\n')", + "got": "'@<|||>mystr__R__\\n')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6530612244897959 + }, + { + "repo": "rubik/radon", + "expected": "dict([h._results[0], h._results[-1]]))", + "got": "h._results)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4489795918367347 + }, + { + "repo": "rubik/radon", + "expected": "['file.py', 'amod.py', 'test_all.py'])", + "got": "['file.py'])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.48 + }, + { + "repo": "rubik/radon", + "expected": "['file.py', 'amod.py', 'test_all.py', 'sub/amod.py', 'sub/bmod.py'])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.08450704225352113 + }, + { + "repo": "rubik/radon", + "expected": "['file.py', 'amod.py', 'tests/run.py', 'sub/amod.py', 'sub/bmod.py'])", + "got": "[])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.08333333333333333 + }, + { + "repo": "rubik/radon", + "expected": "cli.Config( exclude=None, ignore=None, summary=True, include_ipynb=False, ipynb_cells=False, ))", + "got": "cli.Config())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.24074074074074073 + }, + { + "repo": "rubik/radon", + "expected": "[ mocker.call('msg\\n'), mocker.call(' msg\\n'), mocker.call('2 + 1\\n'), mocker.call('{0} + 1\\n'), ])", + "got": "[('msg',), ('msg',), ('{0} + 1', (2,), {}), ('{0} + 1', (2,), {'noformat': True})])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.37362637362637363 + }, + { + "repo": "rubik/radon", + "expected": "cli.Config( min='A', max='C', exclude=None, ignore=None, show=True, multi=False, sort=False, include_ipynb=False, ipynb_cells=False, ))", + "got": "cli.Config())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.17567567567567569 + }, + { + "repo": "rubik/radon", + "expected": "cli.Config( min='A', max='F', exclude=None, ignore=None, show_complexity=False, average=False, order=getattr(cc_mod, 'SCORE'), no_assert=False, total_average=False, show_closures=False, include_ipynb=False, ipynb_cells=False, ))", + "got": "json=True)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.01680672268907563 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "0)", + "got": "10)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "9)", + "got": "10)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "4)", + "got": "4)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "42)", + "got": "\"Constant number\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.09523809523809523 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "-1)", + "got": "-1)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "+1)", + "got": "1)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "11)", + "got": "2)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "1234)", + "got": "\"Hello, World!\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.09523809523809523 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "True)", + "got": "True)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "False)", + "got": "False)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "0", + "got": "0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "2", + "got": "ExitCode.EX_USAGE", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "1", + "got": "2", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "66", + "got": "-1", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "\"\"", + "got": "\"FileNotFoundError: [Errno 2] No such file or directory: 'nonexistent.conf'\\n\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.05 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "[]", + "got": "data.expected_all_filtered", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "0.0", + "got": "0.9", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "None", + "got": "code", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "100.0", + "got": "1.0", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.75 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "18.18", + "got": "0.99", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "count", + "got": "count", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "[item]", + "got": "[item]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "\"value\"", + "got": "\"value\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "results", + "got": "results", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "object()", + "got": "object() in None", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "expected", + "got": "expected", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "original", + "got": "original", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "input_code", + "got": "input_code", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "ExitCode.OK", + "got": "0", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "items[:idx]", + "got": "[f\"job_id_{idx}\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.35714285714285715 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "ExitCode.CONFIG", + "got": "ExitCode.NO_INPUT", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6875 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "\"second result\"", + "got": "\"second result\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "ExitCode.NO_PERM", + "got": "ExitCode.NO_PERM", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "ExitCode.NO_INPUT", + "got": "1", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "isinstance(d, str)", + "got": "d", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.10526315789473684 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "Path(\"foo/bar.py\")", + "got": "\"foo/bar.py\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "initial_num_work_items", + "got": "initial_num_work_items", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "work_db.num_work_items", + "got": "0", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "init_order.initialized", + "got": "init_order.init_order()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7111111111111111 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "data.expected_after_filter", + "got": "data.expected_after_filter", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "data.expected_all_filtered", + "got": "data.expected_all_filtered", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "sorted(items[:idx], key=repr)", + "got": "sorted(items[:idx])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7916666666666666 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "sorted(items[idx:], key=repr)", + "got": "sorted(items[:idx], key=repr)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9655172413793104 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "sample.example.pre_mutation_code", + "got": "sample.example.pre_mutation_code", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "sample.example.post_mutation_code", + "got": "sample.example.post_mutation_code", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "f\"Could not find module path {path}\"", + "got": "\"FileNotFoundError: 'a/inexisting_file.py' not found\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.29213483146067415 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "not list(work_db.pending_work_items)", + "got": "not work_db.has_pending_jobs()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.696969696969697 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "\"Could not find module path example\" + os.sep + \"unknown_file.py\"", + "got": "\"Error: No such file or directory\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2828282828282828 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "KeyError)", + "got": "KeyError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "ConfigError)", + "got": "ConfigError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "FileNotFoundError)", + "got": "FileNotFoundError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "subprocess.CalledProcessError)", + "got": "FileExistsError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.43478260869565216 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "adam.adam_1.bool_and())", + "got": "adam.adam_1.bool_and(False, False))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7931034482758621 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "adam.adam_1.bool_expr_with_not())", + "got": "adam.adam_1.bool_expr_with_not())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "adam.adam_1.bool_or())", + "got": "adam.adam_1.bool_or())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "adam.adam_1.bool_if())", + "got": "adam.adam_1.bool_if())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "adam.adam_1.equals(vals))", + "got": "adam.adam_1.equals(vals))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "adam.adam_1.if_expression())", + "got": "adam.adam_1.if_expression())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "adam.adam_1.assert_in_func())", + "got": "adam.adam_1.assert_in_func())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "adam.adam_2.decorated_func())", + "got": "adam.adam_2.decorator())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8301886792452831 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "adam.adam_2.single_iteration())", + "got": "adam.adam_2.single_iteration())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "adam.adam_2.handle_exception())", + "got": "adam.adam_2.handle_exception())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "adam.adam_2.trigger_infinite_loop())", + "got": "adam.adam_1.trigger_infinite_loop())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9722222222222222 + }, + { + "repo": "stabldev/torrra", + "expected": "0", + "got": "0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stabldev/torrra", + "expected": "1", + "got": "1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stabldev/torrra", + "expected": "5", + "got": "5.0", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "stabldev/torrra", + "expected": "3", + "got": "2", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "stabldev/torrra", + "expected": "2", + "got": "2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stabldev/torrra", + "expected": "17", + "got": "17", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stabldev/torrra", + "expected": "123", + "got": "123", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stabldev/torrra", + "expected": "523", + "got": "523", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stabldev/torrra", + "expected": "key2", + "got": "key2", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stabldev/torrra", + "expected": "key3", + "got": "key3", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stabldev/torrra", + "expected": "True", + "got": "True", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stabldev/torrra", + "expected": "45.6", + "got": "45.6", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stabldev/torrra", + "expected": "None", + "got": "None", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stabldev/torrra", + "expected": "query", + "got": "\"arch linux iso\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.09523809523809523 + }, + { + "repo": "stabldev/torrra", + "expected": "magnet", + "got": "magnet", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stabldev/torrra", + "expected": "\"default\"", + "got": "\"default\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stabldev/torrra", + "expected": "\"1.00 KB\"", + "got": "\"1.00 KB\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stabldev/torrra", + "expected": "\"1.46 KB\"", + "got": "\"1.46 KB\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stabldev/torrra", + "expected": "\"5.00 MB\"", + "got": "\"5.00 MB\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stabldev/torrra", + "expected": "magnet_uri", + "got": "magnet_uri", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stabldev/torrra", + "expected": "\"new_value\"", + "got": "\"new_value\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stabldev/torrra", + "expected": "config_list", + "got": "config_list", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stabldev/torrra", + "expected": "target_theme", + "got": "target_theme", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stabldev/torrra", + "expected": "version_text", + "got": "version_text", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stabldev/torrra", + "expected": "result.output", + "got": "result.stdout", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7692307692307693 + }, + { + "repo": "stabldev/torrra", + "expected": "initial_index", + "got": "initial_index", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stabldev/torrra", + "expected": "\"textual-dark\"", + "got": "\"dark\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6 + }, + { + "repo": "stabldev/torrra", + "expected": "str(exc.value)", + "got": "str(exc.value)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stabldev/torrra", + "expected": "original_theme", + "got": "\"light\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "stabldev/torrra", + "expected": "initial_index + 1", + "got": "initial_index + 1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stabldev/torrra", + "expected": "config_path.exists()", + "got": "os.path.exists(config_path)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5957446808510638 + }, + { + "repo": "stabldev/torrra", + "expected": "\"Arch Linux ISO (Mock)\"", + "got": "\"Arch Linux ISO (Mock)\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stabldev/torrra", + "expected": "table.has_class(\"hidden\")", + "got": "table.has_class(\"hidden\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stabldev/torrra", + "expected": "str(loader_status.content)", + "got": "loader_status.value", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "stabldev/torrra", + "expected": "not table.has_class(\"hidden\")", + "got": "table.rows", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.358974358974359 + }, + { + "repo": "stabldev/torrra", + "expected": "key1.startswith(\"test_prefix:\")", + "got": "not key1.startswith(\"test_prefix\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9230769230769231 + }, + { + "repo": "stabldev/torrra", + "expected": "app.screen.query_one(\"", + "got": "isinstance(app.screen._welcome_text, Static)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3939393939393939 + }, + { + "repo": "stabldev/torrra", + "expected": "app.screen.query_one(\"", + "got": "app.screen.query_one(\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stabldev/torrra", + "expected": "app.screen.query_one(\"", + "got": "app.screen.query_one(\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stabldev/torrra", + "expected": "isinstance(app.screen, HomeScreen)", + "got": "isinstance(app.screen, HomeScreen)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stabldev/torrra", + "expected": "isinstance(app.screen, WelcomeScreen)", + "got": "isinstance(pilot.screen, WelcomeScreen)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9210526315789473 + }, + { + "repo": "stabldev/torrra", + "expected": "snap_compare(app, run_before=run_before)", + "got": "snap_compare(\"home_screen.png\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5070422535211268 + }, + { + "repo": "stabldev/torrra", + "expected": "isinstance(app.screen, ThemeSelectorScreen)", + "got": "not pilot.app.current_screen", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3380281690140845 + }, + { + "repo": "stabldev/torrra", + "expected": "ImportError)", + "got": "ImportError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stabldev/torrra", + "expected": "ConfigError, match=\"key does not contain a value\")", + "got": "ConfigError, match=\"key does not contain a section\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9019607843137255 + }, + { + "repo": "stabldev/torrra", + "expected": "ConfigError, match=\"key does not contain a section\")", + "got": "ConfigError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.375 + }, + { + "repo": "stanford-cs336/assignment1-basics", + "expected": "0", + "got": "0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stanford-cs336/assignment1-basics", + "expected": "3", + "got": "2", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "stanford-cs336/assignment1-basics", + "expected": "1", + "got": "4", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "stanford-cs336/assignment1-basics", + "expected": "[]", + "got": "[\"\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "stanford-cs336/assignment1-basics", + "expected": "1.5", + "got": "1.5", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stanford-cs336/assignment1-basics", + "expected": "None", + "got": "None", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stanford-cs336/assignment1-basics", + "expected": "[\"s\"]", + "got": "[\"s\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stanford-cs336/assignment1-basics", + "expected": "word_bytes", + "got": "word_bytes", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stanford-cs336/assignment1-basics", + "expected": "test_string", + "got": "test_string", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stanford-cs336/assignment1-basics", + "expected": "reference_ids", + "got": "reference_ids", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stanford-cs336/assignment1-basics", + "expected": "decoded_string", + "got": "decoded_string", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stanford-cs336/assignment1-basics", + "expected": "len(t1_c_grads)", + "got": "len(t1_c_grads)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stanford-cs336/assignment1-basics", + "expected": "corpus_contents", + "got": "corpus_contents", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stanford-cs336/assignment1-basics", + "expected": "reference_merges", + "got": "reference_merges", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stanford-cs336/assignment1-basics", + "expected": "loaded_iterations", + "got": "loaded_iterations", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stanford-cs336/assignment1-basics", + "expected": "set(new_model_state.keys())", + "got": "set(new_model_state.keys())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stanford-cs336/assignment1-basics", + "expected": "set(reference_vocab.keys())", + "got": "set(reference_vocab.keys())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stanford-cs336/assignment1-basics", + "expected": "set(reference_vocab.values())", + "got": "set(reference_vocab.values())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stanford-cs336/assignment1-basics", + "expected": "[\"Hello\", \",\", \" how\", \" are\", \" you\", \"?\"]", + "got": "[\"Hello,\", \"how\", \"are\", \"you\", \"?\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9113924050632911 + }, + { + "repo": "stanford-cs336/assignment1-basics", + "expected": "output)", + "got": "output)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stanford-cs336/assignment1-basics", + "expected": "actual_output)", + "got": "actual_output)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stanford-cs336/assignment1-basics", + "expected": "actual_weights)", + "got": "actual_weights)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stanford-cs336/assignment1-basics", + "expected": "y.detach().numpy())", + "got": "y)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.19047619047619047 + }, + { + "repo": "stanford-cs336/assignment1-basics", + "expected": "expected_arrays[key])", + "got": "expected_arrays[key])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stanford-cs336/assignment1-basics", + "expected": "truncated_actual_output)", + "got": "truncated_actual_output)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stanford-cs336/assignment1-basics", + "expected": "expected.detach().numpy())", + "got": "expected)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5142857142857142 + }, + { + "repo": "stanford-cs336/assignment1-basics", + "expected": "numpy.array(expected_lrs))", + "got": "numpy.array(expected_lrs))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stanford-cs336/assignment1-basics", + "expected": "t1_c_grad.detach().numpy())", + "got": "t1_c_grad.detach().numpy())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stanford-cs336/assignment1-basics", + "expected": "expected_output.detach().numpy())", + "got": "expected_output)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6530612244897959 + }, + { + "repo": "stanford-cs336/assignment1-basics", + "expected": "new_model_state[key].detach().numpy())", + "got": "new_model_state[key].detach().numpy())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stanford-cs336/assignment1-basics", + "expected": "large_expected_cross_entropy.detach().numpy())", + "got": "large_expected_cross_entropy.detach().numpy())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "stanford-cs336/assignment1-basics", + "expected": "{ \"vocab_keys\": set(vocab.keys()), \"vocab_values\": set(vocab.values()), \"merges\": merges, })", + "got": "vocab)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.12244897959183673 + }, + { + "repo": "stanford-cs336/assignment1-basics", + "expected": "(RuntimeError, AssertionError))", + "got": "ValueError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.38095238095238093 + }, + { + "repo": "staticjinja/staticjinja", + "expected": "1", + "got": "1", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "staticjinja/staticjinja", + "expected": "0", + "got": "0", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "staticjinja/staticjinja", + "expected": "{}", + "got": "{\"foo\": \"bar\"}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.25 + }, + { + "repo": "staticjinja/staticjinja", + "expected": "[]", + "got": "set()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "staticjinja/staticjinja", + "expected": "b\"\"", + "got": "expected_help_message", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "staticjinja/staticjinja", + "expected": "expected", + "got": "expected", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "staticjinja/staticjinja", + "expected": "{\"a\": 1}", + "got": "{\"foo\": \"bar\"}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5454545454545454 + }, + { + "repo": "staticjinja/staticjinja", + "expected": "{\"b\": 3}", + "got": "{\"a\": 1}", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.75 + }, + { + "repo": "staticjinja/staticjinja", + "expected": "\"Test 1\"", + "got": "\"Hello World!\"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "staticjinja/staticjinja", + "expected": "\"Test 3\"", + "got": "\"Test 3\"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "staticjinja/staticjinja", + "expected": "SystemExit", + "got": "SystemExit", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "staticjinja/staticjinja", + "expected": "[css_path]", + "got": "[css_path]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "staticjinja/staticjinja", + "expected": "[filename]", + "got": "[filename]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "staticjinja/staticjinja", + "expected": "caplog.text", + "got": "caplog.text", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "staticjinja/staticjinja", + "expected": "logging.INFO", + "got": "logging.INFO", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "staticjinja/staticjinja", + "expected": "watch_called", + "got": "watch_called", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "staticjinja/staticjinja", + "expected": "Path(expected)", + "got": "Path(expected)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "staticjinja/staticjinja", + "expected": "logging.CRITICAL", + "got": "logging.CRITICAL", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "staticjinja/staticjinja", + "expected": "[\"outside.html\"]", + "got": "[\"symlink_to_dir/outside.html\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6808510638297872 + }, + { + "repo": "staticjinja/staticjinja", + "expected": "expected_templates", + "got": "expected_templates", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "staticjinja/staticjinja", + "expected": "mock_template_names", + "got": "mock_template_names", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "staticjinja/staticjinja", + "expected": "[\"symlink_to_file\"]", + "got": "[\"symlink_to_file\"]", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "staticjinja/staticjinja", + "expected": "list(site.templates)", + "got": "site.templates", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8235294117647058 + }, + { + "repo": "staticjinja/staticjinja", + "expected": "expected_help_message", + "got": "expected_help_message", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "staticjinja/staticjinja", + "expected": "staticjinja.__version__", + "got": "staticjinja.__version__", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "staticjinja/staticjinja", + "expected": "\"

    Hello world!

    \"", + "got": "\"

    Hello world!

    \"", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "staticjinja/staticjinja", + "expected": "Path.cwd() / searchpath", + "got": "Path(root_path, searchpath)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.64 + }, + { + "repo": "staticjinja/staticjinja", + "expected": "site.get_rule(\"template2.html\")", + "got": "site.get_rule(\"template2.html\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "staticjinja/staticjinja", + "expected": "site.get_rule(\"template1.html\")", + "got": "site.get_rule(\"not a rule\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6896551724137931 + }, + { + "repo": "staticjinja/staticjinja", + "expected": "reloader.should_handle(\"created\", str(exists))", + "got": "not exists.exists()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "staticjinja/staticjinja", + "expected": "[reloader.site.get_template(\"template1.html\")]", + "got": "[\"template1.html\"]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5625 + }, + { + "repo": "staticjinja/staticjinja", + "expected": "reloader.should_handle(\"modified\", str(exists))", + "got": "not reloader.should_handle(\"created\", str(DNE))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7446808510638298 + }, + { + "repo": "staticjinja/staticjinja", + "expected": "not reloader.should_handle(\"modified\", str(DNE))", + "got": "not reloader.should_handle(\"created\", str(DNE))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8842105263157894 + }, + { + "repo": "staticjinja/staticjinja", + "expected": "ValueError)", + "got": "ValueError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "staticjinja/staticjinja", + "expected": "SystemExit)", + "got": "SystemExit)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "sb", + "got": "sb.is_valid()", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.26666666666666666 + }, + { + "repo": "sublimelsp/LSP", + "expected": "None", + "got": "None", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "view", + "got": "view_to_uri(view)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.38095238095238093 + }, + { + "repo": "sublimelsp/LSP", + "expected": "cls.view", + "got": "cls.wm.get_document_listener(cls.view)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.34782608695652173 + }, + { + "repo": "sublimelsp/LSP", + "expected": "self.view", + "got": "self.window.document_sync_listener", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.27906976744186046 + }, + { + "repo": "sublimelsp/LSP", + "expected": "{\"source.sort.json\": True}", + "got": "[CodeActionsOnFormatOnSaveTask.get_code_action(view=self.view)]", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.15730337078651685 + }, + { + "repo": "sublimelsp/LSP", + "expected": "0)", + "got": "0)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "1)", + "got": "2)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "sublimelsp/LSP", + "expected": "3)", + "got": "3)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "4)", + "got": "1)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "sublimelsp/LSP", + "expected": "5)", + "got": "2)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "sublimelsp/LSP", + "expected": "6)", + "got": "6)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "2)", + "got": "2)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "{})", + "got": "{})", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "[])", + "got": "[])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "10)", + "got": "1)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "sublimelsp/LSP", + "expected": "\"\")", + "got": "\"abcba\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5454545454545454 + }, + { + "repo": "sublimelsp/LSP", + "expected": "'d')", + "got": "'d')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "'b')", + "got": "'b')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "'a')", + "got": "'a')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "'c')", + "got": "'c')", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "\"F\")", + "got": "'F')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5 + }, + { + "repo": "sublimelsp/LSP", + "expected": "\"c\")", + "got": "\"abcba\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "sublimelsp/LSP", + "expected": "new)", + "got": "new)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "\"\ud83c\udf7a\")", + "got": "\"\ud83c\udf7a\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "True)", + "got": "False)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "sublimelsp/LSP", + "expected": "\"fd\")", + "got": "'fd')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6 + }, + { + "repo": "sublimelsp/LSP", + "expected": "text)", + "got": "text)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "False)", + "got": "False)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "value)", + "got": "value)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "\"baz\")", + "got": "\"baz\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "\"2.0\")", + "got": "\"2.0\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "\"!@", + "got": "[\"!\", \"@\", \"", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "sublimelsp/LSP", + "expected": "'c\\nd')", + "got": "'c\\nd\\n')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.875 + }, + { + "repo": "sublimelsp/LSP", + "expected": "'axyc')", + "got": "'ax$0ybc')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8235294117647058 + }, + { + "repo": "sublimelsp/LSP", + "expected": "dict())", + "got": "{})", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2 + }, + { + "repo": "sublimelsp/LSP", + "expected": "\"exit\")", + "got": "\"exit\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "\"file\")", + "got": "'file')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7142857142857143 + }, + { + "repo": "sublimelsp/LSP", + "expected": "expect)", + "got": "expect)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "'\ud83d\udd75\ufe0fbye')", + "got": "'bye')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "sublimelsp/LSP", + "expected": "\"hello\")", + "got": "\"hello\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "\"world\")", + "got": "\"world\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "content)", + "got": "content)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "expected)", + "got": "expected)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "(39, 26))", + "got": "(27, 39))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5555555555555556 + }, + { + "repo": "sublimelsp/LSP", + "expected": "(39, 30))", + "got": "(39, 27))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7777777777777778 + }, + { + "repo": "sublimelsp/LSP", + "expected": "'ax$0yc')", + "got": "'x$0ybc')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "sublimelsp/LSP", + "expected": "'a\\n\\nb')", + "got": "'ab\\n\\n$0\\n\\n')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "sublimelsp/LSP", + "expected": "['*.js'])", + "got": "['*.js'])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "['.git'])", + "got": "['.git'])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "['*.ts'])", + "got": "['*.ts'])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "['*.py'])", + "got": "['*.py'])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "\"buffer\")", + "got": "\"buffer\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "\"a
    b\")", + "got": "\"a
    b\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "{\"a\": {}})", + "got": "{\"a\": {}})", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "\"somelang\")", + "got": "\"somelang\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "config.name)", + "got": "config.name)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "expected[i])", + "got": "expected[i])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "set((\"d\",)))", + "got": "(\"d\",))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7368421052631579 + }, + { + "repo": "sublimelsp/LSP", + "expected": "set((\"a\",)))", + "got": "set((\"a\",)))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "set((\"c\",)))", + "got": "set((\"c\",)))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "\"buffer:42\")", + "got": "\"buffer:42\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "\"initialize\")", + "got": "\"initialize\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "(True, True))", + "got": "(False, True))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7407407407407407 + }, + { + "repo": "sublimelsp/LSP", + "expected": "'const x = 1')", + "got": "'const x = 1;')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9655172413793104 + }, + { + "repo": "sublimelsp/LSP", + "expected": "[TEST_CONFIG])", + "got": "[TEST_CONFIG.name])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8484848484848485 + }, + { + "repo": "sublimelsp/LSP", + "expected": "\"hello there\")", + "got": "\"hello\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7272727272727273 + }, + { + "repo": "sublimelsp/LSP", + "expected": "{\"text\": \"\ud83d\ude03\"})", + "got": "{\"text\": \"\ud83d\ude03\"})", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "\"

    A\\\\B

    \")", + "got": "\"

    A\\\\B

    \")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "parent_folder)", + "got": "parent_folder)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "\"/short-path\")", + "got": "\"/short-path\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "'const x = 1;')", + "got": "\"var a = 1;\\n\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4666666666666667 + }, + { + "repo": "sublimelsp/LSP", + "expected": "'{\"keys\": []}')", + "got": "'{\"keys\": [$1]}')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9375 + }, + { + "repo": "sublimelsp/LSP", + "expected": "'{{ turtle }}')", + "got": "'{{ turtletle }}')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "sublimelsp/LSP", + "expected": "'/foo/bar:1:6')", + "got": "'file:///foo/bar')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "sublimelsp/LSP", + "expected": "{\"foo\": \"bar\"})", + "got": "{\"foo\": \"bar\"})", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "nearest_folder)", + "got": "nearest_folder)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "\"/foo/bar/baz\")", + "got": "\"/foo/bar/baz\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "initial_content)", + "got": "initial_content)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "\"source.python\")", + "got": "\"source.python\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "set((\"a\", \"b\")))", + "got": "set((\"a\", \"b\")))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "{\"a\": {\"b\": {}}})", + "got": "{\"a\": {\"b\": {}}})", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "'\\t\\n\\ta\\n\\t\\tb')", + "got": "\"\\ta\\n\\tb\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6428571428571429 + }, + { + "repo": "sublimelsp/LSP", + "expected": "'{{ titletle }}')", + "got": "'{{ title }}')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9032258064516129 + }, + { + "repo": "sublimelsp/LSP", + "expected": "\"hello\\nthere\\n\")", + "got": "\"hello\\nthere\\nworld\\n\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8292682926829268 + }, + { + "repo": "sublimelsp/LSP", + "expected": "unrelated_folder)", + "got": "unrelated_folder)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "self.config1.name)", + "got": "self.config1.name)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "''.join(expected))", + "got": "''.join(expected))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "\"bar\\nbar\\nbar\\n\")", + "got": "\"bar\\nbar\\nbar\\n\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "first_line_length)", + "got": "9999)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.08695652173913043 + }, + { + "repo": "sublimelsp/LSP", + "expected": "settings['unknown'])", + "got": "{\"foo\": 1})", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.12903225806451613 + }, + { + "repo": "sublimelsp/LSP", + "expected": "'", + "got": "'", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "'/foo/bar:1235:4322')", + "got": "'/foo/bar:1234:4321')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9047619047619048 + }, + { + "repo": "sublimelsp/LSP", + "expected": "sublime.Region(2, 2))", + "got": "((0, 2), (0, 3)))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3157894736842105 + }, + { + "repo": "sublimelsp/LSP", + "expected": "['create', 'delete'])", + "got": "['create', 'delete'])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "expected_total_lines)", + "got": "expected_total_lines)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "set((\"x\", \"y\", \"z\")))", + "got": "set((\"x\", \"y\", \"z\")))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "set((\"a\", \"b\", \"c\")))", + "got": "set((\"a\", \"b\", \"c\")))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "expected_language_id)", + "got": "expected_language_id)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "overriddes['unknown'])", + "got": "overriddes['unknown'])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "\"a\\nhello there\\nc\\n\")", + "got": "'a\\nb\\nc\\nhello there')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "sublimelsp/LSP", + "expected": "{\"general\": \"kenobi\"})", + "got": "{\"general\": \"kenobi\"})", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "first_line_length + 3)", + "got": "first_line_length + 2)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9545454545454546 + }, + { + "repo": "sublimelsp/LSP", + "expected": "eval(repr(workspace)))", + "got": "workspace)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.625 + }, + { + "repo": "sublimelsp/LSP", + "expected": "\"no  breaks\")", + "got": "\"no breaks\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.85 + }, + { + "repo": "sublimelsp/LSP", + "expected": "\"

    hello\\\\\\nworld

    \")", + "got": "\"hello\\nworld\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7692307692307693 + }, + { + "repo": "sublimelsp/LSP", + "expected": "experimental_capabilities)", + "got": "experimental_capabilities)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "\"file:///foo/bar/baz.txt\")", + "got": "\"file:///foo/bar/baz.txt\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "\"file:///hello/there.txt\")", + "got": "\"file:///hello/there.txt\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "\"C:/hello;X:/there;Y:/asdf\")", + "got": "\"C:/hello;X:/there;Y:/$foobar\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.847457627118644 + }, + { + "repo": "sublimelsp/LSP", + "expected": "\"C:/hello:X:/there:Y:/asdf\")", + "got": "\"C:/hello:X:/there:Y:/asdf\")", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "\"file:///workspace/file.js\")", + "got": "\"/workspace/file.js\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "sublimelsp/LSP", + "expected": "b'{\"text\":\"\\xF0\\x9F\\x98\\x83\"}')", + "got": "\"\\\"\\\\ud83d\\\\ude0e\\\"\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "sublimelsp/LSP", + "expected": "lsp_color_to_html(response[0]))", + "got": "\"hello\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.20512820512820512 + }, + { + "repo": "sublimelsp/LSP", + "expected": "R'\\\\wsl$\\Ubuntu-20.04\\File.php')", + "got": "\"/mnt/c/Users/username/Documents/File.php\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.32 + }, + { + "repo": "sublimelsp/LSP", + "expected": "f'/a/b/{pathsep}{original_path}')", + "got": "\"/a/b/\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.2926829268292683 + }, + { + "repo": "sublimelsp/LSP", + "expected": "{\"canReturnAnythingHere\": \"asdf\"})", + "got": "\"asdf\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.34146341463414637 + }, + { + "repo": "sublimelsp/LSP", + "expected": "R'\\\\192.168.80.2\\D$\\www\\File.php')", + "got": "\"\\\\192.168.80.2\\\\D$\\\\www\\\\File.php\")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8857142857142857 + }, + { + "repo": "sublimelsp/LSP", + "expected": "filename_to_uri(\"/dir ectory/file.txt\"))", + "got": "filename_to_uri(\"/dir ectory/file.txt\"))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "\"Hello,     world \")", + "got": "\"Hello,\\n\\n world \")", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6 + }, + { + "repo": "sublimelsp/LSP", + "expected": "'\\n')", + "got": "'$hello = \"world\";\\n$he\\n?>\\n')", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8611111111111112 + }, + { + "repo": "sublimelsp/LSP", + "expected": "filename_to_uri(\"c:\\\\dir ectory\\\\file.txt\"))", + "got": "filename_to_uri(\"C:/dir ectory/file.txt\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9069767441860465 + }, + { + "repo": "sublimelsp/LSP", + "expected": "sb)", + "got": "sb)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "self.wm)", + "got": "self.wm)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "self.view)", + "got": "self.window_mgr.all)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4666666666666667 + }, + { + "repo": "sublimelsp/LSP", + "expected": "self.panel)", + "got": "self.panel)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "self.window)", + "got": "self.view.window())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.7741935483870968 + }, + { + "repo": "sublimelsp/LSP", + "expected": "self.session)", + "got": "self.session)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "d)", + "got": "d)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "added)", + "got": "added)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "removed)", + "got": "removed)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "s.has(key))", + "got": "key in s)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "sublimelsp/LSP", + "expected": "view.is_loading())", + "got": "view.is_scratch())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "sublimelsp/LSP", + "expected": "selector.matches(view))", + "got": "selector.matches(view))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "trigger.get(\"characters\"))", + "got": "trigger.get(\"characters\"))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "self.view.is_popup_visible())", + "got": "self.view.hovered)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5957446808510638 + }, + { + "repo": "sublimelsp/LSP", + "expected": "session.should_notify_did_open())", + "got": "session.should_notify_did_open())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "any(manager.match_view(self.view)))", + "got": "DISABLED_CONFIG.enabled)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.1694915254237288 + }, + { + "repo": "sublimelsp/LSP", + "expected": "kind_contains_other_kind('aa', 'a'))", + "got": "kind_contains_other_kind('a.b', 'a.b.c'))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "sublimelsp/LSP", + "expected": "kind_contains_other_kind('a', 'b.a'))", + "got": "kind_contains_other_kind('a', 'a.b.c'))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9473684210526315 + }, + { + "repo": "sublimelsp/LSP", + "expected": "list(window_mgr.all.values())[0].enabled)", + "got": "TEST_CONFIG.enabled)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.32786885245901637 + }, + { + "repo": "sublimelsp/LSP", + "expected": "selector.matches(self._make_html_view(\"ac.html\")))", + "got": "selector.matches(self._make_html_view(\"axyc.html\")))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9803921568627451 + }, + { + "repo": "sublimelsp/LSP", + "expected": "config.is_disabled_capability(\"completionProvider\"))", + "got": "config.is_disabled_capability(\"completionProvider\"))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "config.is_disabled_capability(\"definitionProvider\"))", + "got": "config.is_disabled_capability(\"codeActionProvider.codeActionKinds\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8 + }, + { + "repo": "sublimelsp/LSP", + "expected": "selector.matches(self._make_html_view(\"axyc.html\")))", + "got": "selector.matches(self._make_html_view(\"axyc.html\")))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "selector.matches(self._make_html_view(\"example.0\")))", + "got": "selector.matches(self._make_html_view(\"example.0\")))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "selector.matches(self._make_html_view(\"example.1\")))", + "got": "selector.matches(self._make_html_view(\"example.1\")))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "selector.matches(self._make_html_view(\"example.10\")))", + "got": "selector.matches(self._make_html_view(\"example.a\")))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9714285714285714 + }, + { + "repo": "sublimelsp/LSP", + "expected": "d)", + "got": "bool(d))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4 + }, + { + "repo": "sublimelsp/LSP", + "expected": "view)", + "got": "view)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "self.view)", + "got": "self.view.is_valid())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6451612903225806 + }, + { + "repo": "sublimelsp/LSP", + "expected": "self.window)", + "got": "self.window)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "config.enabled)", + "got": "config.enabled)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "TEST_CONFIG.enabled)", + "got": "TEST_CONFIG.name in sublime.active_window().configurations)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4810126582278481 + }, + { + "repo": "sublimelsp/LSP", + "expected": "self.view.is_valid())", + "got": "self.view.is_valid())", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "selector.matches(view))", + "got": "selector.matches(view))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "\"greeting\" in last_content)", + "got": "last_content)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.65 + }, + { + "repo": "sublimelsp/LSP", + "expected": "session.should_notify_did_open())", + "got": "session.capabilities[\"textDocumentSync\"][\"openClose\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.39080459770114945 + }, + { + "repo": "sublimelsp/LSP", + "expected": "change['uri'].endswith('file.js'))", + "got": "change['uri'])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5833333333333334 + }, + { + "repo": "sublimelsp/LSP", + "expected": "change1['uri'].endswith('file.py'))", + "got": "change1['uri'].endswith('file.py'))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "kind_contains_other_kind('a', 'a.b'))", + "got": "kind_matches('a.b', 'a.b'))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6875 + }, + { + "repo": "sublimelsp/LSP", + "expected": "config.is_disabled_capability(\"colorProvider\"))", + "got": "config.disabled_capabilities[\"colorProvider\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.8817204301075269 + }, + { + "repo": "sublimelsp/LSP", + "expected": "self.wm.get_config_manager().match_view(self.view))", + "got": "self.ensure_document_listener_created())", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.3956043956043956 + }, + { + "repo": "sublimelsp/LSP", + "expected": "selector.matches(self._make_html_view(\"abc.html\")))", + "got": "selector.matches(self._make_html_view(\"abc.html\")))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "selector.matches(self._make_html_view(\"axc.html\")))", + "got": "selector.matches(self._make_html_view(\"ac.html\")))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9900990099009901 + }, + { + "repo": "sublimelsp/LSP", + "expected": "selector.matches(self._make_html_view(\"axyc.html\")))", + "got": "selector.matches(self._make_html_view(\"a.html\")))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.9702970297029703 + }, + { + "repo": "sublimelsp/LSP", + "expected": "selector.matches(self._make_html_view(\"example.0\")))", + "got": "selector.matches(self._make_view(\"Packages/HTML/example.0.html\", \"example.0.html\")))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6911764705882353 + }, + { + "repo": "sublimelsp/LSP", + "expected": "selector.matches(self._make_html_view(\"example.1\")))", + "got": "selector.matches(self._make_html_view(\"example.1\")))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "selector.matches(self._make_html_view(\"example.a\")))", + "got": "selector.matches(self._make_html_view(\"example.a\")))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "self.view.settings().get(\"translate_tabs_to_spaces\"))", + "got": "self.view.settings().get(\"translate_tabs_to_spaces\"))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "signature)", + "got": "signature)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "d.get(\"foo\"))", + "got": "d.get(\"foo\"))", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "d.get(\"some.nonexistant.key\"))", + "got": "d.get(\"foo.bar.baz\"))", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.5098039215686274 + }, + { + "repo": "sublimelsp/LSP", + "expected": "d)", + "got": "d)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "params)", + "got": "params)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "options)", + "got": "options)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "changes)", + "got": "changes)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "diagnostic1)", + "got": "diagnostic1)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "window_mgr.all)", + "got": "window_mgr.all)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "params[\"capabilities\"])", + "got": "params[\"capabilities\"])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "self.session.capabilities)", + "got": "self.session._capabilities[\"textDocument\"][\"completion\"])", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6265060240963856 + }, + { + "repo": "sublimelsp/LSP", + "expected": "d)", + "got": "d)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "options)", + "got": "options)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "payload)", + "got": "payload)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "diagnostic2)", + "got": "diagnostic2)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "window_mgr.all)", + "got": "window_mgr._configs)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.6857142857142857 + }, + { + "repo": "sublimelsp/LSP", + "expected": "params[\"capabilities\"])", + "got": "params)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.4666666666666667 + }, + { + "repo": "sublimelsp/LSP", + "expected": "d_copy['a'])", + "got": "None)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.23529411764705882 + }, + { + "repo": "sublimelsp/LSP", + "expected": "d_copy['c'])", + "got": "d_copy['c'])", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "expected_regex)", + "got": "expected_regex)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "regex.replace(\"\\n\", \"\").replace(\" \", \"\"))", + "got": "regex)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.20689655172413793 + }, + { + "repo": "sublimelsp/LSP", + "expected": "DottedDict)", + "got": "dict)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.625 + }, + { + "repo": "sublimelsp/LSP", + "expected": "TypeError)", + "got": "AttributeError)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.56 + }, + { + "repo": "sublimelsp/LSP", + "expected": "ValueError)", + "got": "ValueError)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "list)", + "got": "list)", + "exact_match": true, + "code_bleu": 0.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "1)", + "got": "n + 3)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.25 + }, + { + "repo": "sublimelsp/LSP", + "expected": "r''' 
     
     
     
     99.99999GB\").matches(99999999999)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.6
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "regex.matches(\".pdf\")",
+      "got": "regex.match(\"test.pdf\")",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8636363636363636
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "Size(\"<1\").matches(0)",
+      "got": "not Size(\"<=1,>=0.000kb\").matches(1)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.7017543859649122
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "not result.skip_action",
+      "got": "not result[0]",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.5714285714285714
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "[\"foo.txt\", \"bar.txt\"]",
+      "got": "[\"foo.txt\", \"bar.txt\"]",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{\"Application.app\": {}}",
+      "got": "{\"Application.app\": {\"info.plist\": {}}}",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.7419354838709677
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{\"found-regex.txt\": \"\"}",
+      "got": "{f\"found-regex.txt\": \"\"}",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.9787234042553191
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "set([\"3.jpg\", \"4.jpg\"])",
+      "got": "{\"3\", \"4\"}",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.48484848484848486
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "name.matches(\"hereend\")",
+      "got": "name.matches(\"hereend\")",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "[\"BAR\", \"BAZ\", \"FOOFOO\"]",
+      "got": "\"FOO\\nFOO\\nFOO\\n\"",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.3902439024390244
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{\"test\": {\"folder\": {}}}",
+      "got": "{}",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.15384615384615385
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "Config.from_string(conf)",
+      "got": "not Config.from_string(conf).filters",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "datetime.utcnow().date()",
+      "got": "now.date()",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.5882352941176471
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "Size(\"1k\").matches(1000)",
+      "got": "not Size(\">1kb,<1kib\").matches(999)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.6779661016949152
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "Path(\"/test/dir1/sub1 2\")",
+      "got": "Path(\"/test/dir1/sub1\")",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.9583333333333334
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "name.matches(\"beginhere\")",
+      "got": "name(\"begin\")",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.6842105263157895
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "name.matches(\"herebegin\")",
+      "got": "not name.matches(\"herebegin\")",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.9259259259259259
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "not name.matches(\"other\")",
+      "got": "not name.matches(\"here\")",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.9387755102040817
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "not regex.matches(\"+pdf\")",
+      "got": "not regex.matches(\".pdfs\")",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.9411764705882353
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "not regex.matches(\"/pdf\")",
+      "got": "not regex.matches(\"pdf\")",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.9795918367346939
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "not Size(\">1\").matches(0)",
+      "got": "not Size(\"<1\").matches(1)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.92
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "not Path(\"/test\").exists()",
+      "got": "not prepare_target_path( src_name=\"dst.txt\", dst=\"/test/\", autodetect_folder=False, simulate=True )",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.288
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "[\"bar.txt\", \"foo.txt\"] * 2",
+      "got": "[\"foo.txt\", \"bar.txt\"]",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.4583333333333333
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "not name.matches(\"conend\")",
+      "got": "not name.matches(\"end\")",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.9387755102040817
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "not name.matches(\"CONEND\")",
+      "got": "name.matches(\"STARconend\")",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.6153846153846154
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "name.matches(\"STARCONEND\")",
+      "got": "name.matches(\"STARconend\")",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.7692307692307693
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "Size(\"1000\").matches(1000)",
+      "got": "not Size(\">1kb,<1kib\").matches(999)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.6229508196721312
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "has_executable(present_exe)",
+      "got": "has_executable(present_exe)",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "Counter([\"fileA\", \"fileB\"])",
+      "got": "Counter({\"fileA\", \"fileB\"})",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.9259259259259259
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "name.matches(\"beginconend\")",
+      "got": "not name.matches(\"end\")",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.76
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "name.matches(\"STAR_conEnD\")",
+      "got": "name.matches(\"STAR_somethgin_con_END\")",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "Size(\"<100 Mb\").matches(20)",
+      "got": "not Size(\"1kb\").matches(1)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.7547169811320755
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{\"original\": {\"folder\": {}}}",
+      "got": "{\"original\": {\"folder\": {\"name.txt\": {}}}}",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "(result[0], Path(result[1]))",
+      "got": "result",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.35294117647058826
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "not name.matches(\"beginend\")",
+      "got": "not name.matches(\"begin_somethgin_con\")",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8059701492537313
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "not name.matches(\"herebegin\")",
+      "got": "not name.matches(\"endhere\")",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8571428571428571
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "not has_executable(absent_exe)",
+      "got": "not has_executable(absent_exe)",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "not name.matches(\".beginhere\")",
+      "got": "not name.matches(\"endhere\")",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.9122807017543859
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "Size(\">1k,<1ki\").matches(1010)",
+      "got": "not Size(\">1kb,<1kib\").matches(1000)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8787878787878788
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "Path(\"/test/dst.txt\").resolve()",
+      "got": "\"/test/dst.txt\"",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.6521739130434783
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "not name.matches(\"theendishere\")",
+      "got": "not name.matches(\"here\")",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8571428571428571
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "Path(\"/test/result.txt\").exists()",
+      "got": "Path(\"test/result.txt\").exists()",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.9846153846153847
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "matches_tags({\"gps\": None}, data)",
+      "got": "not matches_tags({\"image.make\": \"NIKON CORPORATION\"}, {})",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.5333333333333333
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "[1, 2, 10, 11, 12, 23, 3, 4, 5, 6]",
+      "got": "[1, 2, 10, 11, 12, 23, 3, 4, 5, 6]",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "\"3e25960a79dbc69b674cd4ec67a72c62\"",
+      "got": "\"Hello world\"",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.1702127659574468
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "[\"Do this\", \"And this\", \"And that\"]",
+      "got": "\"Do this\\nAnd this\\nAnd that\\n\"",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.7575757575757576
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{\"MiXed\": {\"lower\": {filename: \"\"}}}",
+      "got": "{\"upper\": {\"MiXed\": {\"lower\": {filename: \"\"}}}}",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8674698795180723
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "(Path(test_path) / \"foo.txt\").exists()",
+      "got": "Path(test_path).exists()",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.7741935483870968
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "(Path(test_path) / \"bar.txt\").exists()",
+      "got": "(Path(test_path) / \"bar.txt\").exists()",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{\"regex\": {\"the_number\": test_result}}",
+      "got": "{\"the_number\": test_result}",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8307692307692308
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "Path(\"/some/test/folder/dst\").resolve()",
+      "got": "Path(\"/some/test/folder/dst\").resolve()",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{\"test\": {\"sub1\": {\"sub2\": structure}}}",
+      "got": "structure",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.375
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "not matches_tags({\"other\": None}, data)",
+      "got": "not matches_tags({\"gps\": \"0\"}, data)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "hash_first_chunk(long_foo, algo=\"sha1\")",
+      "got": "hash_first_chunk(long_foo)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{\"2021\": {\"01\": {\"31\": {filename: \"\"}}}}",
+      "got": "\"test\"",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.13043478260869565
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "Size([\"<100 Mb\", \">= 0 Tb\"]).matches(20)",
+      "got": "not Size(\"<100 Mb\").matches(10)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.7323943661971831
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "\"7b502c3a1f48c8609ae212cdfb639dee39673f5e\"",
+      "got": "\"b94d27b6973013055a142b97c1c47c74c5000000\"",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.21428571428571427
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "[f\"Date formatting: {datetime.now().year}\"]",
+      "got": "[datetime.now().strftime(\"%Y\")]",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.5135135135135135
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "[\"bar.txt\", \"foo.txt\", \"bar.txt\", \"foo.txt\"]",
+      "got": "[\"bar.txt\", \"foo.txt\"] * 2",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.6285714285714286
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "user_wants_a_folder(\"/test\", autodetect=True)",
+      "got": "not user_wants_a_folder(\"/test\", autodetect=True)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.9574468085106383
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "matches_tags({\"image.make\": \"NIKON *\"}, data)",
+      "got": "not matches_tags({\"image.make\": \"NIKON CORPORATION\"}, {})",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.7843137254901961
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{1: {2: \"x\", 3: 3, 4: {5: \"new.\", 6: \"fin.\"}}}",
+      "got": "{1: {2: \"x\", 3: 3, 4: {5: \"new.\", 6: \"fin.\"}}}",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "user_wants_a_folder(\"/test/\", autodetect=True)",
+      "got": "user_wants_a_folder(\"/test/\")",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.7733333333333333
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "user_wants_a_folder(\"/test/\", autodetect=False)",
+      "got": "user_wants_a_folder(\"foo\")",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.6301369863013698
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "[\"File hash: 3e25960a79dbc69b674cd4ec67a72c62\"]",
+      "got": "\"File hash: 3e25960a79dbc69b674cd4ec67a72c62\\n\"",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.9574468085106383
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "Size(\"<100 Mb, <10 mb, <1 mb, > 0\").matches(20)",
+      "got": "not Size(\">100 Mb\").matches(20)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "Path(\"/some/original/folder/name.txt\").resolve()",
+      "got": "Path(\"/some/original/folder/name.txt\").resolve()",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "not user_wants_a_folder(\"/test\", autodetect=False)",
+      "got": "not user_wants_a_folder(\"/test/\", autodetect=True)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.92
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "user_wants_a_folder(\"/test.asd/\", autodetect=True)",
+      "got": "not user_wants_a_folder(\"/test.asd/\", autodetect=True)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.9615384615384616
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "Path(\"/some/Application.app/info.plist\").resolve()",
+      "got": "Path(\"/some/Application.app/info.plist\").resolve()",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "next_free_name(dst=Path(\"file.txt\"), template=tmp)",
+      "got": "next_free_name(dst=Path(\"file.txt\"), template=tmp)",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "user_wants_a_folder(\"/test.asd/\", autodetect=False)",
+      "got": "user_wants_a_folder(\"/test/\", autodetect=True)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8865979381443299
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "ct.matches_datetime(now - timedelta(days=10, hours=0))",
+      "got": "ct.matches_datetime(now + timedelta(days=10, hours=11))",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.9541284403669725
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "lm.matches_datetime(now - timedelta(days=10, hours=0))",
+      "got": "lm.matches_datetime(now + timedelta(days=10, hours=13))",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.9541284403669725
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "ct.matches_datetime(now - timedelta(days=10, hours=13))",
+      "got": "ct.matches_datetime(now - timedelta(days=10, hours=1))",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.9908256880733946
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "lm.matches_datetime(now - timedelta(days=10, hours=13))",
+      "got": "lm.matches_datetime(now)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.6075949367088608
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "not ct.matches_datetime(now - timedelta(days=10, hours=0))",
+      "got": "not ct.match(now.shift(days=-10, hours=-12))",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.7058823529411765
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "not lm.matches_datetime(now - timedelta(days=10, hours=0))",
+      "got": "not lm(now)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.3188405797101449
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "not ct.matches_datetime(now - timedelta(days=10, hours=13))",
+      "got": "not ct.matches_datetime(now - timedelta(days=10, hours=13))",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "not lm.matches_datetime(now - timedelta(days=10, hours=13))",
+      "got": "not lm.matches_datetime(now - timedelta(days=10, hours=13))",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "matches_tags([\"Invoice (red)\", \"* (green)\"], [\"Invoice (red)\"])",
+      "got": "not matches_tags([\"Invoice (red)\", \"* (green)\"], [\"Urgent (red)\"])",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8837209302325582
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "matches_tags(filter_tags=filter_tags, file_tags=[\"Voice (red)\"])",
+      "got": "not matches_tags(filter_tags=filter_tags, file_tags=[\"Invoice (*)\"])",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.9090909090909091
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "matches_tags([\"Invoice (red)\", \"* (green)\"], [\"Urgent (green)\"])",
+      "got": "not matches_tags(filter_tags=(\"Invoice (red)\", \"* (blue)\"), file_tags=[\"Invoice (red)\"])",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.6578947368421053
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "\"64ec88ca00b268e5ba1a35678a1b5316d212f4f366b2477232534a8aeca37f3c\"",
+      "got": "\"b94d275d4f004067a7596c40040103000000000000000000000000000000000\"",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.183206106870229
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "[ \"100\", \"200\", \"300\", ]",
+      "got": "\"0100\\n0200\\n0300\\n\"",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{ \"a\": 1, \"b\": {\"b1\": 4, \"b2\": 3}, }",
+      "got": "{ \"a\": 1, \"b\": {\"b1\": 2, \"b2\": 3}, \"c\": 6, }",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.875
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "[ \"200 bar\", \"300 baz\", \"100 foo\", ]",
+      "got": "[ \"100 foo\", \"200 bar\", \"300 baz\", ]",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.6944444444444444
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{ \"test.txt\": \"\", \"test.txt.bak\": \"\", }",
+      "got": "\"test.txt\"",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.40816326530612246
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{ \"Datei1.txt\": \"\", \"Datei2.txt\": \"\", }",
+      "got": "{ \"Datei1.txt\": \"\", \"Datei2.txt\": \"\", }",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{ \"reversed.jpg\": \"\", \"filename.txt\": \"\", }",
+      "got": "[\"desrever.jpg\", \"emanelif.txt\"]",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.5866666666666667
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{ \"sub\": { \"foo.txt\": \"\", \"bar.txt\": \"\", } }",
+      "got": "\"test/sub/foo.txt\\n\"",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.375
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{ \"a\": 1, \"b\": {\"b1\": 4, \"b2\": 3, \"b3\": 5}, \"c\": 6, }",
+      "got": "b",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.037037037037037035
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{ \"halffull\": \"0\" * 1010, \"two_thirds.txt\": \"0\" * 666, }",
+      "got": "{\"empty\": \"\", \"two_thirds.txt\": \"666\"}",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.6808510638297872
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "[ f\"Handling: {Path('/test/file.txt')}\", \"File content\", ]",
+      "got": "\"Handling: /test/file.txt\\n\"",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.627906976744186
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{ \"2022-23-03_20220401_173738.txt\": \"Datum: 23.03.2022\", }",
+      "got": "{\"/scan/20220401_173738.txt\": \"Datum: 23.03.2022\"}",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8148148148148148
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "set( [ \"3: 12.08.2017\", \"4: 22.02.2018\", \"5: 08.07.2015\", ] )",
+      "got": "{\"01.01.2000\", \"02.02.2001\"}",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.47191011235955055
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{ \"empty\": \"\", \"halffull\": \"0\" * 1010, \"two_thirds.txt\": \"0\" * 666, }",
+      "got": "{ \"halffull\": \"0\" * 1010 }",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.5473684210526316
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "[ \"Odd student numbers: student-01\", \"Odd student numbers: student-03\", ]",
+      "got": "[\"Odd student numbers: student-01.txt\", \"Odd student numbers: student-03.txt\"]",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.9271523178807947
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{ \"for-test1.txt\": \"WRITE test1.txt\", \"for-test2.txt\": \"WRITE test2.txt\", }",
+      "got": "\"WRITE test1.txt\\nWRITE test2.txt\\n\"",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.5765765765765766
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "[ \"Found JPG file: asd\", \"Found JPG file: camel\", \"Found JPG file: test\", ]",
+      "got": "\"Found JPG file: test.jpg\\n\"",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.42718446601941745
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{ \"src\": {}, \"dst\": { \"dir\": {\"dst.txt\": \"\"}, \"dir 2\": {\"src.txt\": \"\"}, }, }",
+      "got": "{ \"src.txt\": \"src\", \"dst.txt\": \"dst\", \"dst 2.txt\": \"src2\", }",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.2647058823529412
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{ \"File_12082019.pdf\": \"\", \"File_01011999.pdf\": \"\", \"File_01012000.pdf\": \"\", }",
+      "got": "{ \"File_123456_foo.pdf\": \"\", \"File_0812.pdf\": \"\", \"File_0101.pdf\": \"\", }",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.6133333333333333
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{ \"test.pdf\": \"\", \"image\": { \"jpeg\": { \"other.jpg\": \"\", }, \"png\": { \"other2.png\": \"\", }, }, }",
+      "got": "{ \"image\": { \"jpeg\": { \"other.jpg\": \"\" }, \"png\": { \"other2.png\": \"\" }, }, }",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8928571428571429
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{ \"newname.pdf\": \"\", \"newname 2.pdf\": \"\", \"newname 3.pdf\": \"\", \"test.pdf\": \"\", \"asd.txt\": \"\", }",
+      "got": "{\"test.pdf\": \"\", \"newname.pdf\": \"\", \"newname 2.pdf\": \"\"}",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.543046357615894
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{ \"application\": { \"pdf\": { \"test.pdf\": \"\", }, }, \"image\": { \"jpeg\": { \"other.jpg\": \"\", }, }, }",
+      "got": "\"test.pdf\"",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.19047619047619047
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{ \"file.txt\": \"Hello world\\nAnother line\", \"folder\": { \"empty_sub\": {}, }, \"empty_folder\": {}, }",
+      "got": "\"test.txt\"",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.1320754716981132
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{ \"2000\": {\"01\": {\"12\": {\"photo1\": \"\"}}}, \"2020\": {\"01\": {\"01\": {\"photo2\": \"\", \"photo3\": \"\"}}}, }",
+      "got": "[\"photo3\", \"photo2\"]",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.27350427350427353
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{ \"duplicate\": { \"src.txt\": \"src\", }, \"nonduplicate\": {}, \"dst.txt\": \"src\", \"dst 2.txt\": \"src2\", }",
+      "got": "{ \"src.txt\": \"src\", \"dst.txt\": \"src\", \"nonduplicate\": {\"src.txt\": \"src2\"}, }",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{ \"test.txt\": \"\", \"file.txt\": \"Hello world\\nAnother line\", \"another.txt\": \"\", \"folder\": { \"x.txt\": \"\", }, }",
+      "got": "{}",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.03669724770642202
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{ \"newname.pdf\": \"\", \"newname 2.pdf\": \"\", \"test.pdf\": \"\", \"asd.txt\": \"\", \"newfolder\": {\"test-found.pdf\": \"\"}, }",
+      "got": "{\"/test/newfolder/test-found.pdf\": \"\"}",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.4697986577181208
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{ \"dbetts@mail.de.txt\": \"\", \"acornish@google.com.txt\": \"\", \"dbean@aol.com.txt\": \"\", \"l-frey@frey.org.txt\": \"\", }",
+      "got": "{\"/files/dbetts@mail.de.txt\": \"\", \"/files/acornish@google.com.txt\": \"\", \"/files/dbean@aol.com.txt\": \"\", \"/files/l-frey@frey.org.txt\": \"\"}",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8755020080321285
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{ \"unique.txt\": CONTENT_LARGE + \"1\", \"unique_too.txt\": CONTENT_LARGE + \"2\", \"a.txt\": CONTENT_LARGE, \"other\": {}, }",
+      "got": "{ \"unique.txt\": CONTENT_LARGE + \"1\", \"unique_too.txt\": CONTENT_LARGE + \"2\", \"a.txt\": CONTENT_LARGE, \"other\": { \"large.txt\": CONTENT_LARGE, }, }",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8871595330739299
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{ \"folder\": { \"DONE.pdf\": \"\", \"Other\": \"\", }, \"Another folder\": { \"Subfolder\": { \"DONE.txt\": \"\", }, \"Metadata\": \"\", }, }",
+      "got": "{ \"folder\": { \"DONE.FIRST-RENAME.pdf\": \"\", \"Other\": \"\", }, \"Another folder\": { \"Subfolder\": { \"DONE.RENAME-ME_TOO.txt\": \"\", }, \"Metadata\": \"\", }, }",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.898876404494382
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{ \"C\u00e1lculo Integral\": { \"Periodo",
+      "got": "{\"/test/C\u00e1lculo Integral/Periodo",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.78125
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{ \"copy\": { \"19asd_WF_test2_unread.pdf\": \"\", }, \"19asd_WF_test2_unread.pdf\": \"\", \"other.pdf\": \"\", \"18asd_WFX_test2.pdf\": \"\", }",
+      "got": "{ \"19asd_WF_test2_unread.pdf\": \"\", \"other.pdf\": \"\", \"18asd_WFX_test2.pdf\": \"\", }",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.7766990291262136
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "equal_items( Walker(method=method, min_depth=2, max_depth=2).dirs(\"/test/\"), [ Path(\"/test/d1/d1/d1\"), Path(\"/test/d1/d1/d2\"), ], )",
+      "got": "equal_items(Walker(method=method, min_depth=2).dirs(\"/test/\"), [])",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.6700507614213198
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{ \"My Podcast\": {\"Ep 1.mp3\": \"\", \"Ep 2.mp3\": \"\", \"Ep 3.mp3\": \"\"}, \"Your Podcast\": {\"Ep 1.mp3\": \"\", \"Ep 2.mp3\": \"\", \"Ep 3.mp3\": \"\"}, }",
+      "got": "\"test/My Podcast Ep 1.mp3\\n\"",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.2608695652173913
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{ \"src.txt\": \"src\", \"duplicate\": { \"src.txt\": \"src\", }, \"nonduplicate\": { \"src.txt\": \"src2\", }, \"dst.txt\": \"src\", \"dst 2.txt\": \"src2\", }",
+      "got": "{ \"src.txt\": \"src\", \"dst.txt\": \"src\", \"nonduplicate.txt\": \"src2\", }",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.5615763546798029
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{ \"unique.txt\": \"I'm unique.\", \"unique_too.txt\": \"I'm unique: too.\", \"a.txt\": CONTENT_SMALL, \"other\": { \"large.txt\": CONTENT_LARGE, }, }",
+      "got": "{\".\": {\"a.txt\": \"I'm unique.\", \"copy2.txt\": \"I'm unique.\"}}",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.49230769230769234
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "equal_items( Walker(method=method, min_depth=2, max_depth=2).files(\"/test/\"), [ Path(\"/test/d1/d1/f1.txt\"), Path(\"/test/d1/d1/f2.txt\"), ], )",
+      "got": "not Walker(method=method).files(\"/test/d1/d1/d2\")",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.47619047619047616
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{ \"Homework.txt\": \"My Homework ...\", \"MegaCorp_Invoice_12345.txt\": \"Lorem MegaCorp Ltd. ipsum\\nInvoice 12345\\nMore text\\nID: 98765\", \"Test2.txt\": \"Tests\", }",
+      "got": "{ \"Test1.txt\": \"MegaCorp_Invoice_12345.txt\", \"Test2.txt\": \"Tests.txt\", \"Test3.txt\": \"Homework.txt\", }",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.4980544747081712
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{ \"test\": { \"file1.txt\": \"\", \"file2.txt\": \"\", }, \"folder1\": { \"file1.1.txt\": \"\", \"file1.2.txt\": \"\", }, \"folder2\": { \"file1.2.1.txt\": \"\", \"file1.2.2.txt\": \"\", }, }",
+      "got": "\"file1.txt\\nfile2.txt\\n\"",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.21505376344086022
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{ \"Best_Of_Video_1080\": { \"[DVD] Best Of Video 1080 [1080p]\": \"\", \"Metadata\": \"\", }, \"This_Is_A_Title\": { \"[DVD] This Is A Title [1080p]\": \"\", \"Metadata\": \"\", }, }",
+      "got": "{ \"test\": { \"Renaming DVD folders\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\", \"test\": \"\",",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.10706638115631692
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "equal_items( Walker(method=method, min_depth=1, max_depth=2).files(\"/test/\"), [ Path(\"/test/d1/f1.txt\"), Path(\"/test/d1/d1/f1.txt\"), Path(\"/test/d1/d1/f2.txt\"), ], )",
+      "got": "equal_items(Walker(method=method, max_depth=1).files(\"/test/\"), [Path(\"/test/d1/f1.txt\")])",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.6980392156862745
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{ \"test1\": { \"test1.log\": f\"FOUND {Path('loc1/test1.txt')}\\nFOUND {Path('loc2/test1.txt')}\\n\" }, \"test2\": { \"test2.log\": f\"FOUND {Path('loc1/test2.txt')}\\nFOUND {Path('loc2/test2.txt')}\\n\" }, }",
+      "got": "{ \"loc1\": \"FOUND loc1\\n\", \"loc2\": \"FOUND loc2\\n\", }",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.36885245901639346
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "equal_items( Walker().files(\"test\"), [ Path(\"test/folder/file.txt\"), Path(\"test/folder/subfolder/another.pdf\"), Path(\"test/hi/there\"), Path(\"test/hi/.other\"), Path(\"test/.hidden/some.pdf\"), ], )",
+      "got": "equal_items(Counter(\"test/folder/file.txt\"), Counter(\"test/folder/subfolder/another.pdf\"))",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.5563380281690141
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "equal_items( Walker(method=\"depth\").files(\"test\"), [ Path(\"test/folder/subfolder/another.pdf\"), Path(\"test/folder/file.txt\"), Path(\"test/hi/there\"), Path(\"test/hi/.other\"), Path(\"test/.hidden/some.pdf\"), ], )",
+      "got": "not Walker().files(\"test/folder\")",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.024896265560165973
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{ \"test.txt\": \"\", \"file.txt\": \"Hello world\\nAnother line\", \"another.txt\": \"\", \"folder\": { \"x.txt\": \"\", }, \"a brand\": { \"new\": { \"folder\": { \"test.txt\": \"\", \"file.txt\": \"Hello world\\nAnother line\", \"another.txt\": \"\", } } }, }",
+      "got": "{**FILES, \"a brand/new/folder/\": \"\"}",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.1076923076923077
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{ \"test.txt\": \"\", \"file.txt\": \"Hello world\\nAnother line\", \"another.txt\": \"\", \"folder\": { \"x.txt\": \"\", }, \"a brand\": { \"new\": { \"folder\": { \"test.txt\": \"\", \"file.txt\": \"Hello world\\nAnother line\", \"another.txt\": \"\", \"x.txt\": \"\", } } }, }",
+      "got": "{\"/test\": {\"a brand\": {\"new\": {\"folder\": {\"test.txt\": \"\", \"file.txt\": \"Hello world\\nAnother line\"}, \"file.txt\": \"Hello world\\nAnother line\"}, \"file.txt\": \"Hello world\\nAnother line\"}, \"file.txt\": \"Hello world\\nAnother line\"}, \"file.txt\": \"Hello world\\nAnother line\"}",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.31809145129224653
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "[ Path(\"/test/1989/A\"), Path(\"/test/1989/B\"), Path(\"/test/1989/C\"), Path(\"/test/1989/D\"), Path(\"/test/2000/A/1\"), Path(\"/test/2000/A/2\"), Path(\"/test/2000/B/1\"), Path(\"/test/2000/B/2\"), Path(\"/test/2024/001\"), Path(\"/test/2024/002\"), Path(\"/test/2024/003\"), Path(\"/test/2024/004\"), ]",
+      "got": "[Path(\"/test/2000/B\"), Path(\"/test/2000/A\"), Path(\"/test/1989/D\"), Path(\"/test/2024/004\"), Path(\"/test/2024/003\"), Path(\"/test/2024/002\"), Path(\"/test/2024/001\")]",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.32808988764044944
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "{ \"other.pdf\": \"\", \"2019\": { \"Jobs\": { \"CategoryA\": { \"TagB\": { \"V01\": { \"draft\": { \"eng\": { \"2019_Jobs_CategoryA_TagB_A-Media_content-name_V01_draft_eng.docx\": \"\", } } } } } }, \"Work\": { \"CategoryC\": { \"V14\": { \"final\": { \"2019_Work_CategoryC_V-Test_A-Audio_V14_final.pdf\": \"\", } } } }, }, }",
+      "got": "{ \"2019_Jobs_CategoryA_TagB_A-Media_content-name_V01_draft_eng.docx\": \"\", \"2019_Work_CategoryC_V-Test_A-Audio_V14_final.pdf\": \"\", \"other.pdf\": \"\", }",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.5863636363636363
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "ValueError)",
+      "got": "FileExistsError)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.5925925925925926
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "ConfigError)",
+      "got": "ConfigError)",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "testfile)",
+      "got": "tmp_path)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "tfeldmann/organize",
+      "expected": "testfolder)",
+      "got": "testfolder)",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "0",
+      "got": "\"b\"",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "2",
+      "got": "2",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "1",
+      "got": "1",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "4",
+      "got": "3",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "{}",
+      "got": "{\"q1\": False}",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.26666666666666666
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "42",
+      "got": "42",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "[]",
+      "got": "[]",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "\"P\"",
+      "got": "\"python2\"",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "\"t\"",
+      "got": "\"python3\"",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "None",
+      "got": "None",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "True",
+      "got": "True",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "[\"a\"]",
+      "got": "[\"a\"]",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "{\"a\"}",
+      "got": "[\"a\"]",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.6
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "set()",
+      "got": "set()",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "False",
+      "got": "False",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "\"foo\"",
+      "got": "\"foo\"",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "\"bar\"",
+      "got": "\"bar\"",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "\"ghi\"",
+      "got": "\"ghi\"",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "\"jja\"",
+      "got": "\"jag\"",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "\"jkl\"",
+      "got": "\"jag\"",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.6
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "\"abc\"",
+      "got": "\"abc\"",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "\"bob\"",
+      "got": "\"bob\"",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "\"Doe\"",
+      "got": "\"Doe\"",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "\"001\"",
+      "got": "\"001\"",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "\"bazz\"",
+      "got": "\"bazz\"",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "\"World\"",
+      "got": "\"World\"",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "[\"foo\"]",
+      "got": "[\"foo\"]",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "\"write\"",
+      "got": "\"write\"",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "filtered",
+      "got": "filtered",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "[\"bazz\"]",
+      "got": "[\"bar\"]",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "result_py",
+      "got": "False",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.14285714285714285
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "\"python3\"",
+      "got": "\"python3\"",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "\"python2\"",
+      "got": "\"python3\"",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8888888888888888
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "\"baz.any\"",
+      "got": "str(path_completion_tree / \"foo\")",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.14285714285714285
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "\"python123\"",
+      "got": "\"python123\"",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "\"myfile.py\"",
+      "got": "\"myfile.py\"",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "str(excinfo)",
+      "got": "str(excinfo.value)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "\"Hello World\"",
+      "got": "\"Hello World\"",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "\"my password\"",
+      "got": "\"my password\"",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "\"Hello\\nworld\"",
+      "got": "\"Hello\\nworld\"",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "[\"foo\", \"bar\"]",
+      "got": "[\"foo\"]",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "{\"hello\": None}",
+      "got": "{\"hello\": None}",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "[\"foo\", \"bazz\"]",
+      "got": "[\"foo\", \"bazz\"]",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "[\"foo\", \"buzz\"]",
+      "got": "[\"buzz\"]",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.6956521739130435
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "\"Invalid input\"",
+      "got": "\"Value must be exactly 3 characters long.\"",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.21052631578947367
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "\"invalid input\"",
+      "got": "\"invalid input\"",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "expected_tokens",
+      "got": "expected_tokens",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "\"baz\" + test_path",
+      "got": "\"foobar.py\"",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "{\"continue\": False}",
+      "got": "\"no\"",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.2608695652173913
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "\"Polyergus lucidus\"",
+      "got": "\"Polyergus lucidus\"",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "{\"password\": \"asdf\"}",
+      "got": "{\"password\": \"asdf\"}",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "{\"toppings\": [\"foo\"]}",
+      "got": "{\"theme\": \"Ask opening hours\"}",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.43137254901960786
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "result2_shortcut_keys",
+      "got": "result2_shortcut_keys",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "{\"q1\": True, \"q2\": 42}",
+      "got": "42",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.16666666666666666
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "[\"foo\", \"bar\", \"bazz\"]",
+      "got": "[\"foo\", \"bar\", \"bazz\"]",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "[1, \"foo\", [3, \"bar\"]]",
+      "got": "[\"one\", \"two\", \"three\"]",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.5333333333333333
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "{\"phone\": \"1234567890\"}",
+      "got": "{\"phone_number\": \"1234567890\"}",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8679245283018868
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "{\"q1\": True, \"q2\": \"foo\"}",
+      "got": "\"Y\"",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.14285714285714285
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "[\"b\", \"c\", \"args\", \"kwargs\"]",
+      "got": "{'a': None, 'b': 2, 'c': None}",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.1724137931034483
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "{\"ants\": \"Polyergus lucidus\"}",
+      "got": "{\"name\": \"Polyergus lucidus\"}",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.896551724137931
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "{\"theme\": \"Ask opening hours\"}",
+      "got": "{\"separator\": \"\u2022\"}",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.375
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "{\"theme\": \"Make a reservation\"}",
+      "got": "{\"restaurant\": \"McDonald's\"}",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.4067796610169492
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "1000000000000000000000000000001",
+      "got": "100",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.17647058823529413
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "[\"a\", \"b\", \"c\", \"args\", \"kwargs\"]",
+      "got": "[\"a\", \"b\", \"c\"]",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.625
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "str(path_completion_tree / \"baz\")",
+      "got": "\"ba\"",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.21621621621621623
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "str(path_completion_tree / \"baz\" / \"foo\")",
+      "got": "\"foo\"",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.21739130434782608
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "str(path_completion_tree / \"foo\" / \"buz\")",
+      "got": "str(path_completion_tree / \"foo\")",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8918918918918919
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "response == \"World\" and not response == 42",
+      "got": "response == \"World\"",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.6229508196721312
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "{ \"key\": \"key2\", \"value\": \"k2v2\", }",
+      "got": "{\"theme\": \"Ask opening hours\"}",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.3384615384615385
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "len( InquirerControl.SHORTCUT_KEYS )",
+      "got": "99",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "{ \"intro\": None, \"conditional_step\": True, \"next_question\": \"questionary\", \"second_question\": \"Hello World\", }",
+      "got": "{\"question\": \"Hello World\"}",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.39416058394160586
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "RuntimeError)",
+      "got": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "ValidationError)",
+      "got": "ValidationError)",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "KeyboardInterrupt)",
+      "got": "KeyboardInterrupt)",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "PromptParameterException)",
+      "got": "PromptParameterException)",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmbo/questionary",
+      "expected": "[call(\"Hello World\"), call(\"\\r\\n\")])",
+      "got": "[call(\"Hello World\")])",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.7586206896551724
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "5",
+      "got": "5",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "1",
+      "got": "1",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "2",
+      "got": "2",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "9",
+      "got": "9",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "6",
+      "got": "6",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "3",
+      "got": "3",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "14",
+      "got": "14",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "{}",
+      "got": "{\"match\": (\"Privet\",)}",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.16666666666666666
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "42",
+      "got": "42",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "11",
+      "got": "11",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "100",
+      "got": "100",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "msg",
+      "got": "\"full message text with all attachments\"",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.13953488372093023
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "999",
+      "got": "999",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "rev",
+      "got": "None",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "None",
+      "got": "None",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "True",
+      "got": "False",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.2222222222222222
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "False",
+      "got": "None",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.2222222222222222
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "\"test\"",
+      "got": "\"test\"",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "\"\u041f\u0430\u0432\u0435\u043b\"",
+      "got": "\"\u041f\u0430\u0432\u0435\u043b\"",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "-123456",
+      "got": "-123456",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "\"token\"",
+      "got": "\"token\"",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "types()",
+      "got": "types().code",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.7368421052631579
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "\"12345\"",
+      "got": "\"123\"",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8333333333333334
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "CodeError1",
+      "got": "CodeError[1]",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.9090909090909091
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "CodeError42",
+      "got": "CodeError42",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "\"some text\"",
+      "got": "\"some text\"",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "\"some token\"",
+      "got": "\"some token\"",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "CodeError[2]",
+      "got": "CodeException",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.48
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "KEYBOARD_JSON",
+      "got": "KEYBOARD_JSON",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "TEMPLATE_DICT",
+      "got": "TEMPLATE_DICT",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "\"task_to_run\"",
+      "got": "\"task_to_run\"",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "\"239633676097\"",
+      "got": "\"239633676097\"",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "expected_error",
+      "got": "expected_error",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "task1.__name__",
+      "got": "\"task1\"",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.47619047619047616
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "{\"response\": 1}",
+      "got": "1",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.125
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "bot.polling.api",
+      "got": "await bot.api.token_generator.get_api()",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.25925925925925924
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "validation_type",
+      "got": "validation_type",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "expected_response",
+      "got": "expected_response",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "expected_exception",
+      "got": "expected_exception",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "not required_files",
+      "got": "not required_files",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "\"+7 *** *** ** 89\"",
+      "got": "\"+7 *** *** ** 89\"",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "FirstMockState.MOCK",
+      "got": "FirstMockState.MOCK",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "not e.value.ban_info",
+      "got": "e.value.error_msg.startswith(\"User authorization failed\")",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.23376623376623376
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "{\"a\": \"1,2,3,4,hi!\"}",
+      "got": "{\"a\": [1, 2, 3, 4, \"hi!\"]}",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8260869565217391
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "self.recent_response",
+      "got": "self.recent_response",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "SecondMockState.MOCK",
+      "got": "first_mock_state.state",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.38095238095238093
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "\"FirstMockState:mock\"",
+      "got": "\"FirstMockState:mock\"",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "bot.router.views[\"raw\"]",
+      "got": "bot.router.views[\"raw_event\"]",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8846153846153846
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "formatter_1.format_data",
+      "got": "formatter_1.format_data",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "formatter_2.format_data",
+      "got": "formatter_2.format_data",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "formatter_3.format_data",
+      "got": "formatter_3.format_data",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "EXAMPLE_EVENT[\"updates\"]",
+      "got": "[]",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.15384615384615385
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "{\"unexpected_kwarg\": 123}",
+      "got": "{\"unexpected_kwarg\": 123}",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "isinstance(response, list)",
+      "got": "isinstance(response, list)",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "isinstance(error, Derived)",
+      "got": "isinstance(error, Base)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8163265306122449
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "issubclass(error_1, CodeError)",
+      "got": "isinstance(error_1, CodeError)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "OrRule(c_rule(None)).__class__",
+      "got": "OrRule",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "isinstance(error, CodeError[1])",
+      "got": "isinstance(error, CodeError)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.9491525423728814
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "API_REQUEST.replace(\"%A%\", \"1\")",
+      "got": "API_REQUEST.replace(\"%A%\", \"1\")",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "BASIC_CYCLE.replace(\"%A%\", \"10\")",
+      "got": "\"BASIC_CYCLE\"",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.5333333333333333
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "isinstance(e.value, APIAuthError)",
+      "got": "isinstance(e.value, APIAuthError)",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "{\"response\": {\"some\": \"response\"}}",
+      "got": "{\"some\": \"response\"}",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.7407407407407407
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "issubclass(CodeError[6], CodeError)",
+      "got": "isinstance(CodeError1[1], CodeError1)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.7777777777777778
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "isinstance(response[\"response\"], list)",
+      "got": "isinstance(response, dict)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "f'{{\"type\":\"open_link\",\"link\":\"{some_str}\"}}'",
+      "got": "f'{{\"type\":\"open_link\",\"link\":\"{some_str}\"}}'",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "not await EqualsValidator(\"foo\").check(\"bar\")",
+      "got": "not await IsInstanceValidator((int, str)).check(1)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.631578947368421
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "await CallableValidator(lambda _: True).check(0)",
+      "got": "not await EqualsValidator(\"foo\").check(\"foo\")",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.5806451612903226
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "f'{{\"type\":\"show_snackbar\",\"text\":\"{some_str}\"}}'",
+      "got": "f'{{\"text\": \"{some_str}\"}}'",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.6842105263157895
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "await IsInstanceValidator((int, str)).check(\"foo\")",
+      "got": "await EqualsValidator(\"test\").validate(\"test\")",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.4583333333333333
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "\"One of the parameters specified was missing or invalid: sid is wrong\"",
+      "got": "\"One of the parameters specified was missing or invalid: sid is wrong\"",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "f'{{\"type\":\"open_app\",\"owner_id\":null,\"app_id\":{some_int},\"hash\":\"{some_str}\"}}'",
+      "got": "f'{{\"type\":\"open_app\",\"app_id\":{some_int},\"hash\":\"{some_str}\"}}'",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8888888888888888
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "[ task2.__name__, task3.__name__, ]",
+      "got": "[\"task1\", \"task2\", \"task3\"]",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.4838709677419355
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "{ \"version\": Formatter.VERSION, \"items\": [{\"type\": \"bold\", \"offset\": 0, \"length\": 3}], }",
+      "got": "\"Wow\"",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.06451612903225806
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "{ \"version\": Formatter.VERSION, \"items\": [{\"type\": \"bold\", \"offset\": 0, \"length\": 5}], }",
+      "got": "{ \"version\": Formatter.VERSION, \"items\": [{\"type\": \"bold\", \"offset\": 0, \"length\": 3}, {\"type\": \"emoji\", \"offset\": 3, \"length\": 2}], }",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.7873303167420814
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "{ \"version\": Formatter.VERSION, \"items\": [{\"type\": \"bold\", \"offset\": 4, \"length\": 5}], }",
+      "got": "{ \"version\": Formatter.VERSION, \"items\": [{\"type\": \"bold\", \"offset\": 6, \"length\": 5}], }",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.9886363636363636
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "{ \"version\": Formatter.VERSION, \"items\": [{\"type\": \"italic\", \"offset\": 0, \"length\": 8}], }",
+      "got": "{ \"version\": Formatter.VERSION, \"items\": [{\"type\": \"italic\", \"offset\": 0, \"length\": 10}], }",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.9834254143646409
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "{ \"version\": Formatter.VERSION, \"items\": [{\"type\": \"italic\", \"offset\": 7, \"length\": 6}], }",
+      "got": "{ \"version\": Formatter.VERSION, \"items\": [{\"type\": \"italic\", \"offset\": 0, \"length\": 6}], }",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.9888888888888889
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "{ \"version\": Formatter.VERSION, \"items\": [{\"type\": \"italic\", \"offset\": 2, \"length\": 4}], }",
+      "got": "{ \"version\": Formatter.VERSION, \"items\": [{\"type\": \"italic\", \"offset\": 0, \"length\": 5}], }",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.9777777777777777
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "{ \"version\": Formatter.VERSION, \"items\": [{\"type\": \"underline\", \"offset\": 13, \"length\": 11}], }",
+      "got": "{ \"version\": Formatter.VERSION, \"items\": [{\"type\": \"underline\", \"offset\": 0, \"length\": 10}], }",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.9735449735449735
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "{ \"version\": Formatter.VERSION, \"items\": [ {\"type\": \"bold\", \"offset\": 0, \"length\": 7}, {\"type\": \"italic\", \"offset\": 7, \"length\": 6}, ], }",
+      "got": "{ \"version\": Formatter.VERSION, \"items\": [ {\"type\": \"bold\", \"offset\": 0, \"length\": 6}, {\"type\": \"italic\", \"offset\": 7, \"length\": 5}, ], }",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.9854014598540146
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "{ \"version\": Formatter.VERSION, \"items\": [ {\"type\": \"bold\", \"offset\": 0, \"length\": 5}, {\"type\": \"italic\", \"offset\": 7, \"length\": 6}, ], }",
+      "got": "{ \"version\": Formatter.VERSION, \"items\": [ {\"type\": \"bold\", \"offset\": 0, \"length\": 5}, {\"type\": \"italic\", \"offset\": 10, \"length\": 6}, {\"type\": \"italic\", \"offset\": 16, \"length\": 6}, ], }",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.84472049689441
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "VKAPIError)",
+      "got": "CaptchaError)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "APIAuthError)",
+      "got": "UserAuthError)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.7407407407407407
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "MiddlewareError)",
+      "got": "MiddlewareError)",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "VKAPIError[100])",
+      "got": "APIAuthError)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.6206896551724138
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "ValueError, match=\"some_value_error\")",
+      "got": "MiddlewareError)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.3018867924528302
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "ValueError, match=\"Context update value should be an instance of dict\")",
+      "got": "MiddlewareError)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.1839080459770115
+    },
+    {
+      "repo": "vkbottle/vkbottle",
+      "expected": "True)",
+      "got": "False)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.36363636363636365
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "1",
+      "got": "2",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "0",
+      "got": "0",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "2",
+      "got": "2",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "3",
+      "got": "4",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "4",
+      "got": "3",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "{}",
+      "got": "{}",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "[]",
+      "got": "[]",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "()",
+      "got": "()",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "b''",
+      "got": "b''",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "str",
+      "got": "str",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "int",
+      "got": "int",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "iss",
+      "got": "iss",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "None",
+      "got": "None",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "True",
+      "got": "True",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "auth",
+      "got": "isinstance(auth, list)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.3076923076923077
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "False",
+      "got": "True",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.2222222222222222
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "['a']",
+      "got": "['a']",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "props",
+      "got": "props",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "[1, 2]",
+      "got": "[1, 2]",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "parser",
+      "got": "isinstance(parser, XmlParser)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.34285714285714286
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "header",
+      "got": "header",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "regular",
+      "got": "regular",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "headers",
+      "got": "headers",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "'field'",
+      "got": "'field'",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "'HS256'",
+      "got": "'HS256'",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "expected",
+      "got": "expected",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "username",
+      "got": "username",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "ref2.ref",
+      "got": "ref2.ref",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "'authed'",
+      "got": "{'message': 'authed'}",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.5517241379310345
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "audience",
+      "got": "audience",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "last_name",
+      "got": "last_name",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "field_map",
+      "got": "field_map",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "request_id",
+      "got": "request_id",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "['a', 'b']",
+      "got": "['a', 'b']",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "controller",
+      "got": "controller",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "{'detail'}",
+      "got": "{'detail'}",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "status_code",
+      "got": "status_code",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "snapshot(1)",
+      "got": "{'code': 1}",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.18181818181818182
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "'text/html'",
+      "got": "'text/html'",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "_MIN_LENGTH",
+      "got": "_MIN_LENGTH",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "default_val",
+      "got": "default_val",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "schema_type",
+      "got": "schema_type",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "_TEST_SCHEMA",
+      "got": "_TEST_SCHEMA",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "request_data",
+      "got": "request_data",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "real_request",
+      "got": "isinstance(response.content, bytes)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0851063829787234
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "('', (), {})",
+      "got": "('users/', (), {})",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "'not authed'",
+      "got": "'not authed'",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.OK",
+      "got": "HTTPStatus.OK",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "'CustomValue'",
+      "got": "'Custom header value'",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.7058823529411765
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "(_BodyModel,)",
+      "got": "_ModelT",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.6
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "_RegularAlias",
+      "got": "_list[int]",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.34782608695652173
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "'value_error'",
+      "got": "'invalid'",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.45454545454545453
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "expected_data",
+      "got": "expected_data",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "operation_ids",
+      "got": "operation_ids",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "initial_count",
+      "got": "initial_count",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "response_code",
+      "got": "response_code",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "refresh['jti']",
+      "got": "refresh['jti']",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "list(expected)",
+      "got": "expected",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.7272727272727273
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "'none handled'",
+      "got": "{}",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "['card', 'big']",
+      "got": "json.loads(_XML_DATA)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.1111111111111111
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "ContentType.xml",
+      "got": "ContentType.xml",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "expected_status",
+      "got": "expected_status",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "'Error message'",
+      "got": "{'message': 'Error message'}",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.6976744186046512
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "request.headers",
+      "got": "request.headers",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "snapshot(['a'])",
+      "got": "['a']",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "{HTTPStatus.OK}",
+      "got": "{HTTPStatus.OK}",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "expected_output",
+      "got": "expected_output",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "expected_tokens",
+      "got": "expected_tokens",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "'customGetUser'",
+      "got": "'customGetUser'",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "expected_schema",
+      "got": "expected_schema",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "response.closed",
+      "got": "response.closed",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "ContentType.json",
+      "got": "str(ContentType.json)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8648648648648649
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "_OPENAPI_VERSION",
+      "got": "f'3.0.{_OPENAPI_VERSION}'",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.7804878048780488
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.FOUND",
+      "got": "HTTPStatus.FOUND",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "{'user': 'name'}",
+      "got": "{'user': 'name'}",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "expected_summary",
+      "got": "expected_summary",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "response.content",
+      "got": "response.content",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "expected_headers",
+      "got": "expected_headers",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "{'key': 'value'}",
+      "got": "{'key': 'value'}",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "expected_content",
+      "got": "expected_content",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "'{\"key\":\"value\"}'",
+      "got": "'{\"key\": \"value\"}'",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.9714285714285714
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.CREATED",
+      "got": "HTTPStatus.CREATED",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "'application/json'",
+      "got": "'text/plain'",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.4666666666666667
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "self.request.FILES",
+      "got": "self.request.FILES",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "PydanticSerializer",
+      "got": "PydanticSerializer",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "{'result': 'done'}",
+      "got": "{'result': 'done'}",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "f'/api/{path_str}'",
+      "got": "'/api/'",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.56
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "len(operation_ids)",
+      "got": "len(operation_ids)",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "[{'noScheme': []}]",
+      "got": "[]",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.2
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "OpenAPIFormat.IPV4",
+      "got": "'ipv4'",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.16666666666666666
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "response.streaming",
+      "got": "isinstance(response.content, bytes)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.37735849056603776
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.CONFLICT",
+      "got": "HTTPStatus.BAD_REQUEST",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.5853658536585366
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.ACCEPTED",
+      "got": "202",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "['first', 'second']",
+      "got": "['first', 'second']",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "OpenAPIType.INTEGER",
+      "got": "'integer'",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.14285714285714285
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "request_data.keys()",
+      "got": "request_data.keys()",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "str(exc_info.value)",
+      "got": "str(exc_info.value)",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "not response.closed",
+      "got": "isinstance(response.content, AsyncIterator)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.45161290322580644
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "not response.cookies",
+      "got": "not response.cookies[settings.SESSION_COOKIE_NAME]",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.5714285714285714
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "OpenAPIFormat.BINARY",
+      "got": "\"binary\"",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.14285714285714285
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "b'Example receipt\\n'",
+      "got": "b'Hello World\\n'",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.4444444444444444
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "expected_description",
+      "got": "expected_description",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "[{'testScheme': []}]",
+      "got": "[{'testScheme': []}]",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.FORBIDDEN",
+      "got": "HTTPStatus.NOT_ACCEPTABLE",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.5333333333333333
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.NOT_FOUND",
+      "got": "404",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "expected_content_type",
+      "got": "expected_content_type",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "ErrorType.value_error",
+      "got": "'value_error'",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.6470588235294118
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "{'username': 'admin'}",
+      "got": "{'username': 'admin'}",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "not response.is_async",
+      "got": "response.has_header('Content-Disposition')",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.38095238095238093
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.NO_CONTENT",
+      "got": "HTTPStatus.NO_CONTENT",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "auth[0].error_message",
+      "got": "_TestSyncAuth.error_message",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.7083333333333334
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "snapshot({'jwt': []})",
+      "got": "['jwt']",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.42857142857142855
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "user.is_authenticated",
+      "got": "user",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.32
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.BAD_REQUEST",
+      "got": "HTTPStatus.CREATED",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.7
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "{'X-API-Token': '123'}",
+      "got": "snapshot({})",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.11764705882352941
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "snapshot({'field': 1})",
+      "got": "snapshot({'field': '1'})",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.9565217391304348
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.IM_A_TEAPOT",
+      "got": "HTTPStatus.IM_A_TEAPOT",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "composed.api_endpoints",
+      "got": "composed.api_endpoints",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.UNAUTHORIZED",
+      "got": "HTTPStatus.UNAUTHORIZED",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "schema_registry.schemas",
+      "got": "schema_registry._type_map",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.7916666666666666
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "registry._operation_ids",
+      "got": "registry",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.5161290322580645
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "snapshot(b'{\"a\": \"b\"}')",
+      "got": "json.dumps({'root': {'a': 'b'}})",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "scheme.security_schemes",
+      "got": "scheme.name",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.5294117647058824
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "b'data: simple\\r\\n\\r\\n'",
+      "got": "b'simple\\r\\n\\r\\n'",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.85
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "{'full_name': 'Example'}",
+      "got": "{'full_name': 'Example'}",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "isinstance(raw_data, str)",
+      "got": "isinstance(raw_data, str)",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "response.json()['detail']",
+      "got": "response.json()",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "_Controller.api_endpoints",
+      "got": "_Controller.api_endpoints",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.NOT_ACCEPTABLE",
+      "got": "406",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "_SerializerContextSubclass",
+      "got": "_CustomSerializerContextController",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.6333333333333333
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "isinstance(schema, Schema)",
+      "got": "isinstance(schema, Schema)",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "isinstance(ref, Reference)",
+      "got": "isinstance(ref, Reference)",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.PAYMENT_REQUIRED",
+      "got": "HTTPStatus.PAYMENT_REQUIRED",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "isinstance(ref1, Reference)",
+      "got": "ref1.ref == ref2.ref",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.2978723404255319
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "f'{first_name} {last_name}'",
+      "got": "{'first_name': first_name, 'last_name': last_name}",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.5714285714285714
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "endpoint.metadata.responses",
+      "got": "validator.status_codes",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.TOO_MANY_REQUESTS",
+      "got": "HTTPStatus.TOO_MANY_REQUESTS",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "_Mixed.no_validate_http_spec",
+      "got": "not _Mixed.empty_response_body",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.4482758620689655
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "json.loads(response.content)",
+      "got": "isinstance(json.loads(response.content), typ)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.7671232876712328
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "_TestComponent.error_message",
+      "got": "'From component'",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.36363636363636365
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "snapshot({'http_basic': []})",
+      "got": "['basic']",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.43243243243243246
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.METHOD_NOT_ALLOWED",
+      "got": "405",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "_GoodController.api_endpoints",
+      "got": "_GoodController._metadata['methods']",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.6153846153846154
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "isinstance(schema, Reference)",
+      "got": "not schema",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.41025641025641024
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "_MetaController.api_endpoints",
+      "got": "dmr_rf",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.05714285714285714
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "description.metadata.responses",
+      "got": "description.metadata.responses",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "isinstance(scheme, Components)",
+      "got": "isinstance(scheme, Components.SecurityScheme)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "b'\"%s\"' % email.encode('utf8')",
+      "got": "email",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "{'query': expected_query_value}",
+      "got": "expected_query_value",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.7843137254901961
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.UNPROCESSABLE_ENTITY",
+      "got": "400",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "{'username': 'validated-admin'}",
+      "got": "{'validated-admin'}",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.76
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "_NamedModel.__dmr_schema_name__",
+      "got": "'SchemaName'",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.37209302325581395
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "{'email': email, 'age': user_id}",
+      "got": "{'email': email}",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "TypeError)",
+      "got": "UnsolvableAnnotationsError)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.3783783783783784
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "NotImplementedError)",
+      "got": "err)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "InternalServerError)",
+      "got": "InternalServerError)",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "EndpointMetadataError)",
+      "got": "EndpointMetadataError)",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "TypeError, match='put')",
+      "got": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.4117647058823529
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "RuntimeError, match=match)",
+      "got": "RuntimeError)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "expected, match='unhandled')",
+      "got": "expected)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.4864864864864865
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "ValidationError, match='type')",
+      "got": "ValidationError)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.6956521739130435
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "RuntimeError, match='from put')",
+      "got": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.38095238095238093
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "ValueError, match='status_code')",
+      "got": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.5116279069767442
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "EndpointMetadataError, match=msg)",
+      "got": "TypeError, match=msg)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.7037037037037037
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "ZeroDivisionError, match='custom')",
+      "got": "ZeroDivisionError)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.6923076923076923
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "EndpointMetadataError, match='403')",
+      "got": "EndpointMetadataError)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.7719298245614035
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "EndpointMetadataError, match='GET')",
+      "got": "EndpointMetadataError, match='_get')",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.9014084507042254
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "EndpointMetadataError, match='gEt')",
+      "got": "EndpointMetadataError, match='POST')",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.9014084507042254
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "EndpointMetadataError, match='_get')",
+      "got": "EndpointMetadataError)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.7586206896551724
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "EndpointMetadataError, match='>= 2')",
+      "got": "EndpointMetadataError)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.7586206896551724
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "EndpointMetadataError, match='POST')",
+      "got": "EndpointMetadataError)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.7586206896551724
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "ValueError, match='a single string')",
+      "got": "InternalServerError)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.32142857142857145
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "EndpointMetadataError, match='query')",
+      "got": "EndpointMetadataError, match='_get')",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.9041095890410958
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "EndpointMetadataError, match='`None`')",
+      "got": "EndpointMetadataError)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.7333333333333333
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "EndpointMetadataError, match='@modify')",
+      "got": "EndpointMetadataError)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.7213114754098361
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "ValueError, match='already registered')",
+      "got": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.44
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "EndpointMetadataError, match='OPTIONS')",
+      "got": "EndpointMetadataError)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.7213114754098361
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "UnsolvableAnnotationsError, match='get')",
+      "got": "UnsolvableAnnotationsError)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8059701492537313
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "EndpointMetadataError, match=str(status))",
+      "got": "EndpointMetadataError, match='return `None` not')",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.7333333333333333
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "EndpointMetadataError, match=header_name)",
+      "got": "EndpointMetadataError)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.6984126984126984
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "EndpointMetadataError, match='@validate')",
+      "got": "EndpointMetadataError)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.6984126984126984
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "EndpointMetadataError, match='NewHeader')",
+      "got": "EndpointMetadataError, match='@validate')",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8536585365853658
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "EndpointMetadataError, match='NewCookie')",
+      "got": "EndpointMetadataError, match='NewCookie')",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "EndpointMetadataError, match='XmlParser')",
+      "got": "EndpointMetadataError)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.6984126984126984
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "ValueError, match=r'Annotated\\[YourType')",
+      "got": "TypeError)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.27450980392156865
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "DisallowedRedirect, match='3xx statuses')",
+      "got": "DisallowedRedirect, match='with status code')",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8604651162790697
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "ValueError, match='current or past time')",
+      "got": "ValueError, match='datetime in the past')",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.6829268292682927
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "EndpointMetadataError, match='HeaderSpec')",
+      "got": "EndpointMetadataError, match='@modify')",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8148148148148148
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "EndpointMetadataError, match='CookieSpec')",
+      "got": "EndpointMetadataError, match='CookieSpec')",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "EndpointMetadataError, match='Set-Cookie')",
+      "got": "EndpointMetadataError)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.6875
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "DisallowedRedirect, match='with protocol')",
+      "got": "DisallowedRedirect, match='Unsafe redirect to')",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.7415730337078652
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "EndpointMetadataError, match='custom_name')",
+      "got": "EndpointMetadataError, match='query')",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.85
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "UnsolvableAnnotationsError, match=r'\\.get')",
+      "got": "UnsolvableAnnotationsError)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.7714285714285715
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "ValueError, match='datetime in the future')",
+      "got": "ValueError, match='must be in the future')",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8705882352941177
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "UnsolvableAnnotationsError, match='given 0')",
+      "got": "UnsolvableAnnotationsError)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.7605633802816901
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "UnsolvableAnnotationsError, match='_Custom')",
+      "got": "UnsolvableAnnotationsError)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.7605633802816901
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "EndpointMetadataError, match=_MATCH_PATTERN)",
+      "got": "EndpointMetadataError)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "EndpointMetadataError, match='at least one')",
+      "got": "EndpointMetadataError)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "None)",
+      "got": "_PathModel)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.375
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HttpRequest)",
+      "got": "HttpRequest)",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "2)",
+      "got": "2)",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "1)",
+      "got": "2)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "[])",
+      "got": "[])",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "200)",
+      "got": "200)",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "302)",
+      "got": "302)",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "\"First\")",
+      "got": "\"Second\")",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.35294117647058826
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "\"mypanel\")",
+      "got": "\"my_panel\")",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.9523809523809523
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "\"a_b_c_d\")",
+      "got": "\"a_b_c_d\")",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "\"my_panel\")",
+      "got": "\"my_panel\")",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "\"my-panel\")",
+      "got": "\"my-panel\")",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "\"dashboard\")",
+      "got": "\"dashboard\")",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "\"custom_app\")",
+      "got": "\"custom_app\")",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "{\"p1\", \"p2\"})",
+      "got": "{\"p1\", \"p2\"})",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "\"explicit_app\")",
+      "got": "\"explicit_app\")",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "\"Minimal Panel\")",
+      "got": "\"Minimal Panel\")",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "\"dj_cache_panel\")",
+      "got": "\"dj_cache_panel\")",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "\"dist_derived_id\")",
+      "got": "\"dist_derived_id\")",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "\"explicit-package\")",
+      "got": "\"explicit-package\")",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "\"my-custom-package\")",
+      "got": "\"my-custom-package\")",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "p)",
+      "got": "p)",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "ids)",
+      "got": "ids)",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "[302, 403])",
+      "got": "[302, 403])",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "data[\"url\"])",
+      "got": "data[\"url\"])",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "response.url)",
+      "got": "response.url)",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "str(ctx.exception))",
+      "got": "str(ctx.exception))",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "panel)",
+      "got": "panel._registry_id)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.48
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "status[\"installed_panel\"])",
+      "got": "status[\"installed_panel\"])",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "self.reg.get_panel(\"new_key\"))",
+      "got": "self.reg.get_panel(\"new_key\"))",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "self.reg.get_panel(\"panel_a\"))",
+      "got": "self.reg.get_panel(\"panel_a\"))",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "self.reg.get_panel(\"panel_b\"))",
+      "got": "self.reg.get_panel(\"panel_b\"))",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "status[\"installed_panel\"])",
+      "got": "status[\"error\"])",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.5714285714285714
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "self.reg.get_panel(\"some_old_id\"))",
+      "got": "self.reg.get_panel(\"old_id\"))",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.9206349206349206
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "self.reg.get_panel(\"does_not_exist\"))",
+      "got": "self.reg.get_panel(\"unknown\"))",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.746268656716418
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "len(not_installed) > 0)",
+      "got": "not_installed)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.7567567567567568
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "status[\"pip_installed\"])",
+      "got": "status[\"pip_installed\"])",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "status[\"in_installed_apps\"])",
+      "got": "status[\"in_installed_apps\"])",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "self.reg.is_registered(\"p1\"))",
+      "got": "self.reg.is_registered(\"p1\"))",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "should_register_panel_admin())",
+      "got": "should_register_panel_admin())",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "self.reg.is_registered(\"no_id_panel\"))",
+      "got": "self.reg.is_registered(\"no_id_panel\"))",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "should_register_panel_admin(\"my_panel\"))",
+      "got": "should_register_panel_admin(\"my_panel\"))",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "should_register_panel_admin(\"some_other_panel\"))",
+      "got": "should_register_panel_admin(\"unknown_panel\"))",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8387096774193549
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "p[\"configured\"])",
+      "got": "p[\"installed\"])",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.5806451612903226
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "self.reg._discovered)",
+      "got": "self.reg._discovered)",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "status[\"pip_installed\"])",
+      "got": "status[\"installed\"])",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.9090909090909091
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "status[\"is_configured\"])",
+      "got": "status[\"is_featured\"])",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8260869565217391
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "self.reg._panels.get(\"p1\"))",
+      "got": "self.reg._discovered)",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.5416666666666666
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "should_register_panel_admin())",
+      "got": "should_register_panel_admin(\"my_panel\"))",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8571428571428571
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "self.reg.is_registered(\"absent\"))",
+      "got": "self.reg.is_registered(\"does_not_exist\"))",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.8108108108108109
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "should_register_panel_admin(\"my_panel\"))",
+      "got": "should_register_panel_admin(\"my_panel\"))",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "should_register_panel_admin(\"some_other_panel\"))",
+      "got": "should_register_panel_admin(\"some_other_panel\"))",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "\"my_panel:index\")",
+      "got": "\"my-panel\")",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.7142857142857143
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "ids)",
+      "got": "ids)",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "yassi/dj-control-room",
+      "expected": "data[\"url\"])",
+      "got": "data[\"url\"])",
+      "exact_match": true,
+      "code_bleu": 0.0,
+      "edit_similarity": 1.0
+    }
+  ]
+}
\ No newline at end of file