|
|
| from grader import is_equal |
| import json |
| import re |
|
|
|
|
| def get_gold_list(datapath, dataset_name): |
|
|
| assert dataset_name in ["gsm8k", "math", "minerva_math", "gaokao2023en", "olympiadbench", "collegemath"] |
|
|
| gold_list = [] |
| with open(datapath, "r") as f: |
| for line in f: |
| item = json.loads(line) |
|
|
| if dataset_name == "gsm8k": |
| gold = item['answer'].split("#### ")[-1] |
|
|
| elif dataset_name == "math": |
| gold = item['answer'] |
|
|
| elif dataset_name == "minerva_math": |
| pattern = r"\\boxed\{((?:[^{}]|\{(?:[^{}]|\{[^{}]*\})*\})*)\}" |
| pattern_re = re.compile(pattern, re.DOTALL) |
| solution = item['solution'] |
| matches = pattern_re.findall(solution) |
| if len(matches) == 0: |
| gold = None |
| else: |
| gold = matches[-1] |
| |
| elif dataset_name == "gaokao2023en": |
| gold = re.sub(r'^\$(.*)\$$', r'\1', item['answer']) |
|
|
| elif dataset_name == "olympiadbench": |
| gold = re.sub(r'^\$(.*)\$$', r'\1', item['final_answer'][0]) |
| |
| elif dataset_name == "collegemath": |
| gold = re.sub(r'^\$(.*)\$$', r'\1', item['answer']) |
|
|
| gold_list.append(gold) |
|
|
| return gold_list |
|
|
|
|
| def get_scores_on_math_benchmarks(model_output_path, test_gold_path, dataset_name): |
| |
| gold_list = get_gold_list(test_gold_path, dataset_name) |
|
|
| """TODO |
| Get the output_list from model_output_path |
| output_list is a list of string (List[str]) |
| Each string represents the model's response for a corresponding question in the benchmark |
| Therefore, the length of output_list must match the length of gold_list. |
| |
| output_list = ... |
| """ |
|
|
| correct = 0 |
| for output, gold in zip(output_list, gold_list): |
| if is_equal(output, gold, dataset_name): |
| correct += 1 |
|
|
| print("accuracy on %s is %.4f" % (dataset_name, correct / len(gold_list))) |
|
|
|
|
| if __name__ == "__main__": |
| """TODO |
| Download test benchmarks from Qwen2.5-Math |
| https://github.com/QwenLM/Qwen2.5-Math/tree/main/evaluation/data |
| |
| Prepare model_output_path and test_gold_path for each dataset |
| """ |
|
|
| test_gold_path = "PATH_OF_THE_BENCHMARK" |
| model_output_path = "PATH_OF_YOUR_MODEL_OUTPUTS" |
| dataset_name = "DATASET_NAME" |
|
|
| get_scores_on_math_benchmarks(model_output_path, test_gold_path, dataset_name) |
|
|
