testcase-data/get_rank_result_rank_all.py

import json

import random

def get_random_indices(array_length, num_indices):
    # 确保抽取的数量不超过数组长度
    if num_indices > array_length:
        return random.sample(range(array_length), array_length)

    # 使用 random.sample 抽取指定数量的索引
    return random.sample(range(array_length), num_indices)

def find_first_non_ac(array):
    for element in array:
        if element != "AC":
            return element
    return "AC" 

test_als = ["lcb","ht","algo","crux","predo"]
# test_als = ["lcb"]
model_name_list = [
    "claude-sonnet-4-20250514-thinking",
    "deepseek-v3",
    "qwen3-nothink",
    "claude4",
    "gpt-4o",
    "qwen-coder-plus",
    "Qwen2.5-7B-Instruct",
    "Qwen2.5-14B-Instruct",
    "Qwen2.5-32B-Instruct",
    "Qwen2.5-Coder-7B-Instruct",
    "Qwen2.5-Coder-14B-Instruct",
    "Qwen2.5-Coder-32B-Instruct",  
]

ds = json.load(open("/home/luoxianzhen/yang/data/Ours/TestcaseBench-v28.json", "r", encoding="utf-8"))

import os
for model_name in model_name_list:
    for test_al in test_als:
        result_file = f"/home/luoxianzhen/yang/eval_wrong_code/ALLmode_results/tcb-{model_name}-{test_al}-fliter-{test_al}-rank5-all.json"
        if not os.path.exists(result_file):
            print(f"{model_name}-{test_al} NOT EXSIT!")
            continue 
        results = json.load(open(result_file, "r", encoding="utf-8"))

        rank_result = {
            "rank1": {"AC":0, "CE": 0, "WA":0, "RE": 0, "TLE":0, "MLE":0,"EXE":0},
            "rank2": {"AC":0, "CE": 0, "WA":0, "RE": 0, "TLE":0, "MLE":0,"EXE":0},
            "rank3": {"AC":0, "CE": 0, "WA":0, "RE": 0, "TLE":0, "MLE":0,"EXE":0},
            "rank4": {"AC":0, "CE": 0, "WA":0, "RE": 0, "TLE":0, "MLE":0,"EXE":0},
            "rank5": {"AC":0, "CE": 0, "WA":0, "RE": 0, "TLE":0, "MLE":0,"EXE":0},
        }
        success_k = {
            "rank1": {"total": 0, "hacked": 0},
            "rank2": {"total": 0, "hacked": 0},
            "rank3": {"total": 0, "hacked": 0},
            "rank4": {"total": 0, "hacked": 0},
            "rank5": {"total": 0, "hacked": 0},
        }
        rank_count = 0
        for k, v in results.items():
            rank = len(v['codes'])
            if max([len(code['status']) for code in v['codes']]) < rank * 5:
                continue
            rank_count += 1
            for i in range(5):
                nums_of_tests = rank * (i + 1)
                array_length = max([len(code['status']) for code in v['codes']])
                tests_index = get_random_indices(array_length, nums_of_tests)
                ## 每道题计算 rate
                hacked = 0
                status_present = {
                    "AC":0, "CE": 0, "WA":0, "RE": 0, "TLE":0, "MLE":0,"EXE":0
                }

                success_k[f"rank{i+1}"]["total"] += rank
                if array_length == 0:
                    status_present['AC'] += rank
                else:
                    for code in v['codes']:
                        tests_status = [code['status'][i] for i in tests_index] if max(tests_index) < len(code['status']) else code['status']

                        status_present[find_first_non_ac(tests_status)] += 1
                        if find_first_non_ac(tests_status) != "AC":
                            hacked += 1
                    success_k[f"rank{i+1}"]["hacked"] += hacked / rank

                for key, value in status_present.items():
                    rank_result[f"rank{i+1}"][key] += (value / rank)
        print(f"{model_name}-{test_al} > rank5 Count = {rank_count}")
        # 创建 Markdown 表格
        algorithm_model = f"{test_al}|{model_name}"
        if rank_count <= 0:
            continue
        # 创建 Markdown 表格
        markdown_table = "| Algorithm | Model | Rank | AC | CE | WA | RE | TLE | MLE | EXE | Hack Rate | Problem_count | \n"
        markdown_table += "|----------|--------|------|----|----|----|----|-----|-----|-----|-----------|-----|\n"

        for rank in rank_result:
            total = success_k[rank]["total"]
            hacked = success_k[rank]["hacked"]
            hack_rate = (hacked / rank_count * 100) if total > 0 else 0
            hack_rate = round(hack_rate, 2)  # 保留两位小数

            # 计算每个状态的百分比和数量
            status_percentages = []
            for key in rank_result[rank]:
                count = rank_result[rank][key]
                percentage = (count / rank_count * 100)
                status_percentages.append(f"{percentage:.2f}%")

            # 将每个状态的百分比和数量组合在一起
            markdown_table += f"| {algorithm_model} | {rank} | " + " | ".join(status_percentages) + f" | {hack_rate}% |" + f" {rank_count} |\n"

        # 保存到 .md 文件
        with open(f"/home/luoxianzhen/yang/eval_wrong_code/rank_md/rank_result-{model_name}-{test_al}-rank-scaling.md", "w") as file:
            file.write(markdown_table)

        # print("Markdown 文件已生成: rank_result.md")