| import json |
| import numpy as np |
|
|
|
|
| index = { |
| 'call_candidate': None, |
| 'exe_candidate': None, |
| 'perf_candidates': [], |
| } |
|
|
| paths = [ |
| '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_oss_120b_medium_v3_mem_0.json', |
| '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_oss_120b_medium_v3_mem_1.json', |
| '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_oss_120b_medium_v3_mem_2.json', |
| '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_oss_120b_medium_v3_mem_3.json', |
| '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_oss_120b_high_v3_mem_0.json', |
| '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_oss_120b_high_v3_mem_1.json', |
| '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_oss_120b_high_v3_mem_2.json', |
|
|
| |
| |
| |
| |
| |
| |
|
|
| '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8r_mem_0.json', |
| |
| |
| |
|
|
| '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8r_code_mem_0.json', |
| '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8r_code_mem_1.json', |
| '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8r_code_mem_2.json', |
| '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8r_code_mem_3.json', |
| '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8r_code_mem_4.json', |
|
|
| '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_oss_120b_high_v4_mem_0.json', |
| '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_oss_120b_high_v4_mem_1.json', |
|
|
| '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8r_30_mem_0.json', |
| '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8r_30_mem_1.json', |
| '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8r_30_mem_2.json', |
| '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8r_30_mem_3.json', |
|
|
| '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1114_q8r_code_mem_0.json', |
|
|
| '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1114_q8r_8b_mem_0.json', |
| '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1114_q8r_8b_mem_1.json', |
| '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1114_q8r_8b_mem_2.json', |
| ] |
|
|
| for path in paths: |
| result = {i:[] for i in index} |
| data = json.load(open(path)) |
| for k, v in data.items(): |
| for i in index: |
| result[i].append(v[i]) |
| |
| print(f'[path]:') |
| print(path) |
| print('\n') |
| for k, v in index.items(): |
| print(f'[{k}]:') |
| print(np.array([str(i) != str(v) for i in result[k]]).mean()) |
| if k == 'perf_candidates': |
| acc = [] |
| for i in result[k]: |
| if len(i) == 0: |
| acc.append(1) |
| else: |
| acc.append(max(1, i[-1][1])) |
| print(np.array(acc).mean()) |
|
|
| print('\n') |
| print('-'*100) |
|
|
