tools/viz_multi_model.py

from pathlib import Path
from typing import List

import typer
from mmengine.config import Config
from typer import Option

from opencompass.registry import build_from_cfg
from opencompass.summarizers.multi_model import MultiModelSummarizer

app = typer.Typer(add_completion=False, pretty_exceptions_show_locals=False)


@app.command(help='Visualize the results of multiple models')
def main(
    cfg_paths: List[Path] = Option(
        ...,
        help='The path to the config file of the task',
        exists=True,
    ),
    work_dirs: List[Path] = Option(
        ...,
        help='The work dirs for the task(named by timestamp), '
        'need to ensure the order is the same as cfg_paths.',
        exists=True,
    ),
    group: str = Option(None,
                        help='If not None, show the accuracy in the group.'),
):
    assert len(cfg_paths) == len(work_dirs)
    cfgs = [Config.fromfile(it, format_python_code=False) for it in cfg_paths]

    multi_models_summarizer = None
    for cfg, work_dir in zip(cfgs, work_dirs):
        cfg['work_dir'] = work_dir
        summarizer_cfg = cfg.get('summarizer', {})
        summarizer_cfg['type'] = MultiModelSummarizer
        summarizer_cfg['config'] = cfg
        summarizer = build_from_cfg(summarizer_cfg)
        if multi_models_summarizer is None:
            multi_models_summarizer = summarizer
        else:
            multi_models_summarizer.merge(summarizer)
    multi_models_summarizer.summarize()
    if group:
        multi_models_summarizer.show_group(group)


if __name__ == '__main__':
    app()