IndicTrans2/scripts/merge_bpcc.py

import os
from sys import argv
import multiprocessing as mp


def process_language(lang):

    all_pairs = []
    print(f"lang: {lang}")

    for domain in domains:
        src_fname = f"{base_path}/{domain}/eng_Latn-{lang}/train.eng_Latn"
        tgt_fname = f"{base_path}/{domain}/eng_Latn-{lang}/train.{lang}"

        try:
            with open(src_fname, "r", encoding="utf-8") as f1, open(
                tgt_fname, "r", encoding="utf-8"
            ) as f2:
                src_sents = [x.strip() for x in f1]
                tgt_sents = [x.strip() for x in f2]
            all_pairs.extend([(a, b) for (a, b) in zip(src_sents, tgt_sents)])
        except Exception as e:
            pass

    all_pairs = list(set(all_pairs))
    src_sents, tgt_sents = zip(*all_pairs)

    os.makedirs(f"{out_dir}/eng_Latn-{lang}", exist_ok=True)
    with open(
        f"{out_dir}/eng_Latn-{lang}/train.eng_Latn", "w", encoding="utf-8"
    ) as f1, open(
        f"{out_dir}/eng_Latn-{lang}/train.{lang}", "w", encoding="utf-8"
    ) as f2:
        f1.write("\n".join(src_sents))
        f2.write("\n".join(tgt_sents))


if __name__ == "__main__":

    base_path = argv[1]
    out_dir = argv[2]

    language_codes = [
    'asm_Beng', 'ben_Beng', 'brx_Deva', 'doi_Deva', 'gom_Deva', 
    'guj_Gujr', 'hin_Deva', 'kan_Knda', 'kas_Arab', 'kas_Deva',
    'mai_Deva', 'mal_Mlym', 'mar_Deva', 'mni_Beng', 'mni_Mtei', 
    'npi_Deva', 'ory_Orya', 'pan_Guru', 'san_Deva', 'sat_Olck', 
    'snd_Arab', 'snd_Deva', 'tam_Taml', 'tel_Telu', 'urd_Arab'
    ]

    domains = os.listdir(base_path)

    with mp.Pool(mp.cpu_count()) as pool:
        pool.map(process_language, language_codes)