| import os, shutil |
| from uparser import wordparse |
| from joblib import Parallel, delayed |
| from tqdm import tqdm |
|
|
| num_jobs = 20 |
| infolder = 'Words' |
| outfolder = 'Outputs' |
|
|
| for fdr in [outfolder]: |
| if os.path.exists(fdr): |
| shutil.rmtree(fdr) |
| os.mkdir(fdr) |
|
|
| langs = [] |
| langwdmap = {} |
|
|
| flist = os.listdir(infolder) |
| for fname in flist: |
| with open(f'{infolder}/{fname}', 'r') as f: |
| cnts = f.readlines() |
|
|
| i = 0 |
| |
| words = [] |
| for l in cnts: |
| l = l.strip().split('\t') |
| words.append(l[0]) |
| |
| lan = fname.split('.')[0] |
| langs.append(lan) |
| langwdmap[lan] = words |
|
|
| for opt in [0,1,2]: |
| out_cur = outfolder + '/option-' + str(opt) |
| os.mkdir(out_cur) |
| anslist=[] |
| for l in langs: |
| words = langwdmap[l] |
| print(words) |
| print(len(words)) |
| print(l) |
| if opt == 1: |
| anslist = Parallel(n_jobs=num_jobs)(delayed(wordparse)(wd, 0, opt, 1) for wd in tqdm(words)) |
| else: |
| anslist = Parallel(n_jobs=num_jobs)(delayed(wordparse)(wd, 0, opt, 0) for wd in tqdm(words)) |
| |
| |
| |
| |
|
|
|
|
|
|
| with open(f'{out_cur}/{l}.output', 'w') as f: |
| for i in range(len(words)): |
| f.write(f'{words[i]}\t{anslist[i]}\n') |
| |
| print(f'completed {l}') |
|
|
| |
