| import json |
| from tqdm import tqdm |
|
|
| lang = "" |
|
|
| with open(f"dset{lang}.txt", "r") as f: |
| lines = [x.rstrip("\n").lower().split("→") for x in tqdm(f.readlines())] |
| lines = [(x[0].replace("\\n", "\n"), x[1].replace("\\n", "\n")) for x in lines] |
|
|
| responses = [] |
| for i in tqdm(lines): |
| if i[1] not in responses: |
| responses.append(i[1]) |
|
|
| dset = {} |
| for sample in tqdm(lines): |
| dset[sample[0]] = responses.index(sample[1]) |
|
|
| with open(f"dataset{lang}.json", "w") as f: |
| json.dump(dset, f, ensure_ascii=False) |
|
|
| with open(f"responses{lang}.txt", "w") as f: |
| for i in tqdm(responses): |
| f.write(i+"\n") |
|
|
|
|
