Upload todset.py

todset.py

@@ -0,0 +1,25 @@
+import json
+from tqdm import tqdm
+
+lang = ""
+
+with open(f"dset{lang}.txt", "r") as f:
+    lines = [x.rstrip("\n").lower().split("→") for x in tqdm(f.readlines())]
+    lines = [(x[0].replace("\\n", "\n"), x[1].replace("\\n", "\n")) for x in lines]
+
+responses = []
+for i in tqdm(lines):
+    if i[1] not in responses:
+        responses.append(i[1])
+
+dset = {}
+for sample in tqdm(lines):
+    dset[sample[0]] = responses.index(sample[1])
+
+with open(f"dataset{lang}.json", "w") as f:
+    json.dump(dset, f, ensure_ascii=False)
+
+with open(f"responses{lang}.txt", "w") as f:
+    for i in tqdm(responses):
+        f.write(i+"\n")
+