Update merge_questions.py

merge_questions.py

@@ -1,112 +1,28 @@
-# merge_questions.py
 import os
 import json
-import re
-from pathlib import Path
 
 DATA_DIR = "data"
 OUTPUT_FILE = "questions.json"
 
-VALID_SUBJECT_TOKENS = {
-    "bm": "BM",
-    "english": "English",
-    "math": "Math",
-    "history": "History",
-    "science": "Science",
-    "moralstudies": "MoralStudies",
-    "accounting": "Accounting",
-    "economics": "Economics",
-    "business": "Business",
-}
-
-YEARS = [str(y) for y in range(2018, 2025)]
-
-
-def safe_load(path):
-    try:
-        with open(path, "r", encoding="utf-8") as f:
-            return json.load(f)
-    except Exception:
-        return None
-
-
-def merge_all(data_dir=DATA_DIR, output_file=OUTPUT_FILE):
-    os.makedirs(data_dir, exist_ok=True)
-    merged = []
-    next_id = 1000
-
-    p = Path(data_dir)
-    files = sorted(p.glob("spm_*_*.json"))
-
-    for file in files:
-        fname = file.name
-        if fname.lower().endswith("_scheme.json"):
-            continue
-        m = re.match(r"spm[_\-](\d{4})[_\-]([a-zA-Z]+)\.json$", fname, re.IGNORECASE)
-        if not m:
-            print(f"Skipping non-matching file: {fname}")
-            continue
-        year = m.group(1)
-        subj_token = m.group(2).lower()
-        if year not in YEARS:
-            print(f"Skipping year outside 2018-2024: {fname}")
-            continue
-        if subj_token not in VALID_SUBJECT_TOKENS:
-            print(f"Unknown subject token '{subj_token}' in {fname}; skipping.")
-            continue
-        subj_display = VALID_SUBJECT_TOKENS[subj_token]
-        subj_key = f"Form5_{subj_display}"
-
-        qlist = safe_load(str(file))
-        if not isinstance(qlist, list):
-            print(f"Skipping {fname}: not a JSON list.")
-            continue
-
-        scheme_path = file.with_name(f"{file.stem}_scheme.json")
-        scheme = safe_load(str(scheme_path)) or {}
-
-        for idx, q in enumerate(qlist, start=1):
-            merged_q = {
-                "id": next_id,
-                "subject": subj_key,
-                "question_type": q.get("question_type", "mcq"),
-                "text": q.get("text", "").strip(),
-                "choices": q.get("choices", []),
-                "topics": q.get("topics", []),
-                "difficulty": q.get("difficulty", 3),
-                "source": q.get("source", "pastpaper"),
-                "year": int(year)
-            }
-
-            correct_answer = None
-            val = scheme.get(str(idx)) if isinstance(scheme, dict) else None
-            if isinstance(val, str):
-                v = val.strip()
-                if len(v) == 1 and merged_q["choices"]:
-                    pos = ord(v.upper()) - ord("A")
-                    if 0 <= pos < len(merged_q["choices"]):
-                        correct_answer = merged_q["choices"][pos]
-                    else:
-                        correct_answer = None
-                else:
-                    correct_answer = v if v else None
-            elif isinstance(val, dict):
-                correct_answer = val.get("correct_answer")
-            else:
-                correct_answer = None
-
-            merged_q["correct_answer"] = correct_answer
-            merged.append(merged_q)
-            next_id += 1
-
-    with open(output_file, "w", encoding="utf-8") as f:
-        json.dump(merged, f, indent=2, ensure_ascii=False)
-
-    print(f"✅ Merged {len(merged)} questions into {output_file}")
-
+def merge_json_files():
+    all_questions = []
+    for fname in os.listdir(DATA_DIR):
+        if fname.endswith(".json"):
+            fpath = os.path.join(DATA_DIR, fname)
+            try:
+                with open(fpath, "r", encoding="utf-8") as f:
+                    data = json.load(f)
+                    if isinstance(data, list):
+                        all_questions.extend(data)
+            except Exception as e:
+                print(f"⚠️ Error reading {fname}: {e}")
+    with open(OUTPUT_FILE, "w", encoding="utf-8") as out:
+        json.dump(all_questions, out, ensure_ascii=False, indent=2)
+    print(f"✅ Merged {len(all_questions)} questions into {OUTPUT_FILE}")
 
 if __name__ == "__main__":
-    merge_all()
+    merge_json_files()
+