Update merge_questions.py

merge_questions.py

@@ -1,30 +1,114 @@
+# merge_questions.py
 import os
 import json
+import sqlite3
 
-DATA_DIR = "data"
-OUTPUT_FILE = "questions.json"
+PROCESSED_DIR = "data/processed"
+QUESTIONS_FILE = "questions.json"
+DB_FILE = "exam.db"
 
-def merge_json_files():
-    all_questions = []
-    for fname in os.listdir(DATA_DIR):
-        if fname.endswith(".json") and "paper2" in fname.lower():
-            fpath = os.path.join(DATA_DIR, fname)
+def load_processed_files():
+    items = []
+    if not os.path.exists(PROCESSED_DIR):
+        return items
+    for fname in os.listdir(PROCESSED_DIR):
+        if fname.endswith(".json"):
+            fpath = os.path.join(PROCESSED_DIR, fname)
             try:
                 if os.path.getsize(fpath) == 0:
-                    print(f"⚠️ Skipping empty file: {fname}")
+                    print(f"Skipping empty file {fname}")
                     continue
                 with open(fpath, "r", encoding="utf-8") as f:
                     data = json.load(f)
                     if isinstance(data, list):
-                        all_questions.extend(data)
+                        items.extend(data)
             except Exception as e:
-                print(f"⚠️ Error reading {fname}: {e}")
-    with open(OUTPUT_FILE, "w", encoding="utf-8") as out:
-        json.dump(all_questions, out, ensure_ascii=False, indent=2)
-    print(f"✅ Merged {len(all_questions)} Paper 2 questions into {OUTPUT_FILE}")
+                print(f"Error reading {fname}: {e}")
+    return items
+
+def normalize_question(q, next_id):
+    """
+    Ensure fields id, text, choices(list), answer(letter or text), subject, paper, year, image, source
+    """
+    nq = {}
+    nq["id"] = int(q.get("id") or next_id)
+    nq["text"] = q.get("text") or q.get("question") or ""
+    # choices: if string, try to split by newline
+    c = q.get("choices", [])
+    if isinstance(c, str):
+        # split lines and remove leading A./B.
+        lines = [l.strip() for l in c.splitlines() if l.strip()]
+        choices = []
+        for ln in lines:
+            choices.append(re_sub_strip_choice(ln))
+        nq["choices"] = choices
+    elif isinstance(c, list):
+        nq["choices"] = c
+    else:
+        nq["choices"] = []
+
+    ans = q.get("answer", "")
+    nq["answer"] = ans
+    nq["subject"] = q.get("subject") or q.get("source_subject") or "Unknown"
+    nq["paper"] = int(q.get("paper", 2) or 2)
+    nq["year"] = int(q.get("year", 0) or 0)
+    nq["image"] = q.get("image")
+    nq["source"] = q.get("source") or ""
+    return nq
+
+def re_sub_strip_choice(s):
+    import re
+    return re.sub(r'^[A-D][\.\)\-]?\s*', '', s).strip()
+
+def merge():
+    items = load_processed_files()
+    merged = []
+    next_id = 100000
+    for it in items:
+        next_id += 1
+        try:
+            nq = normalize_question(it, next_id)
+            merged.append(nq)
+        except Exception as e:
+            print(f"Skipping item due to normalize error: {e}")
+
+    # write questions.json
+    with open(QUESTIONS_FILE, "w", encoding="utf-8") as f:
+        json.dump(merged, f, ensure_ascii=False, indent=2)
+    print(f"✅ Merged {len(merged)} questions into {QUESTIONS_FILE}")
+
+    # update sqlite DB
+    conn = sqlite3.connect(DB_FILE)
+    cur = conn.cursor()
+    cur.execute("""
+    CREATE TABLE IF NOT EXISTS questions (
+        id INTEGER PRIMARY KEY,
+        text TEXT,
+        choices TEXT,
+        answer TEXT,
+        subject TEXT,
+        paper INTEGER,
+        year INTEGER,
+        image TEXT,
+        source TEXT
+    )
+    """)
+    conn.commit()
+    inserted = 0
+    for q in merged:
+        try:
+            cur.execute("INSERT OR IGNORE INTO questions (id,text,choices,answer,subject,paper,year,image,source) VALUES (?,?,?,?,?,?,?,?,?)",
+                        (q["id"], q["text"], json.dumps(q["choices"], ensure_ascii=False), q["answer"], q["subject"], q["paper"], q["year"], q["image"], q["source"]))
+            inserted += 1
+        except Exception as e:
+            print("DB insert error:", e)
+    conn.commit()
+    conn.close()
+    print(f"✅ Inserted or ignored {inserted} rows into {DB_FILE}")
 
 if __name__ == "__main__":
-    merge_json_files()
+    merge()
+