Benny-Tang commited on
Commit
1df25bb
·
verified ·
1 Parent(s): 3bb7108

Update merge_questions.py

Browse files
Files changed (1) hide show
  1. merge_questions.py +19 -13
merge_questions.py CHANGED
@@ -4,24 +4,30 @@ import json
4
  DATA_DIR = "data"
5
  OUTPUT_FILE = "questions.json"
6
 
7
- def merge_json_files():
8
- all_questions = []
 
 
 
 
 
 
 
 
 
 
 
9
  for fname in os.listdir(DATA_DIR):
10
  if fname.endswith(".json"):
11
  fpath = os.path.join(DATA_DIR, fname)
12
- try:
13
- with open(fpath, "r", encoding="utf-8") as f:
14
- data = json.load(f)
15
- if isinstance(data, list):
16
- all_questions.extend(data)
17
- except Exception as e:
18
- print(f"⚠️ Error reading {fname}: {e}")
19
- with open(OUTPUT_FILE, "w", encoding="utf-8") as out:
20
- json.dump(all_questions, out, ensure_ascii=False, indent=2)
21
- print(f"✅ Merged {len(all_questions)} questions into {OUTPUT_FILE}")
22
 
23
  if __name__ == "__main__":
24
- merge_json_files()
 
25
 
26
 
27
 
 
4
  DATA_DIR = "data"
5
  OUTPUT_FILE = "questions.json"
6
 
7
+ def safe_load(path):
8
+ """Safely load a JSON file, return [] if invalid/empty."""
9
+ try:
10
+ if os.path.getsize(path) == 0:
11
+ return []
12
+ with open(path, "r", encoding="utf-8") as f:
13
+ return json.load(f)
14
+ except Exception:
15
+ print(f"⚠️ Error reading {path}")
16
+ return []
17
+
18
+ def merge():
19
+ merged = []
20
  for fname in os.listdir(DATA_DIR):
21
  if fname.endswith(".json"):
22
  fpath = os.path.join(DATA_DIR, fname)
23
+ merged.extend(safe_load(fpath))
24
+ with open(OUTPUT_FILE, "w", encoding="utf-8") as f:
25
+ json.dump(merged, f, ensure_ascii=False, indent=2)
26
+ print(f"✅ Merged {len(merged)} questions into {OUTPUT_FILE}")
 
 
 
 
 
 
27
 
28
  if __name__ == "__main__":
29
+ merge()
30
+
31
 
32
 
33