Update app.py

app.py

@@ -1,283 +1,181 @@
+# agents.py
 import os
-import re
-import json
 import random
-import subprocess
-import gradio as gr
-
-from agents import AnalyzerAgent, CoachAgent, PredictiveAgent
-from ocr_agent import OcrAgent
-
-# Paths
-DATA_DIR = "data"
-QUESTIONS_FILE = "questions.json"
-
-# Ensure data dir exists
-os.makedirs(DATA_DIR, exist_ok=True)
-
-# Agents
-analyzer = AnalyzerAgent()
-coach_agent = CoachAgent()
-predictor = PredictiveAgent()
-ocr_agent = OcrAgent()
-
-
-def load_question_bank():
-    """Load merged question bank safely; return [] if file missing/invalid."""
-    if not os.path.exists(QUESTIONS_FILE):
-        return []
-    try:
-        with open(QUESTIONS_FILE, "r", encoding="utf-8") as f:
-            content = f.read().strip()
-            return json.loads(content) if content else []
-    except Exception:
-        return []
-
-
-QUESTION_BANK = load_question_bank()
-
-
-# ---------------- Merge helper ----------------
-def merge_questions():
-    """Run merge_questions.py to rebuild questions.json and reload in memory."""
-    try:
-        subprocess.run(["python", "merge_questions.py"], check=True)
-        global QUESTION_BANK
-        QUESTION_BANK = load_question_bank()
-        return True, "Merge successful"
-    except subprocess.CalledProcessError as e:
-        return False, f"Merge failed: {e}"
-
-
-# ---------------- OCR / Upload ----------------
-def auto_detect_from_filename(path):
-    """Try to detect year and subject (lowercase subject token) from filename.
-    Matches patterns like: spm_2018_bm.pdf or spm-2019-math.pdf etc."""
-    if not path:
-        return None, None
-    fname = os.path.basename(path)
-    m = re.search(r"spm[_\-]?(\d{4})[_\-]?([A-Za-z]+)", fname, re.IGNORECASE)
-    if not m:
-        return None, None
-    year = m.group(1)
-    subj = m.group(2).lower()
-    return year, subj
-
-
-SUBJECT_DISPLAY_ORDER = ["BM", "English", "Math", "History", "Science", "MoralStudies",
-                         "Accounting", "Economics", "Business"]
-
-
-def normalize_display_subject(token):
-    """Return display subject label (capitalized BM / English / Math / MoralStudies, etc.)."""
-    if not token:
-        return "BM"
-    t = token.strip().lower()
-    mapping = {
-        "bm": "BM",
-        "bahasa": "BM",
-        "bahasamelayu": "BM",
-        "english": "English",
-        "eng": "English",
-        "math": "Math",
-        "mathematics": "Math",
-        "history": "History",
-        "sejarah": "History",
-        "science": "Science",
-        "moral": "MoralStudies",
-        "moralstudies": "MoralStudies",
-        "accounting": "Accounting",
-        "economics": "Economics",
-        "business": "Business",
-    }
-    return mapping.get(t, token.capitalize())
-
-
-def subject_token_from_display(display_subj):
-    """Convert display subject (BM, English) to token used in filenames (lowercase)."""
-    if not display_subj:
-        return "bm"
-    dsp = display_subj.strip()
-    return dsp.lower()
-
-
-def process_pdf_and_merge(file_path, display_subject, year):
+from collections import Counter
+from typing import List, Dict, Any
+
+# Accept both env var names for backward compatibility
+GLM_API_KEY = os.getenv("ZHIPUAI_API_KEY") or os.getenv("zhipuai_api_key")
+
+
+class AnalyzerAgent:
     """
-    - Run OCR -> write data/spm_{year}_{subject}.json + scheme file.
-    - Auto-run merge_questions.py to create/refresh questions.json
+    Produces topic-level accuracy and weak-topic recommendations.
+    Input: per_question dict {qid: {"user":..., "correct":..., "topics":[...]}}
     """
-    if not file_path:
-        return "No file provided."
-
-    subj_token = subject_token_from_display(display_subject)
-    # call OCR agent to extract and write files
-    try:
-        out_qfile, out_scheme = ocr_agent.extract_questions_to_files(pdf_path=file_path,
-                                                                     year=str(year),
-                                                                     subject_token=subj_token,
-                                                                     out_dir=DATA_DIR)
-    except Exception as e:
-        return f"❌ OCR failed: {e}"
-
-    ok, msg = merge_questions()
-    if ok:
-        return f"✅ OCR saved {out_qfile} and {out_scheme}. Merge result: {msg}"
-    else:
-        return f"⚠️ OCR saved {out_qfile} and {out_scheme}. Merge result: {msg}"
-
-
-# ---------------- Exam logic ----------------
-def generate_exam(subject_display, num_questions, include_predicted):
+    def analyze(self, per_question: Dict[str, Dict[str, Any]]) -> Dict[str, Any]:
+        topic_stats = {}
+        for qid, info in per_question.items():
+            topics = info.get("topics") or []
+            user = info.get("user")
+            correct = info.get("correct")
+            is_correct = (correct is not None and user is not None and str(user).strip() == str(correct).strip())
+            for t in topics:
+                if t not in topic_stats:
+                    topic_stats[t] = {"correct": 0, "total": 0}
+                topic_stats[t]["total"] += 1
+                if is_correct:
+                    topic_stats[t]["correct"] += 1
+
+        topic_accuracy = {}
+        weak_topics = []
+        for t, stats in topic_stats.items():
+            total = stats["total"]
+            correct = stats["correct"]
+            acc = round((correct / total) * 100, 2) if total > 0 else 0.0
+            topic_accuracy[t] = {"accuracy_percent": acc, "total": total}
+            if total >= 3 and acc < 65.0:
+                weak_topics.append(t)
+
+        recommendation = "Focus on: " + ", ".join(weak_topics) if weak_topics else "No major weak topics detected."
+
+        return {
+            "topic_accuracy": topic_accuracy,
+            "weak_topics": weak_topics,
+            "recommendation": recommendation
+        }
+
+
+class CoachAgent:
     """
-    Returns (exam_data (list), status_message, exam_data) to store exam_data in state.
-    exam_data items: {id:int, text:str, choices:list, topics:list, source:str}
+    Short actionable coaching guidance for Form5 SPM students.
     """
-    # internal lookup subject key stored in questions.json is "Form5_<DisplaySubject>" e.g., Form5_BM
-    subj_key = f"Form5_{subject_display}"
-    pool = [q for q in QUESTION_BANK if q.get("subject") == subj_key]
-
-    predicted_questions = []
-    if include_predicted:
-        # ask predictor to generate predictions using the current bank (so trend info is used)
-        predicted_questions = predictor.generate_predictions(level="Form5",
-                                                            subject=subject_display,
-                                                            n=8,
-                                                            question_bank=QUESTION_BANK)
-
-    combined = pool + predicted_questions
-    if not combined:
-        return [], f"No questions available for {subject_display}. Upload papers (2018–2024) first.", []
-
-    random.shuffle(combined)
-    selected = combined[:min(num_questions, len(combined))]
-
-    # Standardize output shape (do not expose 'correct_answer' for predicted? we include it,
-    # but the UI can show choices; predicted questions have correct_answer set by predictor)
-    exam_data = []
-    for q in selected:
-        # ensure minimal fields exist
-        exam_data.append({
-            "id": q.get("id"),
-            "text": q.get("text"),
-            "choices": q.get("choices", []),
-            "topics": q.get("topics", []),
-            "source": q.get("source", "pastpaper")
-        })
-
-    return exam_data, f"Prepared {len(exam_data)} questions (includes {len(predicted_questions)} predicted)" , exam_data
-
-
-def submit_exam_answers(answers_json, exam_data, subject_display):
+    def coach(self, analysis: Dict[str, Any], level: str, subject: str) -> Dict[str, Any]:
+        weak = analysis.get("weak_topics", [])
+        if not weak:
+            tips = [
+                "Keep revising key topics and time yourself on mock papers.",
+                "Review incorrect solutions and understand each step.",
+                "Do a mixed-topic mock weekly to build stamina."
+            ]
+        else:
+            tips = [
+                f"Spend 20–30 minutes daily on {weak[0]} (split into focused tasks).",
+                "Solve short targeted questions and check worked solutions.",
+                "Teach a concept to someone else — it stabilizes understanding."
+            ]
+
+        practice = []
+        for i, t in enumerate(weak[:3], start=1):
+            practice.append({
+                "text": f"Short practice prompt on {t}: (write/solve one short item)",
+                "topic": t
+            })
+
+        return {"tips": tips, "study_plan": "20 min/day for weak topics + weekly mock", "practice": practice}
+
+
+class PredictiveAgent:
     """
-    answers_json: dict mapping question id (string) -> answer string (the answer text or choice text)
-    exam_data: list (from start)
-    We grade only questions where a correct_answer exists (not None).
+    Generates heuristic or LLM-based predicted Form5 questions (in-memory only).
+    Public methods:
+      - predict(subject, level, count) -> list of question dicts
+      - summary(level, subject) -> dict
     """
-    if not exam_data:
-        return "No exam data found.", {}, {}, {}, gr.update(visible=False), gr.update(visible=True)
-
-    correct = 0
-    graded = 0
-    per_question = {}
-
-    for q in exam_data:
-        qid = q.get("id")
-        k = str(qid)
-        user_ans = answers_json.get(k)
-        # find canonical correct_answer: for past paper, from QUESTION_BANK; for predicted, from q itself if present
-        correct_ans = None
-        if q.get("source") == "predicted":
-            # predicted question object may include a 'correct_answer'
-            # in our design predictor attaches 'correct_answer' to predicted questions
-            # but it's still probabilistic (has 'confidence' field)
-            # q (from exam_data) did not include correct_answer (we stripped), so find from QUESTION_BANK? Not present
-            # We need to find original predicted object — predictor returns dicts; but since predicted questions were not saved to QUESTION_BANK,
-            # the simple way: during generate_exam we should have kept the predicted correct_answer in the exam_data object.
-            # To keep things robust, first attempt to find a matching question in QUESTION_BANK (unlikely),
-            # then try to see if exam_data contains 'correct_answer' directly (shouldn't in UI). We'll assume predicted questions include correct_answer in exam_data if they are to be graded.
-            correct_ans = q.get("correct_answer")  # may be None
+
+    def __init__(self):
+        self.api_key = GLM_API_KEY
+
+    def _top_topics_from_bank(self, question_bank: List[Dict], subject_display: str, top_k=6):
+        subj_key = f"Form5_{subject_display}"
+        counter = Counter()
+        total = 0
+        for q in question_bank:
+            if q.get("subject") != subj_key:
+                continue
+            total += 1
+            for t in q.get("topics", []):
+                counter[t] += 1
+        if total == 0:
+            return []
+        return [t for t, _ in counter.most_common(top_k)]
+
+    def predict(self, subject: str, level: str = "Form5", count: int = 5) -> List[Dict]:
+        """
+        Return `count` predicted MCQs. If no GLM key present, produce conservative heuristic items.
+        Predictions have id >= 900000, source='predicted', and may include 'confidence'.
+        """
+        preds = []
+        base = 900000
+        # fallback topics per subject
+        fallback_topics = {
+            "BM": ["perbendaharaan_kata", "tatabahasa"],
+            "English": ["vocabulary", "grammar"],
+            "Math": ["algebra", "geometry"],
+            "History": ["events", "dates"],
+            "Science": ["physics", "chemistry"],
+            "MoralStudies": ["ethics", "values"]
+        }
+        topics = fallback_topics.get(subject, ["general"])
+
+        # Try to use a simple LLM call if API key present (non-blocking, conservative)
+        # NOTE: We keep the interface simple: if GLM unavailable or fails, fall back to heuristics.
+        if self.api_key:
+            try:
+                # Placeholder: implement GLM call here if you provide endpoint details.
+                # For now, fall back to heuristics to avoid runtime dependency.
+                raise RuntimeError("GLM call not implemented in this environment")
+            except Exception:
+                pass
+
+        # Heuristic generation
+        for i in range(count):
+            t = topics[i % len(topics)]
+            q = self._heuristic_question(subject, t, idx=i + 1)
+            q["id"] = base + i
+            q["source"] = "predicted"
+            q["confidence"] = round(random.uniform(0.35, 0.75), 2)
+            preds.append(q)
+        return preds
+
+    def _heuristic_question(self, subject: str, topic: str, idx: int) -> Dict:
+        # provide realistic-looking stems & 4 choices tailored by subject
+        if subject == "BM":
+            stem = f"Pilih sinonim bagi perkataan 'gembira'."
+            choices = ["Sedih", "Gembira", "Marah", "Letih"]
+            correct = "Gembira"
+        elif subject == "English":
+            stem = "Choose the correct synonym for 'happy'."
+            choices = ["Sad", "Joyful", "Angry", "Tired"]
+            correct = "Joyful"
+        elif subject == "Math":
+            stem = "If 2x + 3 = 11, what is x?"
+            choices = ["2", "3", "4", "5"]
+            correct = "4"
+        elif subject == "Science":
+            stem = "What is the SI unit of force?"
+            choices = ["Joule", "Newton", "Pascal", "Watt"]
+            correct = "Newton"
+        elif subject == "History":
+            stem = "Which year is associated with Malayan independence?"
+            choices = ["1945", "1957", "1963", "1975"]
+            correct = "1957"
+        elif subject == "MoralStudies":
+            stem = "Which value best represents mutual respect?"
+            choices = ["Greed", "Respect", "Laziness", "Selfishness"]
+            correct = "Respect"
         else:
-            # pastpaper: find in QUESTION_BANK by id
-            orig = next((item for item in QUESTION_BANK if item.get("id") == qid), None)
-            if orig:
-                correct_ans = orig.get("correct_answer")
-
-        per_question[str(qid)] = {"user": user_ans, "correct": correct_ans, "topics": q.get("topics", [])}
-
-        if correct_ans is not None:
-            graded += 1
-            # compare string-normalized answers
-            if user_ans is not None and str(user_ans).strip() == str(correct_ans).strip():
-                correct += 1
-
-    score = round(100 * correct / graded, 2) if graded > 0 else "N/A (no answer keys available)"
-
-    analysis = analyzer.analyze(per_question)
-    coach = coach_agent.coach(analysis, "Form5", subject_display)
-    pred_summary = predictor.summary(level="Form5", subject=subject_display, question_bank=QUESTION_BANK)
-
-    return (
-        f"Your Score: {score}%",
-        analysis,
-        coach,
-        pred_summary,
-        gr.update(visible=False),
-        gr.update(visible=True)
-    )
-
-
-# ----------------- UI -----------------
-with gr.Blocks() as demo:
-    gr.Markdown("## SPM Exam Simulator — Form 5 (Past papers 2018–2024) with AI Predictions & OCR")
-
-    with gr.Tab("Upload Papers (OCR → JSON → Merge)"):
-        pdf_file = gr.File(label="Upload SPM PDF (filename like spm_2018_bm.pdf helps auto-detect)",
-                           type="filepath")
-        subject_dropdown = gr.Dropdown(choices=SUBJECT_DISPLAY_ORDER, value="BM", label="Subject (override)")
-        year_dropdown = gr.Dropdown(choices=[str(y) for y in range(2018, 2025)], value="2018", label="Year")
-        process_btn = gr.Button("Process PDF → JSON + Merge")
-        ocr_status = gr.Textbox(label="Status", interactive=False)
-
-        # When a file is uploaded, auto-fill subject/year fields
-        def prefill(file_path):
-            if not file_path:
-                return "BM", "2018"
-            year, subj_token = auto_detect_from_filename(file_path)
-            subj_display = normalize_display_subject(subj_token) if subj_token else "BM"
-            return subj_display, year if year else "2018"
-
-        pdf_file.change(fn=prefill, inputs=[pdf_file], outputs=[subject_dropdown, year_dropdown])
-        process_btn.click(fn=process_pdf_and_merge,
-                          inputs=[pdf_file, subject_dropdown, year_dropdown],
-                          outputs=[ocr_status])
-
-    with gr.Tab("Exam Simulator"):
-        subject_sel = gr.Dropdown(choices=["BM", "English", "Math", "History", "Science", "MoralStudies",
-                                          "Accounting", "Economics", "Business"],
-                                 value="Math", label="Subject")
-        num_q = gr.Slider(minimum=5, maximum=50, step=5, value=10, label="Number of Questions")
-        include_pred = gr.Checkbox(value=True, label="Include AI-predicted questions (in-memory only)")
-        start_btn = gr.Button("Start Exam")
-        exam_state = gr.State()  # will store exam_data (list)
-
-        exam_display = gr.JSON(label="Exam Questions (read-only)")
-        start_btn.click(fn=generate_exam,
-                        inputs=[subject_sel, num_q, include_pred],
-                        outputs=[exam_display, gr.Textbox(label="Status"), exam_state])
-
-    with gr.Tab("Submit & Results"):
-        answers_input = gr.JSON(label="Your Answers (JSON dictionary: {\"<id>\": \"<choice text>\"})")
-        submit_btn = gr.Button("Submit Answers")
-        score_out = gr.Textbox(label="Score")
-        analysis_out = gr.JSON(label="Weakness Analysis")
-        coach_out = gr.JSON(label="Study Coach")
-        pred_out = gr.JSON(label="Predictions Summary")
-
-        submit_btn.click(fn=submit_exam_answers,
-                         inputs=[answers_input, gr.State(), subject_sel, ],
-                         outputs=[score_out, analysis_out, coach_out, pred_out, gr.update(), gr.update()])
-
-demo.launch()
+            stem = f"Practice predicted question on {topic}."
+            choices = ["A", "B", "C", "D"]
+            correct = "A"
+
+        return {"text": stem, "choices": choices, "correct_answer": correct, "topics": [topic], "difficulty": 3}
+
+    def summary(self, level: str, subject: str, question_bank: List[Dict] = None) -> Dict:
+        # Provide simple summary: top topics from bank if available
+        topics = self._top_topics_from_bank(question_bank or [], subject) if question_bank else []
+        return {"level": level, "subject": subject, "top_topics": topics, "note": "Predictions are practice-oriented heuristics."}
+