Upload folder using huggingface_hub

.env.example

@@ -27,7 +27,7 @@ OLLAMA_MODEL=minimax-m2.7:cloud
 # ─────────────────────────────────────────────────────────────────────
 # GGUF BACKEND (if LLM_BACKEND=gguf)
 # ─────────────────────────────────────────────────────────────────────
-# GGUF_MODEL_PATH=./models/qwen2-7b-instruct-q4_k_m.gguf
+# GGUF_MODEL_PATH=./models/Qwen3-32B-Q4_K_M.gguf
 # GGUF_N_CTX=4096              # Context window size
 # GGUF_N_GPU_LAYERS=-1         # -1 = offload all layers to GPU (Metal on Mac)
 

.gitignore

@@ -209,4 +209,4 @@ data/
 
 QModel.index
 metadata.json
-models/qwen2-7b-instruct-q8_0.gguf
+models/Qwen3-32B-Q4_K_M.gguf

README.md

@@ -19,12 +19,12 @@ language:
   - en
 ---
 
-# QModel 6 — Islamic RAG System
+# QModel v6 — Islamic RAG System
 **Specialized Qur'an & Hadith Knowledge System with Dual LLM Support**
 
 > A production-ready Retrieval-Augmented Generation system specialized exclusively in authenticated Islamic knowledge. No hallucinations, no outside knowledge—only content from verified sources.
 
-![Version](https://img.shields.io/badge/version-4.0.0-blue)
+![Version](https://img.shields.io/badge/version-6.0.0-blue)
 ![Backend](https://img.shields.io/badge/backend-ollama%20%7C%20huggingface-green)
 ![Status](https://img.shields.io/badge/status-production--ready-success)
 

app/analysis.py

@@ -132,26 +132,77 @@ async def detect_analysis_intent(query: str, rewrite: dict) -> Optional[str]:
         kw_text = " ".join(kws)
         if any(w in kw_text for w in ("آيات", "آية", "verses", "ayat")):
             return None
-        return kws[0] if kws else None
+        # The rewriter is instructed to put the target word as first keyword
+        if kws:
+            return kws[0]
+        # Fallback: extract from query
+        keyword = _extract_count_keyword(query)
+        return keyword
 
     if not (_COUNT_EN.search(query) or _COUNT_AR.search(query)):
         return None
 
+    keyword = _extract_count_keyword(query)
+    return keyword
+
+
+def _extract_count_keyword(query: str) -> Optional[str]:
+    """Extract the keyword being counted from various question patterns."""
+    # Arabic patterns: كم مرة ذكرت كلمة X / كم مرة وردت X / عدد مرات ذكر X
+    ar_patterns = [
+        re.compile(r"(?:كلمة|لفظ|لفظة)\s+([\u0600-\u06FF\u0750-\u077F]+)"),
+        re.compile(r"(?:ذ[ُ]?كر(?:ت|)|وردت?|تكرر(?:ت|))\s+(?:كلمة\s+)?([\u0600-\u06FF\u0750-\u077F]+)"),
+        re.compile(r"(?:عدد\s+مرات\s+(?:ذكر|ورود))\s+(?:كلمة\s+)?([\u0600-\u06FF\u0750-\u077F]+)"),
+        re.compile(r"كم\s+(?:مرة|مره)\s+(?:ذ[ُ]?كر(?:ت|)|وردت?)\s+(?:كلمة\s+)?([\u0600-\u06FF\u0750-\u077F]+)"),
+    ]
+    for pat in ar_patterns:
+        m = pat.search(query)
+        if m:
+            word = m.group(1).strip()
+            # Skip common non-keyword words
+            if word not in ("في", "من", "عن", "إلى", "على", "هل", "ما", "كم"):
+                return word
+
+    # English patterns: how many times is X mentioned / count of X / occurrences of X
+    en_patterns = [
+        re.compile(r"(?:word|term)\s+['\"]?(\w+)['\"]?", re.I),
+        re.compile(r"(?:times?\s+(?:is|does|has)\s+)(\w+)", re.I),
+        re.compile(r"(?:occurrences?\s+of|frequency\s+of|count\s+of)\s+['\"]?(\w+)['\"]?", re.I),
+        re.compile(r"(?:mentioned|appear[s]?|occur[s]?)\s+.*?['\"]?(\w+)['\"]?\s+(?:in|throughout)", re.I),
+        re.compile(r"(?:how many times)\s+(?:is\s+)?['\"]?(\w+)['\"]?", re.I),
+    ]
+    for pat in en_patterns:
+        m = pat.search(query)
+        if m:
+            word = m.group(1).strip()
+            if word.lower() not in ("the", "a", "an", "in", "of", "is", "are", "was", "how", "many", "quran"):
+                return word
+
+    # Last resort: find the first meaningful word after count-related keywords
     for pat in (_COUNT_EN, _COUNT_AR):
         m = pat.search(query)
         if m:
             tail = query[m.end():].strip().split()
-            if tail:
-                return tail[0]
+            for word in tail:
+                clean = re.sub(r"[؟?!.,،]", "", word).strip()
+                if clean and clean.lower() not in (
+                    "في", "من", "عن", "القرآن", "الكريم", "the", "quran", "in", "of",
+                    "كلمة", "لفظ", "word", "term",
+                ):
+                    return clean
     return None
 
 
 # ═══════════════════════════════════════════════════════════════════════
 # OCCURRENCE COUNTING
 # ═══════════════════════════════════════════════════════════════════════
-async def count_occurrences(keyword: str, dataset: list) -> dict:
-    """Count keyword occurrences with surah grouping."""
-    cached = await analysis_cache.get(keyword)
+async def count_occurrences(
+    keyword: str,
+    dataset: list,
+    source_type: Optional[str] = "quran",
+) -> dict:
+    """Count keyword occurrences with surah/collection grouping."""
+    cached = await analysis_cache.get(keyword, source_type or "all")
     if cached:
         return cached
 
@@ -162,7 +213,7 @@ async def count_occurrences(keyword: str, dataset: list) -> dict:
     examples: list = []
 
     for item in dataset:
-        if item.get("type") != "quran":
+        if source_type and item.get("type") != source_type:
             continue
 
         ar_norm  = normalize_arabic(item.get("arabic", ""), aggressive=True).lower()
@@ -195,7 +246,7 @@ async def count_occurrences(keyword: str, dataset: list) -> dict:
         "by_surah":    dict(sorted(by_surah.items())),
         "examples":    examples,
     }
-    await analysis_cache.set(result, keyword)
+    await analysis_cache.set(result, keyword, source_type or "all")
     return result
 
 

app/arabic_nlp.py

@@ -88,11 +88,18 @@ def language_instruction(lang: str) -> str:
     return {
         "arabic": (
             "يجب أن تكون الإجابة كاملةً باللغة العربية الفصحى تماماً. "
-            "لا تستخدم الإنجليزية أو أي لغة أخرى في أي جزء من الإجابة."
+            "لا تستخدم الإنجليزية أو أي لغة أخرى في أي جزء من الإجابة، "
+            "باستثناء الاقتباسات الموجودة في صناديق الأدلة فقط. "
+            "إذا كان السؤال بالعربية، أجب بالعربية حصراً."
         ),
         "mixed": (
             "The question mixes Arabic and English. Reply primarily in Arabic (الفصحى) "
-            "but you may transliterate key terms in English where essential."
+            "but you may include English transliterations for key Islamic terms where essential. "
+            "Match the dominant language of the question."
+        ),
+        "english": (
+            "You MUST reply entirely in clear, formal English. "
+            "Do NOT use Arabic in your explanation — only inside evidence quotation boxes. "
+            "The user asked in English and expects an English answer."
         ),
-        "english": "You MUST reply entirely in clear, formal English.",
     }.get(lang, "You MUST reply entirely in clear, formal English.")

app/llm.py

@@ -43,6 +43,7 @@ class OllamaProvider(LLMProvider):
                     model=self.model,
                     messages=messages,
                     options={"temperature": temperature, "num_predict": max_tokens},
+                    think=False,
                 ),
             )
             return result["message"]["content"].strip()
@@ -69,12 +70,20 @@ class GGUFProvider(LLMProvider):
     async def chat(
         self, messages: List[dict], temperature: float, max_tokens: int
     ) -> str:
+        # Disable Qwen3 thinking mode by appending /no_think to the system message
+        patched = []
+        for msg in messages:
+            if msg["role"] == "system" and "/no_think" not in msg["content"]:
+                patched.append({"role": "system", "content": msg["content"] + "\n/no_think"})
+            else:
+                patched.append(msg)
+
         loop = asyncio.get_event_loop()
         try:
             result = await loop.run_in_executor(
                 None,
                 lambda: self.llm.create_chat_completion(
-                    messages=messages,
+                    messages=patched,
                     temperature=temperature,
                     max_tokens=max_tokens,
                 ),
@@ -177,18 +186,19 @@ class HuggingFaceProvider(LLMProvider):
 
 def get_llm_provider() -> LLMProvider:
     """Factory function to get the configured LLM provider."""
-    if cfg.LLM_BACKEND == "ollama":
+    backend = cfg.LLM_BACKEND.lower()
+    if backend == "ollama":
         logger.info("Using Ollama backend: %s @ %s", cfg.OLLAMA_MODEL, cfg.OLLAMA_HOST)
         return OllamaProvider(cfg.OLLAMA_HOST, cfg.OLLAMA_MODEL)
-    elif cfg.LLM_BACKEND == "hf":
+    elif backend == "hf":
         logger.info("Using HuggingFace backend: %s on %s", cfg.HF_MODEL_NAME, cfg.HF_DEVICE)
         return HuggingFaceProvider(cfg.HF_MODEL_NAME, cfg.HF_DEVICE)
-    elif cfg.LLM_BACKEND == "gguf":
+    elif backend == "gguf":
         logger.info("Using GGUF backend: %s (ctx=%d, gpu_layers=%d)",
                     cfg.GGUF_MODEL_PATH, cfg.GGUF_N_CTX, cfg.GGUF_N_GPU_LAYERS)
         return GGUFProvider(cfg.GGUF_MODEL_PATH, cfg.GGUF_N_CTX, cfg.GGUF_N_GPU_LAYERS)
-    elif cfg.LLM_BACKEND == "lmstudio":
+    elif backend == "lmstudio":
         logger.info("Using LM Studio backend: %s @ %s", cfg.LMSTUDIO_MODEL, cfg.LMSTUDIO_URL)
         return LMStudioProvider(cfg.LMSTUDIO_URL, cfg.LMSTUDIO_MODEL)
     else:
-        raise ValueError(f"Unknown LLM_BACKEND: {cfg.LLM_BACKEND}")
+        raise ValueError(f"Unknown LLM_BACKEND: {cfg.LLM_BACKEND!r}")

app/prompts.py

@@ -17,72 +17,78 @@ PERSONA = (
 
 TASK_INSTRUCTIONS: Dict[str, str] = {
     "tafsir": (
-        "The user asks about a Quranic verse. Steps:\n"
-        "1. Identify the verse(s) from context.\n"
-        "2. Provide Tafsir: linguistic analysis and deeper meaning.\n"
-        "3. Draw connections to related verses.\n"
-        "4. Answer the user's question directly."
+        "The user asks about a Quranic verse — by partial text, topic, or meaning. Steps:\n"
+        "1. Identify the matching verse(s) from the RETRIEVED RESULTS.\n"
+        "2. Quote the Arabic verse text EXACTLY from the results.\n"
+        "3. Provide the full reference: Surah name (Arabic & English), number, and Ayah number.\n"
+        "4. Provide the English translation EXACTLY as given in the results.\n"
+        "5. If the user searched by partial text, confirm the full verse found.\n"
+        "6. Provide Tafsir: explain the meaning, context, and significance.\n"
+        "7. If related verses appear in the results, draw connections.\n"
+        "8. Answer the user's specific question directly.\n"
+        "9. Do NOT reference verses that are not in the results."
     ),
     "hadith": (
-        "The user asks about a Hadith. Structure your answer:\n\n"
-        "1. الجواب — Give a direct answer to the question first.\n\n"
-        "2. نص الحديث — Quote the hadith text EXACTLY from context\n"
-        "   in the evidence box format. Show ALL relevant narrations found.\n\n"
-        "3. الشرح والتوضيح — Explain the meaning and implications.\n"
-        "   Mention notable scholars, narrators, or jurisprudential points.\n"
-        "   Draw connections to related Hadiths from the context.\n\n"
-        "4. الخلاصة — Summarize the key takeaway.\n\n"
-        "CRITICAL: If the Hadith is NOT in context, say so clearly.\n"
-        "Quote hadith text VERBATIM from context — never paraphrase the matn."
+        "The user asks about a Hadith — by partial text, topic, or meaning. Steps:\n"
+        "1. Find the best matching Hadith from the RETRIEVED RESULTS.\n"
+        "2. Quote the Hadith text EXACTLY — both Arabic and English from the results.\n"
+        "3. State the full reference: collection name, book/chapter, hadith number.\n"
+        "4. State the grade/authenticity (Sahih, Hasan, Da'if) if available in the results.\n"
+        "5. If the user searched by partial text, present the complete hadith found.\n"
+        "6. Explain the meaning, context, and scholarly implications.\n"
+        "7. Note any related Hadiths from the results.\n"
+        "CRITICAL: If the Hadith is NOT in the results, say so clearly — do NOT fabricate."
     ),
     "auth": (
-        "The user asks about Hadith authenticity. Structure your answer:\n\n"
-        "الجواب — Start with a CLEAR, CONFIDENT verdict (صحيح/حسن/ضعيف/موضوع).\n"
-        "Give a one-line ruling summary.\n\n"
-        "أولًا: متن الحديث\n"
-        "Quote ALL matching narrations from the context in evidence boxes.\n"
-        "Show every relevant version found across different collections.\n\n"
-        "ثانيًا: الأدلة على صحته (أو ضعفه)\n"
-        "Provide numbered evidence points (use ١، ٢، ٣):\n"
-        "  - Which authoritative collections contain it\n"
-        "  - The grading given by scholars (from the grade field in context)\n"
-        "  - Notable narrators and scholars who transmitted or commented on it\n\n"
-        "ثالثًا: أهمية الحديث\n"
-        "Explain the hadith's significance, its place in Islamic scholarship,\n"
-        "and any jurisprudential implications.\n\n"
-        "الخلاصة — Comprehensive summary restating the verdict with key evidence.\n\n"
-        "RULES:\n"
-        "• If found in Sahih Bukhari or Sahih Muslim → assert AUTHENTIC (Sahih).\n"
-        "• Quote hadith text VERBATIM from context — never paraphrase the matn.\n"
-        "• You may add scholarly commentary to explain significance and context.\n"
-        "• If NOT found in context → clearly state it is absent from the dataset.\n"
-        "• NEVER fabricate hadith text, grades, or source citations."
+        "The user asks about Hadith authenticity or grade. YOU MUST:\n"
+        "1. Search the RETRIEVED RESULTS carefully for the Hadith.\n"
+        "2. If FOUND:\n"
+        "   a. State the grade (Sahih, Hasan, Da'if, etc.) PROMINENTLY at the start.\n"
+        "   b. Hadiths from Sahih al-Bukhari or Sahih Muslim are AUTHENTIC (Sahih).\n"
+        "   c. Hadiths from Sunan an-Nasa'i are generally Sahih.\n"
+        "   d. Hadiths from Jami' at-Tirmidhi, Sunan Abu Dawud, Sunan Ibn Majah are generally Hasan.\n"
+        "   e. Provide the full reference: collection, hadith number, chapter.\n"
+        "   f. Quote the full Hadith text from the results.\n"
+        "   g. Explain why this grade applies.\n"
+        "3. If NOT FOUND in the results:\n"
+        "   a. Clearly state: the hadith was not found in the authenticated dataset.\n"
+        "   b. Do NOT guess or fabricate a grade.\n"
+        "CRITICAL: Base authenticity ONLY on the retrieved results and collection source."
     ),
     "fatwa": (
-        "The user seeks a religious ruling. Steps:\n"
-        "1. Gather evidence from Quran + Sunnah in context.\n"
-        "2. Reason step-by-step to a conclusion.\n"
-        "3. If insufficient, state so explicitly."
+        "The user seeks a religious ruling or asks about Islamic law. Steps:\n"
+        "1. Give a direct answer to the ruling question first.\n"
+        "2. Gather supporting evidence from Quran and Hadith in the results.\n"
+        "3. Quote verses and hadiths with exact references from the results.\n"
+        "4. Present scholarly reasoning based ONLY on the evidence found.\n"
+        "5. If multiple scholarly opinions exist, mention them briefly.\n"
+        "6. If the results lack sufficient evidence, state so explicitly."
     ),
     "count": (
-        "The user asks for word frequency. Steps:\n"
-        "1. State the ANALYSIS RESULT prominently.\n"
-        "2. List example occurrences with Surah names.\n"
-        "3. Comment on significance."
+        "The user asks about word frequency or occurrence count. Steps:\n"
+        "1. State the ANALYSIS RESULT count PROMINENTLY and FIRST.\n"
+        "2. Use the EXACT numbers from the ANALYSIS RESULT — do NOT recalculate.\n"
+        "3. List the top example occurrences with Surah name (Arabic & English) and Ayah number.\n"
+        "4. Show the per-Surah breakdown from the analysis.\n"
+        "5. Comment on the significance and patterns of usage.\n"
+        "CRITICAL: The numbers in the ANALYSIS RESULT block are authoritative."
     ),
     "surah_info": (
-        "The user asks about surah metadata. Steps:\n"
-        "1. State the answer from the SURAH INFORMATION block EXACTLY.\n"
-        "2. Use the total_verses number precisely — do NOT guess or calculate.\n"
-        "3. Mention the revelation type (Meccan/Medinan) if available.\n"
-        "4. Optionally add brief scholarly context about the surah."
+        "The user asks about surah metadata (verse count, revelation type, etc.). Steps:\n"
+        "1. Answer the SPECIFIC question FIRST using the SURAH INFORMATION block.\n"
+        "2. Use the total_verses number EXACTLY as given — do NOT guess or calculate.\n"
+        "3. State the revelation type (Meccan/Medinan) from the data.\n"
+        "4. Mention the surah name in Arabic, English, and transliteration.\n"
+        "5. Mention the surah number.\n"
+        "6. Optionally add brief scholarly context about the surah.\n"
+        "CRITICAL: The SURAH INFORMATION block is the ONLY authoritative source."
     ),
     "general": (
-        "The user has a general Islamic question. Structure your answer:\n\n"
-        "1. الجواب — Give a direct, clear answer first.\n\n"
-        "2. الأدلة — Support with evidence from context, quoting relevant\n"
-        "   texts in evidence boxes. Explain the evidence with scholarly depth.\n\n"
-        "3. الخلاصة — Conclude with a comprehensive summary."
+        "The user has a general Islamic question. Steps:\n"
+        "1. Give a direct, clear answer first.\n"
+        "2. Support with evidence from the RETRIEVED RESULTS.\n"
+        "3. Quote relevant verses or hadiths from the results with references.\n"
+        "4. Conclude with a brief summary."
     ),
 }
 
@@ -96,14 +102,21 @@ For EVERY supporting evidence, use this exact format:
 └─────────────────────────────────────────────┘
 
 ABSOLUTE RULES:
-• Copy Arabic hadith text, translations, and sources VERBATIM from context. Never paraphrase.
-• You may add scholarly commentary, explanation, and analysis around the quoted evidence.
-• NEVER fabricate hadith text, grades, verse numbers, or source citations.
+• Use ONLY content from the Islamic Context block. Zero outside knowledge.
+• Copy Arabic text and translations VERBATIM from context. Never paraphrase.
 • If a specific Hadith/verse is NOT in context → respond with:
     "هذا الحديث/الآية غير موجود في قاعدة البيانات." (Arabic)
     or "This Hadith/verse is not in the available dataset." (English)
 • Never invent or guess content.
-• End with: "والله أعلم." (Arabic) or "And Allah knows best." (English)
+
+LANGUAGE RULE (CRITICAL — MUST FOLLOW):
+• You MUST answer in the SAME language as the user's question.
+• Arabic question → answer ENTIRELY in Arabic (العربية الفصحى). No English except inside evidence boxes.
+• English question → answer ENTIRELY in English. No Arabic except inside evidence boxes.
+• Mixed question → answer primarily in Arabic with English transliterations where helpful.
+• The evidence boxes always show both Arabic text and English translation regardless of language.
+
+• End with: "والله أعلم." (Arabic response) or "And Allah knows best." (English response)
 """
 
 _SYSTEM_TEMPLATE = """\
@@ -116,18 +129,10 @@ _SYSTEM_TEMPLATE = """\
 
 === OUTPUT FORMAT ===
 {fmt}
-"""
 
-_CONTEXT_TEMPLATE = """\
-IMPORTANT: The database has already been searched for you.
-The relevant results are provided below — use ONLY this data to formulate your answer.
-Do NOT state that you need a database or ask the user for data. Answer from the context below.
-
-=== RETRIEVED DATABASE RESULTS ===
+=== ISLAMIC CONTEXT ===
 {context}
-=== END DATABASE RESULTS ===
-
-Now answer the following question using ONLY the data above:
+=== END CONTEXT ===
 """
 
 
@@ -169,18 +174,17 @@ def build_messages(
         lang_instruction=language_instruction(lang),
         task=TASK_INSTRUCTIONS.get(intent, TASK_INSTRUCTIONS["general"]),
         fmt=FORMAT_RULES,
+        context=context,
     )
 
-    context_block = _CONTEXT_TEMPLATE.format(context=context)
-
     cot = {
-        "arabic": "فكّر خطوةً بخطوة، ثم أجب: ",
-        "mixed":  "Think step by step: ",
-    }.get(lang, "Think step by step: ")
+        "arabic": "فكّر خطوةً بخطوة، ثم أجب باللغة العربية فقط: ",
+        "mixed":  "فكّر خطوةً بخطوة، ثم أجب: ",
+    }.get(lang, "Think step by step, answer in English: ")
 
     return [
         {"role": "system",  "content": system},
-        {"role": "user",    "content": context_block + cot + question},
+        {"role": "user",    "content": cot + question},
     ]
 
 

app/routers/chat.py

@@ -1,23 +1,20 @@
-"""Chat / inference endpoints — OpenAI-compatible + /ask."""
+"""Chat / inference endpoints — OpenAI-compatible."""
 
 from __future__ import annotations
 
 import json
 import logging
 import time
-from typing import Optional
 
-from fastapi import APIRouter, HTTPException, Query
+from fastapi import APIRouter, HTTPException
 from fastapi.responses import StreamingResponse
 
 from app.config import cfg
 from app.models import (
-    AskResponse,
     ChatCompletionChoice,
     ChatCompletionMessage,
     ChatCompletionRequest,
     ChatCompletionResponse,
-    SourceItem,
 )
 from app.state import check_ready, run_rag_pipeline, state
 
@@ -123,41 +120,3 @@ async def _stream_response(result: dict, model: str):
     }
     yield f"data: {json.dumps(final)}\n\n"
     yield "data: [DONE]\n\n"
-
-
-# ───────────────────────────────────────────────────────
-# GET /ask — main inference endpoint
-# ───────────────────────────────────────────────────────
-@router.get("/ask", response_model=AskResponse)
-async def ask(
-    q: str = Query(..., min_length=1, max_length=1000, description="Your Islamic question"),
-    top_k: int = Query(cfg.TOP_K_RETURN, ge=1, le=20, description="Number of sources"),
-    source_type: Optional[str] = Query(None, description="Filter: quran|hadith"),
-    grade_filter: Optional[str] = Query(None, description="Filter Hadith: sahih|hasan|all"),
-):
-    """Main inference endpoint — runs the full RAG pipeline."""
-    check_ready()
-    result = await run_rag_pipeline(q, top_k, source_type, grade_filter)
-
-    sources = [
-        SourceItem(
-            source=r.get("source") or r.get("reference") or "Unknown",
-            type=r.get("type", "unknown"),
-            grade=r.get("grade"),
-            arabic=r.get("arabic", ""),
-            english=r.get("english", ""),
-            _score=r.get("_score", 0.0),
-        )
-        for r in result["sources"]
-    ]
-
-    return AskResponse(
-        question=q,
-        answer=result["answer"],
-        language=result["language"],
-        intent=result["intent"],
-        analysis=result["analysis"],
-        sources=sources,
-        top_score=result["top_score"],
-        latency_ms=result["latency_ms"],
-    )

app/routers/hadith.py

@@ -30,12 +30,36 @@ async def hadith_text_search(
 ):
     """Search for Hadith by partial text match (Arabic or English).
 
-    Performs exact substring matching plus word-overlap scoring.
+    Performs exact substring matching plus word-overlap and n-gram scoring.
+    Falls back to semantic search when text matching yields few results.
     Use this to find a hadith when you know part of the text.
     """
     check_ready()
     results = text_search(q, state.dataset, source_type="hadith", limit=limit)
 
+    # Optional collection filter
+    if collection:
+        col_lower = collection.lower()
+        results = [
+            r for r in results
+            if col_lower in (r.get("collection", "") or r.get("reference", "")).lower()
+        ]
+
+    # If text search returns few results, augment with semantic search
+    if len(results) < 3:
+        rewrite = await rewrite_query(q, state.llm)
+        sem_results = await hybrid_search(
+            q, rewrite,
+            state.embed_model, state.faiss_index, state.dataset,
+            top_n=limit, source_type="hadith",
+        )
+        seen_ids = {r.get("id") for r in results}
+        for sr in sem_results:
+            if sr.get("id") not in seen_ids:
+                results.append(sr)
+                seen_ids.add(sr.get("id"))
+        results = sorted(results, key=lambda x: x.get("_score", 0), reverse=True)[:limit]
+
     # Optional collection filter
     if collection:
         col_lower = collection.lower()
@@ -72,7 +96,11 @@ async def hadith_topic_search(
     top_k: int = Query(10, ge=1, le=20),
     grade_filter: Optional[str] = Query(None, description="Grade filter: sahih|hasan|all"),
 ):
-    """Search for Hadith related to a topic/theme using semantic search."""
+    """Search for Hadith related to a topic/theme using semantic search.
+
+    Finds hadiths about a topic even when the exact words don't appear (e.g. "patience", "charity").
+    Optionally filter by authenticity grade.
+    """
     check_ready()
     rewrite = await rewrite_query(topic, state.llm)
     results = await hybrid_search(
@@ -110,12 +138,13 @@ async def verify_hadith(
     """Verify if a Hadith is in authenticated collections and check its grade.
 
     Uses both semantic search and text matching for best accuracy.
+    Returns authenticity grade based on collection source.
     """
     check_ready()
     t0 = time.perf_counter()
 
     # 1. Try text search first for exact matches
-    text_results = text_search(q, state.dataset, source_type="hadith", limit=5)
+    text_results = text_search(q, state.dataset, source_type="hadith", limit=10)
     if collection:
         col_lower = collection.lower()
         text_results = [
@@ -123,15 +152,17 @@ async def verify_hadith(
             if col_lower in (r.get("collection", "") or r.get("reference", "")).lower()
         ]
 
-    # 2. Also try semantic search
+    # 2. Also try semantic search with auth intent for better matching
+    rewrite = await rewrite_query(q, state.llm)
+    rewrite["intent"] = "auth"  # Force auth intent for grade-aware ranking
     semantic_results = await hybrid_search(
-        q,
-        {"ar_query": q, "en_query": q, "keywords": q.split()[:7], "intent": "auth"},
+        q, rewrite,
         state.embed_model, state.faiss_index, state.dataset,
-        top_n=5, source_type="hadith",
+        top_n=10, source_type="hadith",
     )
 
-    # 3. Pick best result from either approach
+    # 3. Pick best result — prefer high-confidence text matches,
+    #    then semantic results, then lower-confidence text matches
     best = None
     if text_results and text_results[0].get("_score", 0) > 2.0:
         best = text_results[0]

app/routers/ops.py

@@ -1,16 +1,14 @@
-"""Operational endpoints — health, models, debug."""
+"""Operational endpoints — health, models."""
 
 from __future__ import annotations
 
 import time
-from typing import Optional
 
-from fastapi import APIRouter, Query
+from fastapi import APIRouter
 
 from app.config import cfg
 from app.models import ModelInfo, ModelsListResponse
-from app.search import hybrid_search, rewrite_query
-from app.state import check_ready, state
+from app.state import state
 
 router = APIRouter(tags=["ops"])
 
@@ -37,33 +35,3 @@ def list_models():
             ModelInfo(id="qmodel",  created=int(time.time()), owned_by="elgendy"),
         ]
     )
-
-
-@router.get("/debug/scores")
-async def debug_scores(
-    q: str = Query(..., min_length=1, max_length=1000),
-    top_k: int = Query(10, ge=1, le=20),
-):
-    """Debug: inspect raw retrieval scores without LLM generation."""
-    check_ready()
-    rewrite = await rewrite_query(q, state.llm)
-    results = await hybrid_search(
-        q, rewrite,
-        state.embed_model, state.faiss_index, state.dataset, top_k,
-    )
-    return {
-        "intent":    rewrite.get("intent"),
-        "threshold": cfg.CONFIDENCE_THRESHOLD,
-        "results": [
-            {
-                "rank":    i + 1,
-                "source":  r.get("source") or r.get("reference"),
-                "type":    r.get("type"),
-                "grade":   r.get("grade"),
-                "_dense":  round(r.get("_dense", 0), 4),
-                "_sparse": round(r.get("_sparse", 0), 4),
-                "_score":  round(r.get("_score", 0), 4),
-            }
-            for i, r in enumerate(results)
-        ],
-    }

app/routers/quran.py

@@ -35,7 +35,7 @@ async def quran_text_search(
 ):
     """Search for Quran verses by partial text match (Arabic or English).
 
-    This performs exact substring matching plus fuzzy word-overlap matching.
+    This performs exact substring matching, n-gram phrase matching, and word-overlap matching.
     Use this to find a verse when you know part of the text.
     """
     check_ready()
@@ -45,14 +45,15 @@ async def quran_text_search(
         count=len(results),
         results=[
             {
-                "surah_number":   r.get("surah_number"),
-                "surah_name_ar":  r.get("surah_name_ar", ""),
-                "surah_name_en":  r.get("surah_name_en", ""),
-                "ayah":           r.get("ayah_number") or r.get("verse_number"),
-                "arabic":         r.get("arabic", ""),
-                "english":        r.get("english", ""),
-                "source":         r.get("source", ""),
-                "score":          round(r.get("_score", 0), 4),
+                "surah_number":              r.get("surah_number"),
+                "surah_name_ar":             r.get("surah_name_ar", ""),
+                "surah_name_en":             r.get("surah_name_en", ""),
+                "surah_name_transliteration": r.get("surah_name_transliteration", ""),
+                "ayah":                      r.get("ayah_number") or r.get("verse_number"),
+                "arabic":                    r.get("arabic", ""),
+                "english":                   r.get("english", ""),
+                "source":                    r.get("source", ""),
+                "score":                     round(r.get("_score", 0), 4),
             }
             for r in results
         ],
@@ -67,7 +68,10 @@ async def quran_topic_search(
     topic: str = Query(..., min_length=1, max_length=500, description="Topic or theme to search for"),
     top_k: int = Query(10, ge=1, le=20),
 ):
-    """Search for Quran verses related to a topic/theme using semantic search."""
+    """Search for Quran verses related to a topic/theme using semantic search.
+
+    Finds verses about a topic even when the exact words don't appear (e.g. "patience", "charity").
+    """
     check_ready()
     rewrite = await rewrite_query(topic, state.llm)
     results = await hybrid_search(
@@ -80,14 +84,15 @@ async def quran_topic_search(
         count=len(results),
         results=[
             {
-                "surah_number":   r.get("surah_number"),
-                "surah_name_ar":  r.get("surah_name_ar", ""),
-                "surah_name_en":  r.get("surah_name_en", ""),
-                "ayah":           r.get("ayah_number") or r.get("verse_number"),
-                "arabic":         r.get("arabic", ""),
-                "english":        r.get("english", ""),
-                "source":         r.get("source", ""),
-                "score":          round(r.get("_score", 0), 4),
+                "surah_number":              r.get("surah_number"),
+                "surah_name_ar":             r.get("surah_name_ar", ""),
+                "surah_name_en":             r.get("surah_name_en", ""),
+                "surah_name_transliteration": r.get("surah_name_transliteration", ""),
+                "ayah":                      r.get("ayah_number") or r.get("verse_number"),
+                "arabic":                    r.get("arabic", ""),
+                "english":                   r.get("english", ""),
+                "source":                    r.get("source", ""),
+                "score":                     round(r.get("_score", 0), 4),
             }
             for r in results
         ],

app/search.py

@@ -36,24 +36,47 @@ Reply ONLY with a valid JSON object — no markdown, no preamble:
 }
 
 Intent Detection Rules (CRITICAL):
+- 'tafsir' intent = looking up Quranic verse(s) by partial text, topic, word, or asking about meaning
+  (ابحث عن آية, find verse, ما تفسير, verse about X, آية عن, الآية التي فيها, verse that says)
+  IMPORTANT: When user provides Arabic verse text to find, put that text in ar_query verbatim.
+- 'hadith' intent = looking up Hadith by text, topic, or asking about meaning (NOT authenticity)
+  (ابحث عن حديث, find hadith, hadith about, حديث عن, ما معنى حديث, hadith that says)
+  IMPORTANT: When user provides Arabic hadith text to find, put that text in ar_query verbatim.
+- 'auth' intent = asking about Hadith authenticity/grade/verification
+  (صحيح؟, هل صحيح, is it authentic, verify hadith, درجة الحديث, is this hadith real, هل هذا حديث صحيح)
+  IMPORTANT: Include the hadith text fragment in ar_query for matching.
 - 'surah_info' intent = asking about surah metadata: verse count, revelation type, surah number
   (كم عدد آيات سورة, كم آية في سورة, how many verses in surah, is surah X meccan/medinan)
-- 'count' intent = asking for WORD frequency/occurrence count (كم مرة ذُكرت كلمة, how many times is word X mentioned)
+- 'count' intent = asking for WORD frequency/occurrence count
+  (كم مرة ذُكرت كلمة, how many times is word X mentioned, عدد مرات ذكر كلمة)
   NOTE: "كم عدد آيات سورة" is surah_info NOT count!
-- 'auth' intent = asking about authenticity (صحيح؟, هل صحيح, is it authentic, verify hadith grade)
-- 'hadith' intent = asking about specific hadith meaning/text (not authenticity)
-- 'tafsir' intent = asking about Quranic verses or Islamic ruling (fatwa)
-- 'general' intent = other questions
+  IMPORTANT: The word being counted MUST be the first keyword.
+- 'fatwa' intent = asking for a religious ruling (ما حكم, is X halal/haram, حلال أم حرام)
+- 'general' intent = other Islamic questions
+
+Rewriting Rules:
+- For verse/hadith text lookups: include the EXACT Arabic text fragment in ar_query
+- For topic searches: expand the topic with Arabic synonyms and related terms in keywords
+- For word frequency: extract the EXACT keyword being counted as the FIRST keyword
+- keywords MUST include core Arabic terms for matching (e.g. صبر, رحمة, صلاة)
 
 Examples:
-- "كم عدد آيات سورة آل عمران" → intent: surah_info (asking about surah metadata!)
+- "ابحث عن الآية التي فيها إنا أعطيناك الكوثر" → intent: tafsir, ar_query: "إنا أعطيناك الكوثر"
+- "Find the verse about patience" → intent: tafsir, keywords: ["صبر", "patience", "الصبر"]
+- "ما الآية التي تتحدث عن الصدقة" → intent: tafsir, keywords: ["صدقة", "الصدقة", "إنفاق"]
+- "كم عدد آيات سورة آل عمران" → intent: surah_info
 - "كم آية في سورة البقرة" → intent: surah_info
 - "how many verses in surah al-baqara" → intent: surah_info
 - "هل سورة الفاتحة مكية أم مدنية" → intent: surah_info
-- "كم مرة ذُكرت كلمة مريم" → intent: count (asking about WORD frequency!)
-- "هل حديث إنما الأعمال بالنيات صحيح" → intent: auth (asking if authentic!)
-- "ما معنى حديث إنما الأعمال" → intent: hadith
-- "ما حكم الربا في الإسلام" → intent: fatwa
+- "كم مرة ذُكرت كلمة مريم في القرآن" → intent: count, keywords: ["مريم", ...]
+- "how many times is mercy mentioned in Quran" → intent: count, keywords: ["رحمة", "mercy", "الرحمة"]
+- "هل حديث إنما الأعمال بالنيات صحيح" → intent: auth, ar_query: "إنما الأعمال بالنيات"
+- "is the hadith about actions by intentions authentic" → intent: auth, keywords: ["إنما الأعمال بالنيات", "actions", "intentions"]
+- "ما معنى حديث إنما الأعمال" → intent: hadith, ar_query: "إنما الأعمال"
+- "ابحث عن حديث عن الصبر" → intent: hadith, keywords: ["صبر", "الصبر", "patience"]
+- "find hadith about fasting" → intent: hadith, keywords: ["صيام", "صوم", "fasting"]
+- "ما حكم الربا في الإسلام" → intent: fatwa, keywords: ["ربا", "الربا", "usury"]
+- "هل الحديث ده صحيح: من كان يؤمن بالله" → intent: auth, ar_query: "من كان يؤمن بالله"
 """
 
 
@@ -240,18 +263,35 @@ def text_search(
 
         score = 0.0
 
+        # Exact substring in raw Arabic (with diacritics) — highest priority
+        if query.strip() in ar_raw:
+            score = max(score, 5.0)
+
         # Exact substring in normalized Arabic
         if q_norm and q_norm in ar_norm:
             # Boost for shorter docs (more specific match)
-            score = 3.0 + (1.0 / max(len(ar_norm), 1)) * 100
+            score = max(score, 3.0 + (1.0 / max(len(ar_norm), 1)) * 100)
 
         # Exact substring in English
         if q_lower and q_lower in en_lower:
             score = max(score, 2.0 + (1.0 / max(len(en_lower), 1)) * 100)
 
-        # Exact substring in raw Arabic (with diacritics)
-        if query.strip() in ar_raw:
-            score = max(score, 4.0)
+        # N-gram phrase matching for partial Arabic text (3+ word sequences)
+        if score == 0.0 and q_norm:
+            q_words = q_norm.split()
+            if len(q_words) >= 3:
+                # Check sliding windows of 3 words from query against doc
+                for i in range(len(q_words) - 2):
+                    trigram = " ".join(q_words[i:i+3])
+                    if trigram in ar_norm:
+                        score = max(score, 2.0 + (i == 0) * 0.5)
+                        break
+            if score == 0.0 and len(q_words) >= 2:
+                for i in range(len(q_words) - 1):
+                    bigram = " ".join(q_words[i:i+2])
+                    if bigram in ar_norm or bigram in en_lower:
+                        score = max(score, 1.5)
+                        break
 
         # Word-level overlap for lower-confidence matches
         if score == 0.0:

app/state.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 import asyncio
 import json
 import logging
+import re
 import time
 from contextlib import asynccontextmanager
 from typing import Literal, Optional
@@ -28,6 +29,47 @@ from app.search import build_context, hybrid_search, rewrite_query, text_search
 logger = logging.getLogger("qmodel.state")
 
 
+# ═══════════════════════════════════════════════════════════════════════
+# POST-GENERATION HALLUCINATION CHECK
+# ═══════════════════════════════════════════════════════════════════════
+_QUOTE_RE = re.compile(r"❝\s*(.+?)\s*❞", re.DOTALL)
+
+
+def _verify_citations(answer: str, results: list) -> str:
+    """Check that quoted Arabic text in the answer actually appears in retrieved results.
+
+    If a quoted block doesn't match any source, replace it with a warning.
+    This prevents the model from fabricating hadith or verse text.
+    """
+    source_texts = set()
+    for r in results:
+        for field in ("arabic", "english", "text"):
+            val = r.get(field, "")
+            if val:
+                # Normalize whitespace for comparison
+                source_texts.add(re.sub(r"\s+", " ", val.strip()))
+
+    def _check_quote(m: re.Match) -> str:
+        quoted = re.sub(r"\s+", " ", m.group(1).strip())
+        # Check if any source text contains a significant portion of the quote
+        for src in source_texts:
+            # Use a substring match — LLMs sometimes trim edges
+            if len(quoted) < 10:
+                return m.group(0)  # too short to verify
+            if quoted in src or src in quoted:
+                return m.group(0)  # verified
+            # Check overlap: at least 60% of words match
+            q_words = set(quoted.split())
+            s_words = set(src.split())
+            if q_words and len(q_words & s_words) / len(q_words) >= 0.6:
+                return m.group(0)  # close enough match
+        # Quote not found in any source — flag it
+        logger.warning("Hallucination detected: quoted text not in sources: %.80s...", quoted)
+        return "❝ ⚠️ [تم حذف نص غير موثق — النص غير موجود في قاعدة البيانات] ❞"
+
+    return _QUOTE_RE.sub(_check_quote, answer)
+
+
 # ═══════════════════════════════════════════════════════════════════════
 # HADITH GRADE INFERENCE
 # ═══════════════════════════════════════════════════════════════════════
@@ -145,15 +187,27 @@ async def run_rag_pipeline(
     )
 
     # 2b. Text search fallback — catches exact matches missed by FAISS
-    #     (e.g. hadith text buried in long isnad chains)
-    #     Use rewritten ar_query (clean hadith text) + raw question for coverage.
+    #     For auth/hadith/tafsir intents, also search with the rewritten ar_query
+    #     which should contain the actual text fragment to look up.
     seen_ids = {r.get("id") for r in results}
     ar_q = rewrite.get("ar_query", "")
+
+    # Determine text search source filter based on intent
+    text_src = source_type
+    if not text_src and intent in ("tafsir", "count", "surah_info"):
+        text_src = "quran"
+    elif not text_src and intent in ("hadith", "auth"):
+        text_src = "hadith"
+
+    text_limit = top_k * 2 if intent in ("auth", "hadith", "tafsir") else top_k
     for q in dict.fromkeys([ar_q, question]):  # deduplicated, ar_query first
         if not q:
             continue
-        for hit in text_search(q, state.dataset, source_type, limit=top_k):
+        for hit in text_search(q, state.dataset, text_src, limit=text_limit):
             if hit.get("id") not in seen_ids:
+                # Boost text search hits for auth intent (exact text match is crucial)
+                if intent == "auth" and hit.get("_score", 0) > 2.0:
+                    hit["_score"] = hit["_score"] + 1.0
                 results.append(hit)
                 seen_ids.add(hit.get("id"))
     if len(results) > top_k:
@@ -176,8 +230,9 @@ async def run_rag_pipeline(
     # 3b. Word frequency count
     analysis = None
     if analysis_kw and not surah_info:
-        analysis = await count_occurrences(analysis_kw, state.dataset)
-        logger.info("Analysis: kw=%s count=%d", analysis_kw, analysis["total_count"])
+        count_src = "hadith" if intent in ("hadith", "auth") else "quran"
+        analysis = await count_occurrences(analysis_kw, state.dataset, source_type=count_src)
+        logger.info("Analysis: kw=%s src=%s count=%d", analysis_kw, count_src, analysis["total_count"])
 
     # 4. Language detection
     lang      = detect_language(question)
@@ -218,6 +273,9 @@ async def run_rag_pipeline(
         logger.error("LLM call failed: %s", exc)
         raise HTTPException(status_code=502, detail="LLM service unavailable")
 
+    # 7. Post-generation hallucination check — verify quoted text exists in sources
+    answer = _verify_citations(answer, results)
+
     latency = int((time.perf_counter() - t0) * 1000)
     logger.info(
         "Pipeline done | intent=%s | lang=%s | top_score=%.3f | %d ms",

main.py

@@ -33,7 +33,7 @@ logging.basicConfig(
 
 from app.config import cfg
 from app.state import lifespan
-from app.routers import chat, hadith, ops, quran
+from app.routers import chat, ops
 
 # ═══════════════════════════════════════════════════════════════════════
 # FASTAPI APP
@@ -43,11 +43,9 @@ app = FastAPI(
     description=(
         "Specialized Quran & Hadith system with dual LLM backend.\n\n"
         "**Capabilities:**\n"
-        "- Quran verse lookup by text or topic\n"
-        "- Quran word frequency & analytics\n"
-        "- Hadith lookup by text or topic\n"
-        "- Hadith authenticity verification\n"
-        "- OpenAI-compatible chat completions"
+        "- OpenAI-compatible chat completions\n"
+        "- Streaming support\n"
+        "- Islamic knowledge RAG pipeline"
     ),
     version="5.0.0",
     lifespan=lifespan,
@@ -64,8 +62,6 @@ app.add_middleware(
 # Register routers
 app.include_router(ops.router)
 app.include_router(chat.router)
-app.include_router(quran.router)
-app.include_router(hadith.router)
 
 
 if __name__ == "__main__":