v4.0: Add app.py — OCR + RAG Chatbot + Clause Redlining

app.py

@@ -1,7 +1,12 @@
 """
-ClauseGuard — World's Best Legal Contract Analysis Tool (v3.0)
+ClauseGuard — World's Best Legal Contract Analysis Tool (v4.0)
 ═══════════════════════════════════════════════════════════════
-Fixes in v3.0:
+New in v4.0:
+  • OCR support for scanned PDFs (docTR engine with smart native/scanned routing)
+  • Contract Q&A Chatbot (RAG: embedding retrieval + HF Inference API streaming)
+  • Clause Redlining (3-tier: template lookup + RAG + LLM refinement)
+
+Carried from v3.0:
   • Fixed CUAD label mapping (added missing index 6: "Notice Period to Terminate Renewal")
   • Switched from softmax → sigmoid for proper multi-label classification
   • Per-class optimized thresholds instead of flat 0.15
@@ -21,6 +26,9 @@ Models:
     (LoRA adapter on nlpaueb/legal-bert-base-uncased, 41 CUAD classes)
   • Legal NER: matterstack/legal-bert-ner (token classification)
   • NLI: cross-encoder/nli-deberta-v3-base (contradiction detection)
+  • Embeddings: sentence-transformers/all-MiniLM-L6-v2 (RAG retrieval)
+  • OCR: docTR fast_base + crnn_vgg16_bn (scanned PDF extraction)
+  • LLM: Qwen/Qwen2.5-7B-Instruct via HF Inference API (chatbot + redlining)
 """
 
 import os
@@ -71,6 +79,9 @@ except Exception:
 from compare import compare_contracts, render_comparison_html
 from obligations import extract_obligations, render_obligations_html
 from compliance import check_compliance, render_compliance_html
+from ocr_engine import parse_pdf_smart, get_ocr_status
+from chatbot import index_contract, chat_respond, get_chatbot_status
+from redlining import generate_redlines, render_redlines_html
 
 # ═══════════════════════════════════════════════════════════════════════
 # 1. CONFIGURATION — FIXED label mapping (41 labels, index 6 restored)
@@ -335,20 +346,15 @@ _load_nli_model()
 # ═══════════════════════════════════════════════════════════════════════
 
 def parse_pdf(file_path):
-    if not _HAS_PDF:
-        return None, "PDF parsing not available (pdfplumber not installed)"
-    try:
-        text = ""
-        with pdfplumber.open(file_path) as pdf:
-            for page in pdf.pages:
-                page_text = page.extract_text()
-                if page_text:
-                    text += page_text + "\n\n"
-        if not text.strip():
-            return None, "PDF appears to be scanned/image-based. OCR is not yet supported. Please use a digital PDF or paste text directly."
-        return text.strip(), None
-    except Exception as e:
-        return None, f"PDF parse error: {e}"
+    """Smart PDF parser: native text extraction with OCR fallback for scanned PDFs."""
+    text, error, method = parse_pdf_smart(file_path)
+    if text:
+        if method == "ocr":
+            print(f"[ClauseGuard] PDF extracted via OCR ({len(text)} chars)")
+        return text, None
+    if error:
+        return None, error
+    return None, "Could not extract text from PDF. Try uploading a clearer scan or digital PDF."
 
 def parse_docx(file_path):
     if not _HAS_DOCX:
@@ -1196,11 +1202,11 @@ def process_upload(file):
 def run_analysis(text):
     if not text or len(text.strip()) < 50:
         err_html = '<p style="color:#dc2626;padding:16px;">Document too short (minimum 50 characters)</p>'
-        return [err_html] * 7 + [None, None, ""]
+        return [err_html] * 8 + [None, None, "", None]
     result, error = analyze_contract(text)
     if error:
         err_html = f'<p style="color:#dc2626;padding:16px;">{error}</p>'
-        return [err_html] * 7 + [None, None, error]
+        return [err_html] * 8 + [None, None, error, None]
 
     # FIXED: per-session temp files
     session_id = uuid.uuid4().hex[:8]
@@ -1213,6 +1219,10 @@ def run_analysis(text):
     with open(csv_path, "w") as f:
         f.write(csv_content)
 
+    # Generate redline suggestions (Tier 1 template + Tier 3 LLM for critical/high)
+    redlines = generate_redlines(result, use_llm=True)
+    redlines_html = render_redlines_html(redlines)
+
     return [
         render_summary(result),
         render_clause_cards(result),
@@ -1221,13 +1231,15 @@ def run_analysis(text):
         render_document_viewer(result),
         render_obligations_html(result.get("obligations", [])),
         render_compliance_html(result.get("compliance", {})),
+        redlines_html,
         json_path,
         csv_path,
         "Analysis complete",
+        result,  # Store analysis result for chatbot
     ]
 
 def do_clear():
-    return [""] * 7 + [None, None, ""]
+    return [""] * 8 + [None, None, "", None]
 
 # ── Example contracts ──
 SPOTIFY_TOS = """By using the Spotify Service, you agree to be bound by these Terms of Use.
@@ -1311,17 +1323,22 @@ with gr.Blocks(
     """
 ) as demo:
 
+    # ── Shared State (for chatbot RAG) ──────────────────────────────
+    analysis_state = gr.State(None)       # Full analysis result dict
+    chunks_state = gr.State([])           # Contract text chunks for RAG
+    embeddings_state = gr.State(None)     # Chunk embeddings (numpy array)
+
     gr.HTML("""
     <div style="display:flex;align-items:center;justify-content:space-between;padding:12px 0;border-bottom:2px solid #e5e7eb;margin-bottom:16px;">
       <div>
         <h1 style="font-size:24px;font-weight:700;margin:0;color:#1f2937;">🛡️ ClauseGuard</h1>
-        <p style="font-size:13px;color:#6b7280;margin:4px 0 0 0;">AI-Powered Legal Contract Analysis · 41 Clause Categories · Risk Scoring · ML NER · NLI Contradictions · Compliance · Obligations</p>
+        <p style="font-size:13px;color:#6b7280;margin:4px 0 0 0;">AI-Powered Legal Contract Analysis · 41 Clause Categories · Risk Scoring · ML NER · NLI Contradictions · Compliance · Obligations · <strong>Q&A Chatbot</strong> · <strong>Clause Redlining</strong> · <strong>OCR</strong></p>
       </div>
-      <div style="font-size:12px;color:#9ca3af;">v3.0 · Precision Legal AI</div>
+      <div style="font-size:12px;color:#9ca3af;">v4.0 · Precision Legal AI</div>
     </div>
     """)
 
-    # ── Main Tabs: Analysis vs Comparison ──
+    # ── Main Tabs: Analysis vs Comparison vs Chatbot ──
     with gr.Tabs():
 
         # ═══════ TAB 1: Single Contract Analysis ═══════
@@ -1338,7 +1355,7 @@ with gr.Blocks(
                 with gr.Column(scale=3):
                     text_input = gr.Textbox(
                         label="📄 Contract Text",
-                        placeholder="Paste contract text here, or upload a file above...",
+                        placeholder="Paste contract text here, or upload a file above...\n\n💡 Scanned PDFs are automatically processed with OCR.",
                         lines=14,
                         max_lines=40,
                         show_copy_button=True,
@@ -1381,6 +1398,8 @@ with gr.Blocks(
                             obligations_html = gr.HTML(label="Obligation Tracker")
                         with gr.Tab("⚖️ Compliance"):
                             compliance_html = gr.HTML(label="Compliance Checker")
+                        with gr.Tab("✏️ Redlining"):
+                            redlining_html = gr.HTML(label="Clause Redlining Suggestions")
 
         # ═══════ TAB 2: Contract Comparison ═══════
         with gr.Tab("🔀 Compare Contracts"):
@@ -1429,6 +1448,53 @@ with gr.Blocks(
                 with gr.Column(scale=2):
                     comp_json = gr.JSON(label="Raw Comparison Data")
 
+        # ═══════ TAB 3: Contract Q&A Chatbot ═══════
+        with gr.Tab("💬 Contract Q&A"):
+            gr.HTML("""
+            <div style="padding:12px 16px;background:linear-gradient(135deg,#eff6ff,#faf5ff);border-radius:10px;margin-bottom:12px;border:1px solid #e5e7eb;">
+                <div style="display:flex;align-items:center;gap:8px;margin-bottom:6px;">
+                    <span style="font-size:20px;">💬</span>
+                    <h3 style="margin:0;font-size:16px;color:#1f2937;">Contract Q&A Chatbot</h3>
+                </div>
+                <p style="font-size:12px;color:#6b7280;margin:0;line-height:1.5;">
+                    Ask questions about your analyzed contract. The chatbot uses <strong>RAG</strong> (Retrieval-Augmented Generation) 
+                    to find relevant clauses and generate accurate answers grounded in your contract text.
+                    <br>
+                    <strong>Step 1:</strong> Analyze a contract in the "📄 Single Contract Analysis" tab.
+                    <strong>Step 2:</strong> Come here and ask questions!
+                </p>
+            </div>
+            """)
+
+            chatbot_index_status = gr.Textbox(
+                label="📡 Chatbot Index Status",
+                interactive=False,
+                lines=1,
+                value="⏳ No contract indexed yet — analyze a contract first",
+            )
+
+            def _chatbot_fn(message, history, chunks, embeddings, analysis):
+                """Wrapper for ChatInterface fn signature."""
+                yield from chat_respond(message, history, chunks, embeddings, analysis)
+
+            gr.ChatInterface(
+                fn=_chatbot_fn,
+                type="messages",
+                additional_inputs=[chunks_state, embeddings_state, analysis_state],
+                examples=[
+                    "What are the main risks in this contract?",
+                    "Who are the parties involved?",
+                    "What happens if the contract is terminated?",
+                    "Are there any liability limitations?",
+                    "What are my obligations under this contract?",
+                    "Is there an arbitration clause?",
+                    "What is the governing law?",
+                    "Summarize the key terms in plain language.",
+                ],
+                title="",
+                description="",
+            )
+
     # ── Events ──
     def _load_file(file):
         text, err = parse_document(file) if file else ("", "No file")
@@ -1436,23 +1502,41 @@ with gr.Blocks(
             return "", err
         return text, "Loaded successfully" if not err else err
 
+    def _analysis_and_index(text):
+        """Run analysis AND index for chatbot in one call."""
+        # Run the standard analysis
+        analysis_outputs = run_analysis(text)
+
+        # Index for chatbot (uses the raw text)
+        chunks, embeddings, index_status = index_contract(text)
+
+        # analysis_outputs has 12 items: 8 HTML + json_path + csv_path + status + result
+        # We need to add: chunks_state, embeddings_state, chatbot_index_status
+        return analysis_outputs + [chunks, embeddings, index_status]
+
     load_btn.click(_load_file, inputs=[file_input], outputs=[text_input, load_status])
     comp_load_a.click(_load_file, inputs=[comp_file_a], outputs=[comp_text_a, comp_status_a])
     comp_load_b.click(_load_file, inputs=[comp_file_b], outputs=[comp_text_b, comp_status_b])
 
     scan_btn.click(
-        run_analysis,
+        _analysis_and_index,
         inputs=[text_input],
-        outputs=[summary_html, clauses_html, entities_html, nli_html,
-                 doc_html, obligations_html, compliance_html,
-                 json_file, csv_file, status_msg]
+        outputs=[
+            summary_html, clauses_html, entities_html, nli_html,
+            doc_html, obligations_html, compliance_html, redlining_html,
+            json_file, csv_file, status_msg, analysis_state,
+            chunks_state, embeddings_state, chatbot_index_status,
+        ]
     )
 
     clear_btn.click(
-        do_clear,
-        outputs=[summary_html, clauses_html, entities_html, nli_html,
-                 doc_html, obligations_html, compliance_html,
-                 json_file, csv_file, status_msg]
+        lambda: [""] * 8 + [None, None, "", None, [], None, "⏳ No contract indexed"],
+        outputs=[
+            summary_html, clauses_html, entities_html, nli_html,
+            doc_html, obligations_html, compliance_html, redlining_html,
+            json_file, csv_file, status_msg, analysis_state,
+            chunks_state, embeddings_state, chatbot_index_status,
+        ]
     )
 
     comp_btn.click(
@@ -1468,6 +1552,8 @@ with gr.Blocks(
         · Model: <a href="https://huggingface.co/Mokshith31/legalbert-contract-clause-classification" style="color:#6b7280;">Legal-BERT + CUAD (41 classes)</a>
         · NER: <a href="https://huggingface.co/matterstack/legal-bert-ner" style="color:#6b7280;">Legal-BERT NER</a>
         · NLI: <a href="https://huggingface.co/cross-encoder/nli-deberta-v3-base" style="color:#6b7280;">DeBERTa-v3 NLI</a>
+        · LLM: <a href="https://huggingface.co/Qwen/Qwen2.5-7B-Instruct" style="color:#6b7280;">Qwen2.5-7B</a>
+        · OCR: <a href="https://github.com/mindee/doctr" style="color:#6b7280;">docTR</a>
         · Dataset: <a href="https://huggingface.co/datasets/theatticusproject/cuad-qa" style="color:#6b7280;">CUAD</a>
         · <a href="https://huggingface.co/spaces/gaurv007/ClauseGuard" style="color:#6b7280;">ClauseGuard Space</a>
       </p>