TheTruthSchool_RAG

.dockerignore

@@ -0,0 +1,68 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+*.egg-info/
+dist/
+build/
+*.egg
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+
+# Virtual Environments
+venv/
+env/
+ENV/
+.venv
+
+# IDEs
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Git
+.git/
+.gitignore
+.gitattributes
+
+# Documentation
+*.md
+!README.md
+
+# Logs
+*.log
+logs/
+
+# Environment files (will be passed via docker-compose)
+.env
+.env.*
+
+# Storage and uploads (will be mounted as volumes)
+storage/
+uploads/
+backend/output/
+
+# Frontend
+frontend/node_modules/
+frontend/build/
+frontend/.env
+frontend/.env.local
+
+# Temporary files
+tmp/
+temp/
+*.tmp
+
+# Docker
+Dockerfile
+docker-compose*.yml
+.dockerignore

.env.docker.example

@@ -0,0 +1,15 @@
+
+GEMINI_API_KEY=your_gemini_api_key_here
+
+GEMINI_TEXT_MODEL=models/gemini-flash-latest
+GEMINI_VERIFIER_MODEL=models/gemini-pro-latest
+GEMINI_VISION_MODEL=models/gemini-flash-latest
+GEMINI_EMBEDDING_MODEL=models/text-embedding-004
+
+TAVILY_API_KEY=your_tavily_api_key_here
+
+REACT_APP_BACKEND_URL=http://localhost:8000
+
+BACKEND_PORT=8000
+
+FRONTEND_PORT=3000

.env.example

@@ -0,0 +1,51 @@
+# Agentic RAG System - Environment Variables
+# Copy this file to .env and fill in your actual API keys
+
+# ============================================
+# Required: Google Gemini API Key
+# ============================================
+# Get your free API key from: https://makersuite.google.com/app/apikey
+GEMINI_API_KEY=your_gemini_api_key_here
+
+# ============================================
+# Gemini Model Configuration (Optional)
+# ============================================
+# Text generation model (fast responses)
+GEMINI_TEXT_MODEL=models/gemini-flash-latest
+
+# Verification model (quality checking)
+GEMINI_VERIFIER_MODEL=models/gemini-pro-latest
+
+# Vision model (image processing)
+GEMINI_VISION_MODEL=models/gemini-flash-latest
+
+# Embedding model (vector embeddings)
+GEMINI_EMBEDDING_MODEL=models/text-embedding-004
+
+# ============================================
+# Optional: Tavily API Key (Web Search)
+# ============================================
+# Get your free API key from: https://tavily.com
+# Leave empty to disable web search features
+TAVILY_API_KEY=your_tavily_api_key_here
+
+# ============================================
+# Application Configuration
+# ============================================
+# Backend API URL (used by frontend)
+REACT_APP_BACKEND_URL=http://localhost:8000
+
+# Backend port
+BACKEND_PORT=8000
+
+# Frontend port
+FRONTEND_PORT=3000
+
+# ============================================
+# Hugging Face Space Configuration
+# ============================================
+# When deploying to Hugging Face Spaces:
+# 1. Go to your Space settings
+# 2. Add secrets for GEMINI_API_KEY and TAVILY_API_KEY
+# 3. The Dockerfile will use port 7860 automatically
+# 4. REACT_APP_BACKEND_URL will be set to /api automatically

.gitignore

@@ -0,0 +1,92 @@
+# Dependencies
+node_modules/
+frontend/node_modules/
+backend/__pycache__/
+venv/
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+.Python
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Build outputs
+frontend/build/
+dist/
+build/
+*.egg-info/
+.eggs/
+
+# Cache
+frontend/node_modules/.cache/
+.cache/
+.pytest_cache/
+.mypy_cache/
+
+# Environment variables
+.env
+.env.local
+.env.*.local
+*.env
+backend/.env
+frontend/.env
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# OS
+.DS_Store
+Thumbs.db
+*.log
+
+# Output files
+backend/output/
+*.out
+
+# Storage (runtime data - don't commit to git)
+storage/
+uploads/
+rag_anything_smaranika/__pycache__/
+
+# Logs
+logs/
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# Test coverage
+htmlcov/
+.coverage
+.coverage.*
+coverage.xml
+*.cover
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# Docker
+*.pid
+.docker/
+
+# Temporary files
+tmp/
+temp/
+*.tmp
+
+# Binary assets (use Git LFS if needed)
+*.png
+*.jpg
+*.jpeg
+*.gif
+*.svg
+*.ico
+
+# Local LightRAG package in vendor directory
+vendor/lightrag/__pycache__/
+vendor/**/__pycache__/

Dockerfile

@@ -0,0 +1,178 @@
+
+FROM python:3.12-slim
+
+
+WORKDIR /app
+
+
+RUN apt-get update && apt-get install -y \
+    curl \
+    wget \
+    git \
+    build-essential \
+    nginx \
+    nodejs \
+    npm \
+    && rm -rf /var/lib/apt/lists/*
+
+
+COPY backend/requirements.txt /app/backend/requirements.txt
+RUN pip install --no-cache-dir --use-pep517 -r /app/backend/requirements.txt
+
+# Copy local modified LightRAG package in vendor directory
+COPY vendor/ /app/vendor/
+
+COPY backend/ /app/backend/
+COPY rag_anything_smaranika/ /app/rag_anything_smaranika/
+
+
+COPY frontend/ /app/frontend/
+
+
+WORKDIR /app/frontend
+RUN npm install
+RUN REACT_APP_API_URL=/api npm run build
+
+
+
+RUN mkdir -p /var/lib/nginx/body /var/lib/nginx/fastcgi \
+    /var/lib/nginx/proxy /var/lib/nginx/scgi /var/lib/nginx/uwsgi \
+    /var/log/nginx /var/cache/nginx && \
+    chmod -R 777 /var/lib/nginx /var/log/nginx /var/cache/nginx && \
+    touch /var/run/nginx.pid && chmod 666 /var/run/nginx.pid
+
+
+RUN echo 'pid /tmp/nginx.pid;\n\
+error_log /var/log/nginx/error.log;\n\
+events {\n\
+    worker_connections 1024;\n\
+}\n\
+http {\n\
+    include /etc/nginx/mime.types;\n\
+    default_type application/octet-stream;\n\
+    access_log /var/log/nginx/access.log;\n\
+    client_body_temp_path /tmp/client_body;\n\
+    proxy_temp_path /tmp/proxy;\n\
+    fastcgi_temp_path /tmp/fastcgi;\n\
+    uwsgi_temp_path /tmp/uwsgi;\n\
+    scgi_temp_path /tmp/scgi;\n\
+    \n\
+    server {\n\
+        listen 7860;\n\
+        server_name _;\n\
+        \n\
+        location / {\n\
+            root /app/frontend/build;\n\
+            try_files $uri $uri/ /index.html;\n\
+        }\n\
+        \n\
+        location /api/ {\n\
+            proxy_pass http://127.0.0.1:8000/;\n\
+            proxy_http_version 1.1;\n\
+            proxy_set_header Upgrade $http_upgrade;\n\
+            proxy_set_header Connection "upgrade";\n\
+            proxy_set_header Host $host;\n\
+            proxy_set_header X-Real-IP $remote_addr;\n\
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n\
+            proxy_set_header X-Forwarded-Proto $scheme;\n\
+            proxy_buffering off;\n\
+            proxy_cache_bypass $http_upgrade;\n\
+        }\n\
+        \n\
+        location /health {\n\
+            proxy_pass http://127.0.0.1:8000/health;\n\
+            proxy_http_version 1.1;\n\
+            proxy_set_header Host $host;\n\
+        }\n\
+    }\n\
+}' > /etc/nginx/nginx.conf
+
+
+RUN mkdir -p /app/storage /app/uploads /app/backend/output /app/output /app/.cache/huggingface && \
+    chmod -R 777 /app/storage /app/uploads /app/backend/output /app/output /app/.cache
+
+WORKDIR /app
+
+WORKDIR /app/rag_anything_smaranika
+RUN pip install --no-cache-dir -e .
+
+WORKDIR /app
+
+RUN mkdir -p /app/storage/medical /app/storage/legal /app/storage/financial \
+    /app/storage/technical /app/storage/academic && \
+    chmod -R 777 /app/storage
+
+# Create output directory in the working directory for the parser
+RUN mkdir -p /app/output && chmod -R 777 /app/output
+
+
+RUN echo '#!/bin/bash\n\
+set -e\n\
+\n\
+echo "===== Application Startup at $(date +"%Y-%m-%d %H:%M:%S") ====="\n\
+echo ""\n\
+echo "Starting Agentic RAG System for Hugging Face Space..."\n\
+\n\
+# Check for required environment variables\n\
+if [ -z "$GEMINI_API_KEY" ]; then\n\
+    echo "ERROR: GEMINI_API_KEY environment variable is not set!"\n\
+    echo "Please set it in your Hugging Face Space settings."\n\
+    exit 1\n\
+fi\n\
+\n\
+# Start backend in background\n\
+echo "Starting FastAPI backend on port 8000..."\n\
+cd /app\n\
+export PYTHONPATH=/app:/app/vendor:$PYTHONPATH\n\
+python -m uvicorn backend.main:app --host 127.0.0.1 --port 8000 --log-level info &\n\
+BACKEND_PID=$!\n\
+\n\
+# Wait for backend to be ready\n\
+echo "Waiting for backend to be ready..."\n\
+for i in {1..30}; do\n\
+    if curl -s http://127.0.0.1:8000/health > /dev/null 2>&1; then\n\
+        echo "Backend is ready!"\n\
+        break\n\
+    fi\n\
+    echo "Waiting for backend... ($i/30)"\n\
+    sleep 2\n\
+done\n\
+\n\
+# Start nginx\n\
+echo "Starting nginx on port 7860..."\n\
+nginx -g "daemon off;" &\n\
+NGINX_PID=$!\n\
+\n\
+echo ""\n\
+echo "==========================================="\n\
+echo "Agentic RAG System is running!"\n\
+echo "Backend: http://localhost:8000"\n\
+echo "Frontend: http://localhost:7860"\n\
+echo "API Docs: http://localhost:8000/docs"\n\
+echo "==========================================="\n\
+echo ""\n\
+\n\
+# Wait for both processes\n\
+wait $BACKEND_PID $NGINX_PID\n\
+' > /app/start.sh && chmod +x /app/start.sh
+
+
+EXPOSE 7860
+
+
+HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \
+    CMD curl -f http://localhost:7860/health || exit 1
+
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONPATH=/app:/app/vendor:$PYTHONPATH
+ENV BACKEND_PORT=8000
+ENV FRONTEND_PORT=7860
+ENV HF_HOME=/app/.cache/huggingface
+ENV TRANSFORMERS_CACHE=/app/.cache/huggingface
+ENV HF_DATASETS_CACHE=/app/.cache/huggingface/datasets
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONHASHSEED=0
+ENV PYTHONOPTIMIZE=0
+
+# Start the application
+CMD ["/app/start.sh"]

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Agentic RAG System Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,269 @@
+---
+title: Agentic RAG System
+emoji: 🤖
+colorFrom: blue
+colorTo: green
+sdk: docker
+app_port: 7860
+pinned: false
+license: mit
+---
+
+# 🤖 Agentic RAG System
+
+A production-ready **Retrieval-Augmented Generation (RAG)** system with multi-domain support, advanced AI features, and intelligent web search integration. Built with FastAPI, React, and Google Gemini API.
+
+## ✨ Features
+
+### 🎯 Multi-Domain Support
+- **Medical & Healthcare**: Medical documents, research papers, clinical guidelines
+- **Legal & Compliance**: Legal documents, contracts, regulations, case law
+- **Financial & Analytics**: Financial reports, analysis, market research
+- **Technical Documentation**: Technical docs, APIs, code, architecture
+- **Academic Research**: Research papers, academic publications, studies
+
+### 🚀 Advanced AI Capabilities
+- **Query Improvement**: Automatic query enhancement with abbreviation expansion
+- **Dual-LLM Verification**: Two-stage answer verification using Gemini Pro
+- **Web Search Integration**: Augment answers with real-time web search via Tavily
+- **Conversation Memory**: Context-aware responses with conversation history
+- **Multimodal Processing**: Support for images, tables, and equations (MinerU parser)
+- **Smart Reranking**: Gemini-powered relevance reranking for better results
+- **Streaming Responses**: Real-time token streaming for responsive UX
+
+### 🔧 Technical Features
+- **Gemini API Integration**: Free-tier Gemini Flash & Pro models
+- **Async Processing**: Background document processing with status tracking
+- **RESTful API**: Clean, well-documented FastAPI endpoints
+- **Modern React Frontend**: Beautiful, responsive UI with Tailwind CSS
+- **Docker Support**: One-command deployment with docker-compose
+- **Performance Optimized**: Query caching, fast mode (2-3x speedup), batch processing
+
+## 🚀 Quick Start (Docker)
+
+### Prerequisites
+- Docker and Docker Compose
+- Google Gemini API Key ([Get one free](https://makersuite.google.com/app/apikey))
+- (Optional) Tavily API Key for web search ([Get one free](https://tavily.com))
+
+### 1. Clone the Repository
+```bash
+git clone <your-repo-url>
+cd Agentic_RAG
+```
+
+### 2. Set Up Environment Variables
+Create a `.env` file in the project root:
+```bash
+GEMINI_API_KEY=your_gemini_api_key_here
+GEMINI_TEXT_MODEL=models/gemini-flash-latest
+GEMINI_VERIFIER_MODEL=models/gemini-pro-latest
+GEMINI_VISION_MODEL=models/gemini-flash-latest
+GEMINI_EMBEDDING_MODEL=models/text-embedding-004
+TAVILY_API_KEY=your_tavily_api_key_here  # Optional, for web search
+```
+
+### 3. Start the Application
+```bash
+docker-compose up -d
+```
+
+### 4. Access the Application
+- **Frontend**: http://localhost:3000
+- **Backend API**: http://localhost:8000
+- **API Docs**: http://localhost:8000/docs
+
+## 📖 Usage
+
+### Upload Documents
+1. Navigate to the frontend at http://localhost:3000
+2. Select a domain (Medical, Legal, Financial, Technical, or Academic)
+3. Upload PDF, DOCX, TXT, or other supported documents
+4. Wait for processing to complete (tracked with real-time status)
+
+### Query Documents
+1. Enter your question in the query interface
+2. Select query mode:
+   - **Mix**: Balanced combination of local and global search (recommended)
+   - **Local**: Focused chunk-based search
+   - **Global**: Knowledge graph entity search
+   - **Hybrid**: Advanced combination
+   - **Web**: RAG + real-time web search
+3. Toggle advanced features:
+   - **Query Improvement**: Enhance your query automatically
+   - **Verification**: Dual-LLM quality check
+   - **Web Search**: Augment with real-time web results
+   - **Fast Mode**: 2-3x faster queries (slightly lower quality)
+4. Get streaming responses with sources and confidence scores
+
+### API Usage
+```python
+import requests
+
+# Upload document
+files = {"file": open("document.pdf", "rb")}
+data = {"domain": "medical"}
+response = requests.post("http://localhost:8000/upload", files=files, data=data)
+print(response.json())
+
+# Query documents
+query_data = {
+    "query": "What are the treatment options for hypertension?",
+    "domain": "medical",
+    "mode": "mix",
+    "enable_web_search": False,
+    "fast_mode": False,
+    "return_metadata": True
+}
+response = requests.post("http://localhost:8000/query", json=query_data)
+print(response.json())
+```
+
+## 🏗️ Architecture
+
+```
+Agentic_RAG/
+├── backend/                 # FastAPI backend
+│   ├── main.py             # Main API server
+│   ├── reranker.py         # Gemini-powered reranking
+│   ├── web_search.py       # Tavily web search integration
+│   ├── url_fetcher.py      # URL content fetching
+│   ├── requirements.txt    # Python dependencies
+│   └── Dockerfile          # Backend container
+├── frontend/               # React frontend
+│   ├── src/               # React components
+│   ├── public/            # Static assets
+│   ├── package.json       # Node dependencies
+│   ├── Dockerfile         # Frontend container
+│   └── nginx.conf         # Nginx configuration
+├── storage/               # RAG storage (created at runtime)
+│   ├── medical/          # Medical domain storage
+│   ├── legal/            # Legal domain storage
+│   └── ...               # Other domains
+├── uploads/              # Uploaded documents
+├── docker-compose.yml    # Docker orchestration
+├── Dockerfile            # Hugging Face Space Dockerfile
+└── README.md            # This file
+```
+
+## 🔑 Environment Variables
+
+| Variable | Description | Required | Default |
+|----------|-------------|----------|---------|
+| `GEMINI_API_KEY` | Google Gemini API key | Yes | - |
+| `GEMINI_TEXT_MODEL` | Text generation model | No | `models/gemini-flash-latest` |
+| `GEMINI_VERIFIER_MODEL` | Verification model | No | `models/gemini-pro-latest` |
+| `GEMINI_VISION_MODEL` | Vision processing model | No | `models/gemini-flash-latest` |
+| `GEMINI_EMBEDDING_MODEL` | Embedding model | No | `models/text-embedding-004` |
+| `TAVILY_API_KEY` | Tavily web search API key | No | - |
+
+## 📊 API Endpoints
+
+### Health Check
+```bash
+GET /health
+```
+
+### List Domains
+```bash
+GET /domains
+```
+
+### Upload Document
+```bash
+POST /upload
+Content-Type: multipart/form-data
+
+file: <document file>
+domain: medical
+```
+
+### Query Documents (Streaming)
+```bash
+POST /query/stream
+Content-Type: application/json
+
+{
+  "query": "What are the treatment options?",
+  "domain": "medical",
+  "mode": "mix",
+  "enable_web_search": false,
+  "fast_mode": false
+}
+```
+
+### Query Documents (Standard)
+```bash
+POST /query
+Content-Type: application/json
+
+{
+  "query": "What are the treatment options?",
+  "domain": "medical",
+  "mode": "mix"
+}
+```
+
+### Check Processing Status
+```bash
+GET /status/{processing_id}
+```
+
+### List Documents
+```bash
+GET /documents?domain=medical
+```
+
+### Delete Document
+```bash
+DELETE /documents/{doc_id}
+```
+
+## 🎯 Performance
+
+- **Fast Mode**: 2-3x faster queries with optimized parameters
+- **Query Caching**: 5-minute TTL cache for repeated queries
+- **Batch Processing**: Parallel document processing (up to 10 documents)
+- **Streaming**: Real-time token streaming for responsive UX
+- **Reranking**: Gemini-powered relevance scoring
+
+## 🛠️ Development
+
+### Backend Development
+```bash
+cd backend
+pip install -r requirements.txt
+python main.py
+```
+
+### Frontend Development
+```bash
+cd frontend
+npm install
+npm start
+```
+
+## 🤝 Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.
+
+## 📝 License
+
+This project is licensed under the MIT License - see the LICENSE file for details.
+
+## 🙏 Acknowledgments
+
+- [LightRAG](https://github.com/HKUDS/LightRAG) - RAG framework
+- [Google Gemini](https://ai.google.dev/) - LLM and embeddings
+- [Tavily](https://tavily.com/) - Web search API
+- [MinerU](https://github.com/opendatalab/MinerU) - Document parsing
+- [FastAPI](https://fastapi.tiangolo.com/) - Backend framework
+- [React](https://react.dev/) - Frontend framework
+
+## 📧 Support
+
+For issues and questions, please open an issue on GitHub.
+
+---
+
+Built with ❤️ using FastAPI, React, and Google Gemini

backend/Dockerfile

@@ -0,0 +1,44 @@
+FROM python:3.12-slim
+
+# Set working directory
+WORKDIR /app
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    curl \
+    git \
+    wget \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy backend requirements first for better caching
+COPY backend/requirements.txt /app/backend/requirements.txt
+
+# Install Python dependencies
+RUN pip install --no-cache-dir -r /app/backend/requirements.txt
+
+# Copy rag_anything_smaranika (contains raganything module)
+COPY rag_anything_smaranika /app/rag_anything_smaranika
+
+# Install raganything as an editable package
+RUN pip install -e /app/rag_anything_smaranika
+
+# Copy backend application code
+COPY backend /app/backend
+
+# Create necessary directories
+RUN mkdir -p /app/storage /app/uploads /app/backend/output
+
+# Set environment variables
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONPATH=/app
+
+# Expose port
+EXPOSE 8000
+
+# Health check
+HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \
+    CMD curl -f http://localhost:8000/health || exit 1
+
+# Run the application
+CMD ["uvicorn", "backend.main:app", "--host", "0.0.0.0", "--port", "8000"]

backend/README.md

@@ -0,0 +1,353 @@
+# Enhanced RAG-Anything Backend API
+
+Production-ready FastAPI backend for the RAG-Anything system with multi-domain support and advanced AI features.
+
+## Features
+
+### 🎯 Multi-Domain Support
+- **Medical & Healthcare**: Medical documents, research papers, clinical guidelines
+- **Legal & Compliance**: Legal documents, contracts, regulations, case law
+- **Financial & Analytics**: Financial reports, analysis, market research
+- **Technical Documentation**: Technical docs, APIs, code, architecture
+- **Academic Research**: Research papers, academic publications, studies
+
+### 🚀 Advanced AI Capabilities
+- **Query Improvement**: Automatic query enhancement with abbreviation expansion
+- **Dual-LLM Verification**: Two-stage answer verification for quality assurance
+- **Conversation Memory**: Context-aware responses with conversation history
+- **Multimodal Processing**: Support for images, tables, and equations
+- **Domain-Specific Prompts**: Optimized prompts for each domain
+
+### 🔧 Technical Features
+- **Gemini API Integration**: Free-tier Gemini 1.5 Flash model
+- **Async Processing**: Background document processing
+- **RESTful API**: Clean, well-documented endpoints
+- **CORS Support**: Cross-origin resource sharing enabled
+- **Error Handling**: Comprehensive error handling and logging
+
+## Installation
+
+### Prerequisites
+- Python 3.9+
+- Gemini API Key ([Get one here](https://makersuite.google.com/app/apikey))
+
+### Setup
+
+1. **Clone the repository**
+```bash
+cd /mnt/data/Agentic_RAG/backend
+```
+
+2. **Install dependencies**
+```bash
+pip install -r requirements.txt
+```
+
+3. **Set up environment variables**
+```bash
+export GEMINI_API_KEY="your-api-key-here"
+```
+
+Or create a `.env` file:
+```env
+GEMINI_API_KEY=your-api-key-here
+```
+
+4. **Run the server**
+```bash
+python main.py
+```
+
+Or using uvicorn directly:
+```bash
+uvicorn main:app --host 0.0.0.0 --port 8000 --reload
+```
+
+## API Endpoints
+
+### Health Check
+```bash
+GET /health
+```
+
+Response:
+```json
+{
+  "status": "healthy",
+  "timestamp": "2025-01-04T10:00:00",
+  "version": "1.0.0",
+  "features": {
+    "query_improvement": true,
+    "dual_llm_verification": true,
+    "conversation_memory": true,
+    "multi_domain": true,
+    "multimodal_processing": true,
+    "gemini_integration": true
+  },
+  "domains": ["medical", "legal", "financial", "technical", "academic"]
+}
+```
+
+### List Domains
+```bash
+GET /domains
+```
+
+### Upload Document
+```bash
+POST /upload
+Content-Type: multipart/form-data
+
+file: <document file>
+domain: medical
+```
+
+Response:
+```json
+{
+  "success": true,
+  "message": "Document uploaded and queued for processing",
+  "file_name": "research_paper.pdf",
+  "domain": "medical",
+  "processing_id": "uuid-here"
+}
+```
+
+### Query Documents
+```bash
+POST /query
+Content-Type: application/json
+
+{
+  "query": "What are the treatment options for hypertension?",
+  "domain": "medical",
+  "mode": "mix",
+  "conversation_id": "conv_123",
+  "return_metadata": true
+}
+```
+
+Response:
+```json
+{
+  "answer": "Hypertension treatment includes lifestyle modifications...",
+  "sources": ["medical_guidelines.pdf"],
+  "confidence_score": 0.92,
+  "query_improved": true,
+  "verification_performed": true,
+  "conversation_id": "conv_123",
+  "metadata": {
+    "original_query": "What is HTN treatment?",
+    "improved_query": "What are the treatment options for hypertension?",
+    "verification_score": 8.5,
+    "modification_attempts": 1
+  }
+}
+```
+
+### Get Conversation History
+```bash
+GET /conversation/{conversation_id}
+```
+
+### Clear Conversation
+```bash
+DELETE /conversation/{conversation_id}
+```
+
+### Clear Domain Data
+```bash
+DELETE /clear/{domain}
+```
+
+## Usage Examples
+
+### Using cURL
+
+**Upload a document:**
+```bash
+curl -X POST "http://localhost:8000/upload" \
+  -F "file=@medical_paper.pdf" \
+  -F "domain=medical"
+```
+
+**Query documents:**
+```bash
+curl -X POST "http://localhost:8000/query" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "query": "What are the side effects of ACE inhibitors?",
+    "domain": "medical",
+    "mode": "mix",
+    "return_metadata": true
+  }'
+```
+
+### Using Python
+
+```python
+import requests
+
+# Upload document
+with open("medical_paper.pdf", "rb") as f:
+    files = {"file": f}
+    data = {"domain": "medical"}
+    response = requests.post("http://localhost:8000/upload", files=files, data=data)
+    print(response.json())
+
+# Query documents
+query_data = {
+    "query": "What are the treatment options for hypertension?",
+    "domain": "medical",
+    "mode": "mix",
+    "return_metadata": True
+}
+response = requests.post("http://localhost:8000/query", json=query_data)
+print(response.json())
+```
+
+### Using JavaScript/TypeScript
+
+```typescript
+// Upload document
+const formData = new FormData();
+formData.append('file', fileInput.files[0]);
+formData.append('domain', 'medical');
+
+const uploadResponse = await fetch('http://localhost:8000/upload', {
+  method: 'POST',
+  body: formData
+});
+
+// Query documents
+const queryResponse = await fetch('http://localhost:8000/query', {
+  method: 'POST',
+  headers: { 'Content-Type': 'application/json' },
+  body: JSON.stringify({
+    query: 'What are the treatment options for hypertension?',
+    domain: 'medical',
+    mode: 'mix',
+    return_metadata: true
+  })
+});
+
+const result = await queryResponse.json();
+console.log(result);
+```
+
+## Configuration
+
+### Domain-Specific Settings
+
+Each domain has customized settings in `DOMAIN_CONFIGS`:
+
+```python
+{
+    "medical": {
+        "enable_query_improvement": True,
+        "query_improvement_method": "hybrid",
+        "expand_abbreviations": True,
+        "verification_threshold": 7.5,
+        # ... more settings
+    }
+}
+```
+
+### Gemini Model Configuration
+
+Currently using `gemini-1.5-flash` (free tier). To use a different model:
+
+```python
+GEMINI_MODEL = "gemini-1.5-pro"  # More capable, paid tier
+```
+
+## Architecture
+
+```
+backend/
+├── main.py              # FastAPI application
+├── requirements.txt     # Python dependencies
+└── README.md           # This file
+
+storage/                 # Created at runtime
+├── medical/            # Medical domain storage
+├── legal/              # Legal domain storage
+├── financial/          # Financial domain storage
+├── technical/          # Technical domain storage
+└── academic/           # Academic domain storage
+
+uploads/                # Uploaded files
+├── medical/
+├── legal/
+└── ...
+```
+
+## API Documentation
+
+Interactive API documentation is available at:
+- **Swagger UI**: http://localhost:8000/docs
+- **ReDoc**: http://localhost:8000/redoc
+
+## Error Handling
+
+The API uses standard HTTP status codes:
+
+- `200`: Success
+- `400`: Bad Request (invalid parameters)
+- `404`: Not Found
+- `500`: Internal Server Error
+
+All errors return JSON:
+```json
+{
+  "detail": "Error message here"
+}
+```
+
+## Logging
+
+Logs are output to console with the format:
+```
+2025-01-04 10:00:00 - main - INFO - Message here
+```
+
+## Production Deployment
+
+For production deployment:
+
+1. **Set proper CORS origins** in `main.py`:
+```python
+allow_origins=["https://your-frontend-domain.com"]
+```
+
+2. **Use a production ASGI server**:
+```bash
+gunicorn main:app -w 4 -k uvicorn.workers.UvicornWorker
+```
+
+3. **Set up environment variables** securely (don't commit `.env` files)
+
+4. **Enable HTTPS** using a reverse proxy (nginx, Caddy, etc.)
+
+5. **Set up proper logging** (file-based, log rotation)
+
+6. **Monitor** with tools like Prometheus, Grafana
+
+## Troubleshooting
+
+### "GEMINI_API_KEY not set"
+Set your API key as an environment variable or in a `.env` file.
+
+### "Failed to initialize RAG system"
+Check that the storage directories are writable and all dependencies are installed.
+
+### "File type not supported"
+Verify the file extension is in the allowed list for the target domain.
+
+## License
+
+[Your License Here]
+
+## Support
+
+For issues and questions, please open an issue on GitHub.

backend/__init__.py

@@ -0,0 +1,7 @@
+"""
+Backend package for RAG-Anything API
+
+This package contains the FastAPI backend and supporting modules.
+"""
+
+__version__ = "1.1.0"

backend/main.py

@@ -0,0 +1,2078 @@
+"""
+FastAPI Backend for Enhanced RAG-Anything System (v1.1 - Updated)
+
+Production-ready backend with:
+- Multi-domain support (medical, legal, financial, technical, academic)
+- Gemini API integration (LLM, Vision, Embeddings)
+- Query improvement and dual-LLM verification
+- Conversation history management
+- Document processing and querying
+"""
+
+import os
+import asyncio
+import logging
+from pathlib import Path
+from typing import Dict, List, Optional, Any
+from datetime import datetime
+import uuid
+import hashlib
+import time
+import json
+from contextlib import asynccontextmanager
+from dotenv import load_dotenv
+from cachetools import TTLCache
+
+from fastapi import FastAPI, HTTPException, UploadFile, File, Form, BackgroundTasks
+from fastapi.responses import StreamingResponse
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel, Field
+import google.generativeai as genai
+
+# Add project root to path for imports
+import sys
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+# Load environment variables from .env file
+load_dotenv(Path(__file__).parent / ".env")
+
+from raganything.raganything import RAGAnything, RAGAnythingConfig, create_rag_anything
+from backend.reranker import GeminiReranker
+from backend.web_search import WebSearcher, create_web_searcher
+from backend.url_fetcher import URLFetcher, create_url_fetcher
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+# =============================================================================
+# Domain Configurations
+# =============================================================================
+
+DOMAIN_CONFIGS = {
+    "medical": {
+        "name": "Medical & Healthcare",
+        "description": "Optimized for medical documents, research papers, clinical guidelines",
+        "system_prompt": (
+            "You are a medical AI assistant with expertise in healthcare, clinical medicine, "
+            "and medical research. Provide accurate, evidence-based responses with appropriate "
+            "medical terminology. Always cite sources and indicate confidence levels."
+        ),
+        "analysis_prompt": (
+            "Analyze this medical document focusing on: diagnoses, treatments, medications, "
+            "clinical findings, patient outcomes, and evidence-based recommendations."
+        ),
+        "file_extensions": [".pdf", ".doc", ".docx", ".txt", ".md", ".csv", ".xlsx"],
+        "config_overrides": {
+            "domain": "medical",
+            "enable_query_improvement": True,
+            "query_improvement_method": "hybrid",
+            "expand_abbreviations": True,
+            "add_domain_keywords": True,
+            "extract_query_entities": True,
+            "enable_dual_llm_verification": True,
+            "enable_answer_verification": True,
+            "enable_answer_modification": True,
+            "verification_threshold": 7.5,
+            "check_factual_consistency": True,
+            "check_completeness": True,
+            "check_relevance": True,
+        }
+    },
+    "legal": {
+        "name": "Legal & Compliance",
+        "description": "Specialized for legal documents, contracts, regulations, case law",
+        "system_prompt": (
+            "You are a legal AI assistant with expertise in law, regulations, and compliance. "
+            "Provide precise legal analysis with proper citations. Note that this is for "
+            "informational purposes only and not legal advice."
+        ),
+        "analysis_prompt": (
+            "Analyze this legal document focusing on: key provisions, obligations, rights, "
+            "legal precedents, regulatory requirements, and potential implications."
+        ),
+        "file_extensions": [".pdf", ".doc", ".docx", ".txt", ".csv", ".xlsx"],
+        "config_overrides": {
+            "domain": "legal",
+            "enable_query_improvement": True,
+            "query_improvement_method": "llm",
+            "expand_abbreviations": True,
+            "extract_query_entities": True,
+            "enable_dual_llm_verification": True,
+            "enable_answer_verification": True,
+            "enable_answer_modification": True,
+            "verification_threshold": 8.0,
+            "check_factual_consistency": True,
+            "check_completeness": True,
+        }
+    },
+    "financial": {
+        "name": "Financial & Analytics",
+        "description": "Tailored for financial reports, analysis, market research, forecasts",
+        "system_prompt": (
+            "You are a financial AI assistant with expertise in finance, accounting, and "
+            "market analysis. Provide data-driven insights with numerical precision. "
+            "Include relevant financial metrics and trends."
+        ),
+        "analysis_prompt": (
+            "Analyze this financial document focusing on: financial metrics, trends, "
+            "performance indicators, risk factors, market conditions, and forecasts."
+        ),
+        "file_extensions": [".pdf", ".xlsx", ".csv", ".doc", ".docx"],
+        "config_overrides": {
+            "domain": "financial",
+            "enable_query_improvement": True,
+            "query_improvement_method": "hybrid",
+            "expand_abbreviations": True,
+            "add_domain_keywords": True,
+            "enable_dual_llm_verification": True,
+            "enable_answer_verification": True,
+            "verification_threshold": 7.5,
+            "check_factual_consistency": True,
+        }
+    },
+    "technical": {
+        "name": "Technical Documentation",
+        "description": "Optimized for technical docs, APIs, code, system architecture",
+        "system_prompt": (
+            "You are a technical AI assistant with expertise in software development, "
+            "system architecture, and technical documentation. Provide clear, precise "
+            "technical explanations with code examples when relevant."
+        ),
+        "analysis_prompt": (
+            "Analyze this technical document focusing on: system design, APIs, configurations, "
+            "dependencies, implementation details, and best practices."
+        ),
+        "file_extensions": [".pdf", ".md", ".txt", ".rst", ".doc", ".docx", ".csv", ".xlsx"],
+        "config_overrides": {
+            "domain": "technical",
+            "enable_query_improvement": True,
+            "query_improvement_method": "hybrid",
+            "expand_abbreviations": True,
+            "extract_query_entities": True,
+            "enable_dual_llm_verification": True,
+            "enable_answer_verification": True,
+            "verification_threshold": 7.0,
+        }
+    },
+    "academic": {
+        "name": "Academic Research",
+        "description": "Designed for research papers, academic publications, studies",
+        "system_prompt": (
+            "You are an academic AI assistant with expertise in research methodology, "
+            "scholarly analysis, and scientific literature. Provide well-reasoned responses "
+            "with proper academic citations and methodology discussion."
+        ),
+        "analysis_prompt": (
+            "Analyze this academic document focusing on: research questions, methodology, "
+            "findings, conclusions, citations, and contributions to the field."
+        ),
+        "file_extensions": [".pdf", ".doc", ".docx", ".txt", ".tex", ".csv", ".xlsx"],
+        "config_overrides": {
+            "domain": "academic",
+            "enable_query_improvement": True,
+            "query_improvement_method": "llm",
+            "expand_abbreviations": True,
+            "add_domain_keywords": True,
+            "extract_query_entities": True,
+            "enable_dual_llm_verification": True,
+            "enable_answer_verification": True,
+            "enable_answer_modification": True,
+            "verification_threshold": 8.0,
+            "check_completeness": True,
+            "check_relevance": True,
+        }
+    }
+}
+
+# =============================================================================
+# Global State & Configuration
+# =============================================================================
+
+# RAG instances per domain
+rag_instances: Dict[str, RAGAnything] = {}
+
+# Web searcher instance
+web_searcher: Optional[WebSearcher] = None
+
+# URL fetcher instance
+url_fetcher: Optional[URLFetcher] = None
+
+# Conversation history storage
+conversation_histories: Dict[str, List[Dict[str, str]]] = {}
+
+# Processing status tracker
+processing_status: Dict[str, Dict[str, Any]] = {}
+
+# Query result cache (TTL: 5 minutes, max 100 entries)
+query_cache: TTLCache = TTLCache(maxsize=100, ttl=300)
+
+# Performance metrics storage
+performance_metrics: Dict[str, List[float]] = {
+    "query_times": [],
+    "processing_times": [],
+}
+
+# Base paths
+BASE_DIR = Path(__file__).parent.parent
+STORAGE_DIR = BASE_DIR / "storage"
+UPLOAD_DIR = BASE_DIR / "uploads"
+STATUS_FILE = STORAGE_DIR / "processing_status.json"
+
+# --- IMPROVEMENT: Centralized and configurable Gemini model names ---
+GEMINI_API_KEY = os.getenv("GEMINI_API_KEY", "")
+GEMINI_TEXT_MODEL = os.getenv("GEMINI_TEXT_MODEL", "models/gemini-flash-latest")  # Fast generation (alias to latest Flash)
+GEMINI_VERIFIER_MODEL = os.getenv("GEMINI_VERIFIER_MODEL", "models/gemini-pro-latest")  # Quality verification (alias to latest Pro)
+GEMINI_VISION_MODEL = os.getenv("GEMINI_VISION_MODEL", "models/gemini-flash-latest")  # Vision model
+GEMINI_EMBEDDING_MODEL = os.getenv("GEMINI_EMBEDDING_MODEL", "models/text-embedding-004")  # Embedding model
+TAVILY_API_KEY = os.getenv("TAVILY_API_KEY", "")  # For web search
+
+
+# =============================================================================
+# Status Persistence Functions
+# =============================================================================
+
+def load_processing_status() -> Dict[str, Dict[str, Any]]:
+    """Load processing status from disk."""
+    try:
+        if STATUS_FILE.exists():
+            with open(STATUS_FILE, 'r') as f:
+                status_data = json.load(f)
+                logger.info(f"Loaded {len(status_data)} processing status entries from disk")
+                return status_data
+        return {}
+    except Exception as e:
+        logger.error(f"Error loading processing status: {e}", exc_info=True)
+        return {}
+
+
+def save_processing_status():
+    """Save processing status to disk."""
+    try:
+        STATUS_FILE.parent.mkdir(parents=True, exist_ok=True)
+        with open(STATUS_FILE, 'w') as f:
+            json.dump(processing_status, f, indent=2)
+        logger.debug(f"Saved {len(processing_status)} processing status entries to disk")
+    except Exception as e:
+        logger.error(f"Error saving processing status: {e}", exc_info=True)
+
+
+def update_processing_status(processing_id: str, status_update: Dict[str, Any]):
+    """Update processing status both in-memory and on disk."""
+    processing_status[processing_id] = status_update
+    save_processing_status()
+
+
+# =============================================================================
+# Lifespan Management (Startup/Shutdown)
+# =============================================================================
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """Handles application startup and shutdown events."""
+    # --- STARTUP ---
+    logger.info("Starting Enhanced RAG-Anything API...")
+    STORAGE_DIR.mkdir(parents=True, exist_ok=True)
+    UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
+    for domain in DOMAIN_CONFIGS.keys():
+        (STORAGE_DIR / domain).mkdir(parents=True, exist_ok=True)
+    logger.info(f"Created storage directories: {STORAGE_DIR}")
+
+    # Load processing status from disk
+    global processing_status
+    processing_status.update(load_processing_status())
+
+    if GEMINI_API_KEY:
+        try:
+            genai.configure(api_key=GEMINI_API_KEY)
+            logger.info("Gemini API initialized successfully")
+            logger.info(f"Model Configuration:")
+            logger.info(f"  TEXT_MODEL: {GEMINI_TEXT_MODEL}")
+            logger.info(f"  VERIFIER_MODEL: {GEMINI_VERIFIER_MODEL}")
+            logger.info(f"  VISION_MODEL: {GEMINI_VISION_MODEL}")
+            logger.info(f"  EMBEDDING_MODEL: {GEMINI_EMBEDDING_MODEL}")
+        except Exception as e:
+            logger.error(f"Failed to initialize Gemini API: {e}", exc_info=True)
+            logger.warning("Application will start but Gemini features will not work")
+    else:
+        logger.warning("GEMINI_API_KEY not set. Set it in environment variables.")
+
+    # Initialize web searcher if Tavily API key is available
+    global web_searcher, url_fetcher
+    if TAVILY_API_KEY:
+        try:
+            web_searcher = create_web_searcher(api_key=TAVILY_API_KEY, max_results=5)
+            logger.info("Tavily web search initialized successfully")
+        except Exception as e:
+            logger.warning(f"Failed to initialize Tavily: {e}. Web search will not be available.")
+            web_searcher = None
+    else:
+        logger.info("TAVILY_API_KEY not set. Web search features disabled.")
+
+    # Initialize URL fetcher
+    try:
+        url_download_dir = UPLOAD_DIR / "url_downloads"
+        url_download_dir.mkdir(parents=True, exist_ok=True)
+        url_fetcher = create_url_fetcher(download_dir=str(url_download_dir))
+        logger.info("URL fetcher initialized successfully")
+    except Exception as e:
+        logger.warning(f"Failed to initialize URL fetcher: {e}. URL ingestion will not be available.")
+        url_fetcher = None
+
+    logger.info("Enhanced RAG-Anything API started successfully!")
+    
+    yield  # Application runs here
+
+    # --- SHUTDOWN ---
+    logger.info("Shutting down API...")
+    for domain, rag_instance in rag_instances.items():
+        logger.info(f"Finalizing storages for domain: {domain}")
+        await rag_instance.finalize_storages()
+    logger.info("API shutdown complete.")
+
+# =============================================================================
+# FastAPI App Setup
+# =============================================================================
+
+app = FastAPI(
+    title="Enhanced RAG-Anything API",
+    description="Production-ready RAG system with multi-domain support and advanced features",
+    version="1.1.0",
+    docs_url="/docs",
+    redoc_url="/redoc",
+    lifespan=lifespan  # --- FIX: Using modern lifespan event handler ---
+)
+
+# CORS Configuration
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+# =============================================================================
+# Request/Response Models
+# =============================================================================
+
+class QueryRequest(BaseModel):
+    query: str = Field(..., description="User query text", min_length=1)
+    domain: str = Field("medical", description="Domain context (medical, legal, etc.)")
+    mode: str = Field("mix", description="Query mode (local, global, hybrid, naive, mix, web, hybrid_web)")
+    conversation_id: Optional[str] = Field(None, description="Conversation ID for context")
+    return_metadata: bool = Field(True, description="Include detailed metadata in response")
+    enable_web_search: bool = Field(False, description="Enable web search augmentation")
+    web_search_only: bool = Field(False, description="Use only web search (no RAG)")
+    enable_verification: bool = Field(True, description="Enable dual-LLM verification")
+    # Performance optimization parameters
+    fast_mode: bool = Field(False, description="Use optimized parameters for faster queries (2-3x speedup)")
+    top_k: Optional[int] = Field(None, description="Number of top results to retrieve (default: 40, fast: 20)")
+    enable_cache: bool = Field(True, description="Enable query result caching")
+    enable_query_improvement: bool = Field(True, description="Enable query improvement/expansion")
+    enable_verification_check: bool = Field(True, description="Enable verification step (separate from enable_verification)")
+
+    class Config:
+        json_schema_extra = {
+            "example": {
+                "query": "What are the treatment options for hypertension?",
+                "domain": "medical",
+                "mode": "mix",
+                "conversation_id": "conv_123",
+                "return_metadata": True,
+                "enable_web_search": False,
+                "web_search_only": False,
+                "enable_verification": True
+            }
+        }
+
+
+class QueryResponse(BaseModel):
+    answer: str = Field(..., description="Generated answer")
+    sources: List[str] = Field(default_factory=list, description="Source documents used")
+    confidence_score: float = Field(0.0, description="Confidence score (0-1)")
+    query_improved: bool = Field(False, description="Whether query was improved")
+    verification_performed: bool = Field(False, description="Whether answer was verified")
+    conversation_id: str = Field(..., description="Conversation ID")
+    metadata: Optional[Dict[str, Any]] = Field(None, description="Additional metadata")
+
+    class Config:
+        json_schema_extra = {
+            "example": {
+                "answer": "Hypertension treatment includes lifestyle modifications and medications...",
+                "sources": ["medical_guidelines.pdf", "research_paper.pdf"],
+                "confidence_score": 0.92,
+                "query_improved": True,
+                "verification_performed": True,
+                "conversation_id": "conv_123",
+                "metadata": {
+                    "original_query": "What is HTN treatment?",
+                    "improved_query": "What are the treatment options for hypertension?",
+                    "verification_score": 8.5
+                }
+            }
+        }
+
+
+class UploadResponse(BaseModel):
+    success: bool
+    message: str
+    file_name: str
+    domain: str
+    processing_id: str
+
+
+class BatchUploadResponse(BaseModel):
+    success: bool
+    message: str
+    total_files: int
+    accepted_files: int
+    processing_ids: List[str]
+    domain: str
+
+
+class URLUploadRequest(BaseModel):
+    url: str = Field(..., description="URL to fetch and process")
+    domain: str = Field("medical", description="Domain context")
+    convert_to_markdown: bool = Field(True, description="Convert HTML to markdown")
+
+    class Config:
+        json_schema_extra = {
+            "example": {
+                "url": "https://example.com/medical-article.pdf",
+                "domain": "medical",
+                "convert_to_markdown": True
+            }
+        }
+
+
+class DomainInfo(BaseModel):
+    domain_id: str
+    name: str
+    description: str
+    file_extensions: List[str]
+    features: Dict[str, Any]
+
+
+class HealthResponse(BaseModel):
+    status: str
+    timestamp: str
+    version: str
+    features: Dict[str, bool]
+    domains: List[str]
+
+# =============================================================================
+# Gemini Integration Functions
+# =============================================================================
+
+async def gemini_llm_func(
+    prompt: str,
+    system_prompt: Optional[str] = None,
+    history_messages: Optional[List[Dict[str, str]]] = None,
+    **kwargs,
+) -> str:
+    """
+    Gemini LLM function for text generation (Improved with format validation).
+
+    Enhancements:
+    - Increased token limits for entity extraction tasks
+    - Better temperature control for structured outputs
+    - Response validation and auto-append of completion delimiter
+    """
+    def _sync_call():
+        try:
+            from google.generativeai.types import HarmCategory, HarmBlockThreshold
+
+            safety_settings = [
+                {
+                    "category": HarmCategory.HARM_CATEGORY_HARASSMENT,
+                    "threshold": HarmBlockThreshold.BLOCK_NONE,
+                },
+                {
+                    "category": HarmCategory.HARM_CATEGORY_HATE_SPEECH,
+                    "threshold": HarmBlockThreshold.BLOCK_NONE,
+                },
+                {
+                    "category": HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT,
+                    "threshold": HarmBlockThreshold.BLOCK_NONE,
+                },
+                {
+                    "category": HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT,
+                    "threshold": HarmBlockThreshold.BLOCK_NONE,
+                },
+            ]
+            # --- IMPROVEMENT: Use system_instruction parameter ---
+            logger.info(f"Creating GenerativeModel with model_name: {GEMINI_TEXT_MODEL}")
+            model = genai.GenerativeModel(
+                model_name=GEMINI_TEXT_MODEL,
+                system_instruction=system_prompt,
+                safety_settings=safety_settings
+            )
+            config_params = {}
+
+            # Smart temperature control: lower for extraction tasks
+            is_extraction_task = system_prompt and ("entity" in system_prompt.lower() or "extraction" in system_prompt.lower())
+            if "temperature" in kwargs:
+                config_params["temperature"] = kwargs["temperature"]
+            else:
+                # Use lower temperature for structured extraction tasks
+                config_params["temperature"] = 0.1 if is_extraction_task else 0.3
+
+            # Increase token limit for extraction tasks to avoid truncation
+            if "max_tokens" in kwargs:
+                config_params["max_output_tokens"] = kwargs["max_tokens"]
+            else:
+                # Larger limits for extraction to ensure completion delimiter is included
+                config_params["max_output_tokens"] = 16384 if is_extraction_task else 8192
+
+            generation_config = genai.types.GenerationConfig(**config_params)
+
+            # --- IMPROVEMENT: Build structured history for chat model ---
+            history = []
+            if history_messages:
+                for msg in history_messages[-5:]:
+                    role = "user" if msg.get("role") == "user" else "model"
+                    content = msg.get("content", "")
+                    if content:
+                        history.append({"role": role, "parts": [content]})
+
+            chat = model.start_chat(history=history)
+            response = chat.send_message(prompt, generation_config=generation_config)
+            try:
+                result = response.text
+
+                # Post-processing: Ensure completion delimiter is present for extraction tasks
+                if is_extraction_task and result:
+                    # Check if completion delimiter is missing
+                    if "<|COMPLETE|>" not in result and "<|complete|>" not in result:
+                        logger.warning("Completion delimiter missing from extraction result, appending it")
+                        # Append the delimiter to the end
+                        result = result.strip() + "\n<|COMPLETE|>"
+
+                return result
+            except ValueError as ve:
+                logger.warning(f"Response blocked or empty. Reason: {ve}. Candidates: {response.candidates}")
+                if response.prompt_feedback:
+                    logger.warning(f"Prompt feedback: {response.prompt_feedback}")
+                return ""
+        except Exception as e:
+            logger.error(f"Gemini LLM error: {e}", exc_info=True)
+            raise
+    return await asyncio.to_thread(_sync_call)
+
+
+async def gemini_verifier_llm_func(
+    prompt: str,
+    system_prompt: Optional[str] = None,
+    history_messages: Optional[List[Dict[str, str]]] = None,
+    **kwargs,
+) -> str:
+    """Gemini Pro LLM function for answer verification (more powerful, thorough)."""
+    def _sync_call():
+        try:
+            from google.generativeai.types import HarmCategory, HarmBlockThreshold
+
+            safety_settings = [
+                {
+                    "category": HarmCategory.HARM_CATEGORY_HARASSMENT,
+                    "threshold": HarmBlockThreshold.BLOCK_NONE,
+                },
+                {
+                    "category": HarmCategory.HARM_CATEGORY_HATE_SPEECH,
+                    "threshold": HarmBlockThreshold.BLOCK_NONE,
+                },
+                {
+                    "category": HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT,
+                    "threshold": HarmBlockThreshold.BLOCK_NONE,
+                },
+                {
+                    "category": HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT,
+                    "threshold": HarmBlockThreshold.BLOCK_NONE,
+                },
+            ]
+            # Use Pro model for better verification
+            logger.info(f"Creating Verifier GenerativeModel with model_name: {GEMINI_VERIFIER_MODEL}")
+            model = genai.GenerativeModel(
+                model_name=GEMINI_VERIFIER_MODEL,
+                system_instruction=system_prompt,
+                safety_settings=safety_settings
+            )
+            config_params = {}
+            if "temperature" in kwargs:
+                config_params["temperature"] = kwargs["temperature"]
+            if "max_tokens" in kwargs:
+                config_params["max_output_tokens"] = kwargs["max_tokens"]
+            else:
+                # Default to larger token limit for verification responses
+                config_params["max_output_tokens"] = 8192
+            generation_config = genai.types.GenerationConfig(**config_params)
+
+            # Build history
+            history = []
+            if history_messages:
+                for msg in history_messages[-5:]:
+                    role = "user" if msg.get("role") == "user" else "model"
+                    content = msg.get("content", "")
+                    if content:
+                        history.append({"role": role, "parts": [content]})
+
+            chat = model.start_chat(history=history)
+            response = chat.send_message(prompt, generation_config=generation_config)
+            try:
+                return response.text
+            except ValueError as ve:
+                logger.warning(f"Response blocked or empty. Reason: {ve}. Candidates: {response.candidates}")
+                if response.prompt_feedback:
+                    logger.warning(f"Prompt feedback: {response.prompt_feedback}")
+                return ""
+        except Exception as e:
+            logger.error(f"Gemini Verifier LLM error: {e}", exc_info=True)
+            raise
+    return await asyncio.to_thread(_sync_call)
+
+
+async def gemini_vision_func(
+    prompt: str,
+    system_prompt: Optional[str] = None,
+    image_data: Optional[str] = None,
+    **kwargs,
+) -> str:
+    """Gemini Vision function for image analysis."""
+    def _sync_call():
+        try:
+            from google.generativeai.types import HarmCategory, HarmBlockThreshold
+
+            safety_settings = [
+                {
+                    "category": HarmCategory.HARM_CATEGORY_HARASSMENT,
+                    "threshold": HarmBlockThreshold.BLOCK_NONE,
+                },
+                {
+                    "category": HarmCategory.HARM_CATEGORY_HATE_SPEECH,
+                    "threshold": HarmBlockThreshold.BLOCK_NONE,
+                },
+                {
+                    "category": HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT,
+                    "threshold": HarmBlockThreshold.BLOCK_NONE,
+                },
+                {
+                    "category": HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT,
+                    "threshold": HarmBlockThreshold.BLOCK_NONE,
+                },
+            ]
+            # --- FIX: Use dedicated vision model ---
+            logger.info(f"Creating Vision GenerativeModel with model_name: {GEMINI_VISION_MODEL}")
+            model = genai.GenerativeModel(GEMINI_VISION_MODEL, safety_settings=safety_settings)
+            config_params = {}
+            if "temperature" in kwargs:
+                config_params["temperature"] = kwargs["temperature"]
+            if "max_tokens" in kwargs:
+                config_params["max_output_tokens"] = kwargs["max_tokens"]
+            generation_config = genai.types.GenerationConfig(**config_params)
+
+            content_parts = []
+            if system_prompt:
+                content_parts.append(system_prompt)
+            content_parts.append(prompt)
+
+            if image_data:
+                import base64
+                import io
+                from PIL import Image
+                image_bytes = base64.b64decode(image_data)
+                image = Image.open(io.BytesIO(image_bytes))
+                content_parts.append(image)
+
+            response = model.generate_content(content_parts, generation_config=generation_config)
+            try:
+                return response.text
+            except ValueError as ve:
+                logger.warning(f"Vision response blocked or empty. Reason: {ve}. Candidates: {response.candidates}")
+                if response.prompt_feedback:
+                    logger.warning(f"Vision prompt feedback: {response.prompt_feedback}")
+                return ""
+        except Exception as e:
+            logger.error(f"Gemini Vision error: {e}", exc_info=True)
+            raise
+    return await asyncio.to_thread(_sync_call)
+
+
+async def gemini_embedding_func(texts: List[str]) -> List[List[float]]:
+    """Gemini Embedding function for text vectorization."""
+    def _sync_call():
+        try:
+            # --- IMPROVEMENT: Use newer embedding model ---
+            result = genai.embed_content(
+                model=GEMINI_EMBEDDING_MODEL,
+                content=texts,
+                task_type="retrieval_document"
+            )
+            return result['embedding']
+        except Exception as e:
+            logger.error(f"Gemini Embedding error: {e}", exc_info=True)
+            raise
+    return await asyncio.to_thread(_sync_call)
+
+gemini_embedding_func.embedding_dim = 768
+
+
+async def synthesize_web_results_with_gemini(
+    query: str,
+    web_context: str,
+    rag_context: Optional[str] = None
+) -> str:
+    """
+    Use Gemini to synthesize web search results into a coherent, direct answer
+
+    Args:
+        query: User's original query
+        web_context: Formatted web search results
+        rag_context: Optional RAG results to incorporate
+
+    Returns:
+        Synthesized answer from Gemini
+    """
+    try:
+        logger.info("Synthesizing web results with Gemini")
+
+        # Build synthesis prompt
+        if rag_context:
+            system_prompt = """You are an expert research assistant. Your task is to synthesize information from both
+a knowledge base and recent web search results to provide a comprehensive, accurate answer.
+
+Guidelines:
+- Provide a direct, clear answer to the user's question
+- Combine insights from both the knowledge base and web sources
+- Cite sources when making specific claims (use [Source N] notation)
+- If there are contradictions, acknowledge them and explain
+- Be concise but thorough
+- Use a professional, informative tone"""
+
+            prompt = f"""User Question: {query}
+
+Knowledge Base Information:
+{rag_context}
+
+Web Search Results:
+{web_context}
+
+Based on the above information, provide a comprehensive answer to the user's question. Synthesize the information from both sources and cite your sources appropriately."""
+
+        else:
+            system_prompt = """You are an expert research assistant. Your task is to synthesize web search results
+into a clear, direct answer to the user's question.
+
+Guidelines:
+- Provide a direct, clear answer to the user's question
+- Cite sources when making specific claims (use [Source N] notation)
+- Be concise but comprehensive
+- If information is limited or unclear, acknowledge it
+- Use a professional, informative tone
+- Include relevant details like dates, statistics, or examples when available"""
+
+            prompt = f"""User Question: {query}
+
+Web Search Results:
+{web_context}
+
+Based on the web search results above, provide a clear and comprehensive answer to the user's question. Cite your sources appropriately."""
+
+        # Call Gemini to synthesize the answer
+        answer = await gemini_llm_func(
+            prompt=prompt,
+            system_prompt=system_prompt,
+            temperature=0.3,  # Lower temperature for more focused answers
+            max_tokens=1500
+        )
+
+        if not answer or len(answer.strip()) < 10:
+            logger.warning("Gemini synthesis produced minimal output, using fallback")
+            return web_context
+
+        return answer
+
+    except Exception as e:
+        logger.error(f"Error synthesizing web results with Gemini: {e}", exc_info=True)
+        # Fallback to raw web context
+        return web_context
+
+
+async def gemini_rerank_func(query: str, documents: List[str], top_n: Optional[int] = None) -> List[Dict[str, Any]]:
+    """
+    Gemini-based reranking function for LightRAG
+
+    This follows LightRAG's reranking API signature which expects:
+    - documents: List of strings (not dict chunks)
+    - top_n: Number of top results (not top_k)
+    - Returns: List of {"index": int, "relevance_score": float}
+
+    Args:
+        query: Search query
+        documents: List of document strings to rerank
+        top_n: Number of top documents to return (None = return all, reranked)
+
+    Returns:
+        List of {"index": int, "relevance_score": float} in descending score order
+    """
+    try:
+        # Convert documents (strings) to chunks format for our reranker
+        chunks = [{"content": doc} for doc in documents]
+
+        # Initialize reranker with Gemini LLM function
+        reranker = GeminiReranker(
+            llm_func=gemini_llm_func,
+            batch_size=3,  # Process 3 chunks at a time to avoid rate limits
+            temperature=0.1
+        )
+
+        # Perform reranking
+        reranked_chunks = await reranker.rerank(query, chunks, top_n)
+
+        # Convert back to LightRAG format: List[{"index": int, "relevance_score": float}]
+        results = []
+        for i, chunk in enumerate(reranked_chunks):
+            # Find original index of this chunk
+            original_content = chunk.get("content", "")
+            try:
+                original_index = documents.index(original_content)
+            except ValueError:
+                # Fallback: use current index if not found
+                original_index = i
+
+            results.append({
+                "index": original_index,
+                "relevance_score": chunk.get("relevance_score", 0.0)
+            })
+
+        logger.debug(f"Reranked {len(documents)} documents, returning {len(results)} results")
+        return results
+
+    except Exception as e:
+        logger.error(f"Reranking error: {e}", exc_info=True)
+        # Return original order on error - format: List[{"index": int, "relevance_score": float}]
+        result_count = top_n if top_n and top_n < len(documents) else len(documents)
+        return [{"index": i, "relevance_score": 1.0} for i in range(result_count)]
+
+
+# =============================================================================
+# RAG Instance Management
+# =============================================================================
+
+async def get_rag_instance(domain: str) -> RAGAnything:
+    """Get or create RAG instance for a specific domain."""
+    if domain not in DOMAIN_CONFIGS:
+        raise HTTPException(
+            status_code=400,
+            detail=f"Invalid domain '{domain}'. Valid domains: {list(DOMAIN_CONFIGS.keys())}"
+        )
+    if domain in rag_instances:
+        logger.debug(f"Using cached RAG instance for domain: {domain}")
+        return rag_instances[domain]
+    
+    logger.info(f"Creating new RAG instance for domain: {domain}")
+    try:
+        domain_config = DOMAIN_CONFIGS[domain]
+        domain_storage = STORAGE_DIR / domain
+        domain_storage.mkdir(parents=True, exist_ok=True)
+
+        config = RAGAnythingConfig(
+            working_dir=str(domain_storage),
+            parser="mineru",
+            parse_method="auto",
+            enable_image_processing=True,
+            enable_table_processing=True,
+            enable_equation_processing=True,
+            **domain_config["config_overrides"]
+        )
+        rag = await create_rag_anything(
+            llm_model_func=gemini_llm_func,              # Flash for generation
+            vision_model_func=gemini_vision_func,         # Flash for vision
+            embedding_func=gemini_embedding_func,         # Embedding model
+            verifier_llm_func=gemini_verifier_llm_func,  # Pro for verification
+            config=config,
+            rerank_model_func=gemini_rerank_func,        # Enable reranking (passed directly)
+        )
+        rag_instances[domain] = rag
+        logger.info(f"RAG instance created successfully for domain: {domain}")
+        return rag
+    except Exception as e:
+        logger.error(f"Failed to create RAG instance for domain {domain}: {e}", exc_info=True)
+        raise HTTPException(
+            status_code=500,
+            detail=f"Failed to initialize RAG system for domain '{domain}': {str(e)}"
+        )
+
+# =============================================================================
+# API Endpoints
+# =============================================================================
+
+@app.get("/health", response_model=HealthResponse)
+async def health_check():
+    """Health check endpoint."""
+    return HealthResponse(
+        status="healthy",
+        timestamp=datetime.now().isoformat(),
+        version="2.0.0",
+        features={
+            "query_improvement": True,
+            "dual_llm_verification": True,
+            "gemini_pro_verifier": True,
+            "reranking": True,
+            "conversation_memory": True,
+            "multi_domain": True,
+            "multimodal_processing": True,
+            "gemini_integration": bool(GEMINI_API_KEY),
+            "web_search": bool(web_searcher),
+            "url_ingestion": bool(url_fetcher),
+        },
+        domains=list(DOMAIN_CONFIGS.keys())
+    )
+
+
+@app.get("/domains", response_model=List[DomainInfo])
+async def list_domains():
+    """List all available domains."""
+    domains = []
+    for domain_id, config in DOMAIN_CONFIGS.items():
+        domains.append(DomainInfo(
+            domain_id=domain_id,
+            name=config["name"],
+            description=config["description"],
+            file_extensions=config["file_extensions"],
+            features={k: v for k, v in config["config_overrides"].items() if isinstance(v, bool)}
+        ))
+    return domains
+
+
+@app.post("/upload", response_model=UploadResponse)
+async def upload_document(
+    file: UploadFile = File(...),
+    domain: str = Form(...),
+    background_tasks: BackgroundTasks = None
+):
+    """Upload and process a document in the background."""
+    logger.info(f"Upload request: {file.filename} to domain: {domain}")
+    try:
+        if domain not in DOMAIN_CONFIGS:
+            raise HTTPException(400, f"Invalid domain. Valid: {list(DOMAIN_CONFIGS.keys())}")
+
+        file_ext = Path(file.filename).suffix.lower()
+        allowed_extensions = DOMAIN_CONFIGS[domain]["file_extensions"]
+        if file_ext not in allowed_extensions:
+            raise HTTPException(400, f"File type {file_ext} not for '{domain}'. Allowed: {allowed_extensions}")
+
+        processing_id = str(uuid.uuid4())
+        domain_upload_dir = UPLOAD_DIR / domain
+        domain_upload_dir.mkdir(parents=True, exist_ok=True)
+        file_path = domain_upload_dir / f"{processing_id}_{file.filename}"
+
+        with open(file_path, "wb") as f:
+            f.write(await file.read())
+        logger.info(f"File saved: {file_path}")
+
+        # Initialize status and save to disk
+        update_processing_status(processing_id, {
+            "status": "processing",
+            "message": "Processing document...",
+            "file_name": file.filename,
+            "domain": domain,
+            "started_at": datetime.now().isoformat()
+        })
+
+        async def process_document_task():
+            try:
+                logger.info(f"Processing document: {file_path}")
+                rag = await get_rag_instance(domain)
+                result = await rag.process_document_complete(str(file_path))
+
+                # Check result (process_document_complete returns None on success)
+                if result is None or (isinstance(result, dict) and result.get("success") is not False):
+                    logger.info(f"Document processed successfully: {file.filename}")
+                    update_processing_status(processing_id, {
+                        "status": "completed",
+                        "message": "Document processed successfully",
+                        "file_name": file.filename,
+                        "domain": domain,
+                        "completed_at": datetime.now().isoformat()
+                    })
+                else:
+                    error_msg = result.get('error', 'Unknown processing error') if isinstance(result, dict) else "Processing error"
+                    logger.error(f"Document processing failed: {error_msg}")
+                    update_processing_status(processing_id, {
+                        "status": "failed",
+                        "message": f"Processing failed: {error_msg}",
+                        "file_name": file.filename,
+                        "domain": domain,
+                        "error": error_msg
+                    })
+            except Exception as e:
+                logger.error(f"Error in background processing of {file.filename}: {e}", exc_info=True)
+                update_processing_status(processing_id, {
+                    "status": "failed",
+                    "message": f"Error: {str(e)}",
+                    "file_name": file.filename,
+                    "domain": domain,
+                    "error": str(e)
+                })
+
+        background_tasks.add_task(process_document_task)
+
+        return UploadResponse(
+            success=True,
+            message="Document uploaded and queued for processing",
+            file_name=file.filename,
+            domain=domain,
+            processing_id=processing_id
+        )
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Upload error: {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail=f"Upload failed: {str(e)}")
+
+
+@app.post("/upload-batch", response_model=BatchUploadResponse)
+async def upload_documents_batch(
+    files: List[UploadFile] = File(...),
+    domain: str = Form(...),
+    background_tasks: BackgroundTasks = None
+):
+    """
+    Upload and process multiple documents in batch using optimized processing.
+
+    Uses BatchOptimizer for 2-3x faster processing through:
+    - Parallel parsing (up to 4 documents simultaneously)
+    - Parallel processing (up to 10 documents simultaneously)
+    - Pipeline architecture (parse + process in parallel)
+    """
+    logger.info(f"Batch upload request: {len(files)} files to domain: {domain}")
+    try:
+        if domain not in DOMAIN_CONFIGS:
+            raise HTTPException(400, f"Invalid domain. Valid: {list(DOMAIN_CONFIGS.keys())}")
+
+        allowed_extensions = DOMAIN_CONFIGS[domain]["file_extensions"]
+        domain_upload_dir = UPLOAD_DIR / domain
+        domain_upload_dir.mkdir(parents=True, exist_ok=True)
+
+        # Process and save all files
+        file_paths = []
+        processing_ids = []
+        rejected_files = []
+
+        for file in files:
+            file_ext = Path(file.filename).suffix.lower()
+            if file_ext not in allowed_extensions:
+                rejected_files.append(file.filename)
+                logger.warning(f"Rejected file {file.filename}: extension {file_ext} not allowed")
+                continue
+
+            processing_id = str(uuid.uuid4())
+            file_path = domain_upload_dir / f"{processing_id}_{file.filename}"
+
+            with open(file_path, "wb") as f:
+                f.write(await file.read())
+
+            file_paths.append(str(file_path))
+            processing_ids.append(processing_id)
+
+            # Initialize status for each file
+            update_processing_status(processing_id, {
+                "status": "queued",
+                "message": "Queued for batch processing...",
+                "file_name": file.filename,
+                "domain": domain,
+                "started_at": datetime.now().isoformat()
+            })
+
+        logger.info(f"Accepted {len(file_paths)}/{len(files)} files, rejected: {rejected_files}")
+
+        if not file_paths:
+            raise HTTPException(400, f"No valid files provided. Allowed extensions: {allowed_extensions}")
+
+        # Process documents in batch using optimized processing
+        async def process_batch_task():
+            start_time = time.time()
+            try:
+                logger.info(f"Starting optimized batch processing of {len(file_paths)} files")
+                rag = await get_rag_instance(domain)
+
+                # Use optimized batch processing if available
+                if hasattr(rag, 'process_documents_batch_optimized'):
+                    result = await rag.process_documents_batch_optimized(
+                        file_paths=file_paths,
+                        max_concurrent_parsers=4,  # MinerU optimal
+                        max_concurrent_processors=10,  # Higher for I/O-bound tasks
+                        enable_progress_tracking=True,
+                    )
+
+                    # Update statuses based on results
+                    successful_files = result.get('successful_files', [])
+                    failed_files = result.get('failed_files', {})
+
+                    for idx, file_path in enumerate(file_paths):
+                        processing_id = processing_ids[idx]
+                        filename = Path(file_path).name.split('_', 1)[1] if '_' in Path(file_path).name else Path(file_path).name
+
+                        if file_path in successful_files:
+                            update_processing_status(processing_id, {
+                                "status": "completed",
+                                "message": "Document processed successfully",
+                                "file_name": filename,
+                                "domain": domain,
+                                "completed_at": datetime.now().isoformat()
+                            })
+                        elif file_path in failed_files:
+                            error_msg = failed_files[file_path]
+                            update_processing_status(processing_id, {
+                                "status": "failed",
+                                "message": f"Processing failed: {error_msg}",
+                                "file_name": filename,
+                                "domain": domain,
+                                "error": error_msg
+                            })
+
+                    total_time = time.time() - start_time
+                    throughput = len(successful_files) / total_time if total_time > 0 else 0
+                    logger.info(
+                        f"Batch processing complete: {len(successful_files)}/{len(file_paths)} successful "
+                        f"in {total_time:.2f}s ({throughput:.2f} docs/sec)"
+                    )
+
+                    # Track performance
+                    performance_metrics["processing_times"].append(total_time)
+                    if len(performance_metrics["processing_times"]) > 100:
+                        performance_metrics["processing_times"] = performance_metrics["processing_times"][-100:]
+
+                else:
+                    # Fallback: process sequentially
+                    logger.warning("Optimized batch processing not available, using sequential processing")
+                    for idx, file_path in enumerate(file_paths):
+                        processing_id = processing_ids[idx]
+                        filename = Path(file_path).name.split('_', 1)[1] if '_' in Path(file_path).name else Path(file_path).name
+
+                        current_status = processing_status[processing_id].copy()
+                        current_status["status"] = "processing"
+                        current_status["message"] = "Processing document..."
+                        update_processing_status(processing_id, current_status)
+
+                        try:
+                            result = await rag.process_document_complete(file_path)
+                            if result is None or (isinstance(result, dict) and result.get("success") is not False):
+                                update_processing_status(processing_id, {
+                                    "status": "completed",
+                                    "message": "Document processed successfully",
+                                    "file_name": filename,
+                                    "domain": domain,
+                                    "completed_at": datetime.now().isoformat()
+                                })
+                            else:
+                                error_msg = result.get('error', 'Unknown error') if isinstance(result, dict) else "Processing error"
+                                update_processing_status(processing_id, {
+                                    "status": "failed",
+                                    "message": f"Processing failed: {error_msg}",
+                                    "file_name": filename,
+                                    "domain": domain,
+                                    "error": error_msg
+                                })
+                        except Exception as e:
+                            logger.error(f"Error processing {filename}: {e}", exc_info=True)
+                            update_processing_status(processing_id, {
+                                "status": "failed",
+                                "message": f"Error: {str(e)}",
+                                "file_name": filename,
+                                "domain": domain,
+                                "error": str(e)
+                            })
+
+            except Exception as e:
+                logger.error(f"Batch processing error: {e}", exc_info=True)
+                # Mark all as failed
+                for idx, file_path in enumerate(file_paths):
+                    processing_id = processing_ids[idx]
+                    filename = Path(file_path).name.split('_', 1)[1] if '_' in Path(file_path).name else Path(file_path).name
+                    update_processing_status(processing_id, {
+                        "status": "failed",
+                        "message": f"Batch processing error: {str(e)}",
+                        "file_name": filename,
+                        "domain": domain,
+                        "error": str(e)
+                    })
+
+        background_tasks.add_task(process_batch_task)
+
+        return BatchUploadResponse(
+            success=True,
+            message=f"Batch upload queued: {len(file_paths)} files accepted" + (f", {len(rejected_files)} rejected" if rejected_files else ""),
+            total_files=len(files),
+            accepted_files=len(file_paths),
+            processing_ids=processing_ids,
+            domain=domain
+        )
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Batch upload error: {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail=f"Batch upload failed: {str(e)}")
+
+
+@app.post("/upload-url", response_model=UploadResponse)
+async def upload_url(
+    request: URLUploadRequest,
+    background_tasks: BackgroundTasks
+):
+    """Fetch document from URL and process it."""
+    logger.info(f"URL upload request: {request.url} to domain: {request.domain}")
+    try:
+        if not url_fetcher:
+            raise HTTPException(503, "URL fetcher not available. Check server configuration.")
+
+        if request.domain not in DOMAIN_CONFIGS:
+            raise HTTPException(400, f"Invalid domain. Valid: {list(DOMAIN_CONFIGS.keys())}")
+
+        processing_id = str(uuid.uuid4())
+
+        # Initialize status
+        update_processing_status(processing_id, {
+            "status": "fetching",
+            "message": "Fetching URL content...",
+            "url": request.url,
+            "domain": request.domain,
+            "started_at": datetime.now().isoformat()
+        })
+
+        async def fetch_and_process_url():
+            try:
+                logger.info(f"[URL UPLOAD] Starting fetch for: {request.url}")
+
+                # Fetch URL content with timeout
+                fetch_result = await asyncio.wait_for(
+                    url_fetcher.fetch_url(
+                        url=request.url,
+                        convert_to_markdown=request.convert_to_markdown
+                    ),
+                    timeout=60.0  # 60 second timeout for URL fetching
+                )
+
+                if not fetch_result.get("success"):
+                    error_msg = fetch_result.get("error", "Unknown fetch error")
+                    logger.error(f"[URL UPLOAD] Fetch failed: {error_msg}")
+                    update_processing_status(processing_id, {
+                        "status": "failed",
+                        "message": f"Failed to fetch URL: {error_msg}",
+                        "domain": request.domain,
+                        "error": error_msg
+                    })
+                    return
+
+                file_path = fetch_result.get("file_path")
+                if not file_path:
+                    logger.error("[URL UPLOAD] No file path returned from URL fetch")
+                    update_processing_status(processing_id, {
+                        "status": "failed",
+                        "message": "No file path returned from URL fetch",
+                        "domain": request.domain,
+                        "error": "No file path"
+                    })
+                    return
+
+                logger.info(f"[URL UPLOAD] Content saved to: {file_path}")
+
+                # Update status
+                update_processing_status(processing_id, {
+                    "status": "processing",
+                    "message": "Processing document...",
+                    "domain": request.domain,
+                    "file_path": file_path
+                })
+
+                # Get RAG instance
+                rag = await get_rag_instance(request.domain)
+
+                # Check if we have a content list with images (advanced HTML parsing)
+                content_list = fetch_result.get("content_list")
+                images_count = fetch_result.get("images_count", 0)
+
+                if content_list and len(content_list) > 0 and images_count > 0:
+                    # Advanced pathway: Process pre-parsed content list with images
+                    logger.info(f"[URL UPLOAD] Using advanced processing: {len(content_list)} blocks, {images_count} images")
+                    result = await asyncio.wait_for(
+                        rag.process_content_list_direct(
+                            content_list=content_list,
+                            source_identifier=request.url,
+                            enable_image_processing=True
+                        ),
+                        timeout=300.0  # 5 minute timeout for processing
+                    )
+                else:
+                    # Standard pathway: Process as regular document (PDF or text-only HTML)
+                    logger.info("[URL UPLOAD] Using standard document processing")
+                    result = await asyncio.wait_for(
+                        rag.process_document_complete(file_path),
+                        timeout=300.0  # 5 minute timeout for processing
+                    )
+
+                # Check result and update status
+                # Note: process_document_complete returns None on success (not a dict)
+                if result is None or (isinstance(result, dict) and result.get("success") is not False):
+                    logger.info(f"[URL UPLOAD] ✓ Successfully processed: {request.url}")
+                    update_processing_status(processing_id, {
+                        "status": "completed",
+                        "message": "Document processed successfully",
+                        "domain": request.domain,
+                        "file_path": file_path,
+                        "completed_at": datetime.now().isoformat()
+                    })
+                else:
+                    error_msg = result.get('error', 'Unknown processing error') if isinstance(result, dict) else "Processing returned error"
+                    logger.error(f"[URL UPLOAD] ✗ Processing failed: {error_msg}")
+                    update_processing_status(processing_id, {
+                        "status": "failed",
+                        "message": f"Processing failed: {error_msg}",
+                        "domain": request.domain,
+                        "error": error_msg
+                    })
+
+            except asyncio.TimeoutError:
+                logger.error(f"[URL UPLOAD] ✗ Timeout processing {request.url}")
+                update_processing_status(processing_id, {
+                    "status": "failed",
+                    "message": "Processing timeout",
+                    "domain": request.domain,
+                    "error": "Timeout"
+                })
+            except Exception as e:
+                logger.error(f"[URL UPLOAD] ✗ Error processing {request.url}: {e}", exc_info=True)
+                update_processing_status(processing_id, {
+                    "status": "failed",
+                    "message": f"Error: {str(e)}",
+                    "domain": request.domain,
+                    "error": str(e)
+                })
+
+        background_tasks.add_task(fetch_and_process_url)
+
+        return UploadResponse(
+            success=True,
+            message="URL queued for fetching and processing",
+            file_name=request.url,
+            domain=request.domain,
+            processing_id=processing_id
+        )
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"URL upload error: {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail=f"URL upload failed: {str(e)}")
+
+
+@app.post("/query/stream")
+async def query_documents_stream(request: QueryRequest):
+    """
+    Stream query responses with real-time token generation and verification.
+
+    This endpoint provides Server-Sent Events (SSE) streaming for real-time
+    response generation while maintaining dual-LLM verification.
+    Supports web search augmentation when enabled.
+    """
+    logger.info(f"Streaming query request: '{request.query[:50]}...' in domain: {request.domain}, web_search: {request.enable_web_search}, web_only: {request.web_search_only}")
+
+    async def generate_sse():
+        """Generate Server-Sent Events stream"""
+        import json
+
+        try:
+            conversation_id = request.conversation_id or f"conv_{uuid.uuid4()}"
+
+            # Handle web search only mode
+            if request.web_search_only:
+                if not web_searcher:
+                    error_data = {"type": "error", "content": {"message": "Web search not available. Set TAVILY_API_KEY."}, "done": True}
+                    yield f"event: error\ndata: {json.dumps(error_data)}\n\n"
+                    return
+
+                logger.info("Using web search only mode (streaming)")
+                try:
+                    web_results = await web_searcher.search(request.query, max_results=5)
+                    web_context = web_searcher.format_results_for_llm(web_results)
+
+                    # Synthesize answer using Gemini (streaming simulation)
+                    logger.info("Synthesizing web search results with Gemini (streaming)")
+                    answer = await synthesize_web_results_with_gemini(
+                        query=request.query,
+                        web_context=web_context,
+                        rag_context=None
+                    )
+
+                    # Stream the answer word by word
+                    words = answer.split()
+                    for i, word in enumerate(words):
+                        token = word + " " if i < len(words) - 1 else word
+                        data = {"type": "token", "content": token, "done": False}
+                        yield f"event: token\ndata: {json.dumps(data)}\n\n"
+                        await asyncio.sleep(0.01)  # Small delay for streaming effect
+
+                    # Send completion event
+                    yield f"event: done\ndata: {json.dumps({'message': 'Stream complete', 'conversation_id': conversation_id})}\n\n"
+                    return
+
+                except Exception as e:
+                    logger.error(f"Web search only error: {e}", exc_info=True)
+                    error_data = {"type": "error", "content": {"message": f"Web search failed: {str(e)}"}, "done": True}
+                    yield f"event: error\ndata: {json.dumps(error_data)}\n\n"
+                    return
+
+            # Get RAG instance
+            rag = await get_rag_instance(request.domain)
+
+            # Determine optimal parameters based on fast_mode
+            if request.fast_mode:
+                # Optimized parameters for 2-3x speedup
+                top_k = request.top_k if request.top_k is not None else 20
+                chunk_top_k = 10
+                max_entity_tokens = 4000
+                max_relation_tokens = 6000
+                max_total_tokens = 20000
+                logger.info(f"⚡ Fast mode enabled for streaming: top_k={top_k}, chunk_top_k={chunk_top_k}")
+            else:
+                # Default parameters (higher quality, slower)
+                top_k = request.top_k if request.top_k is not None else 40
+                chunk_top_k = 20
+                max_entity_tokens = 6000
+                max_relation_tokens = 8000
+                max_total_tokens = 30000
+
+            # Log toggle settings
+            logger.info(f"Query settings - improvement: {request.enable_query_improvement}, verification: {request.enable_verification_check}, web_search: {request.enable_web_search}")
+
+            # If web search augmentation is enabled, we need to collect the RAG answer first
+            # then augment with web search
+            if request.enable_web_search and web_searcher:
+                logger.info("Web search augmentation enabled for streaming")
+
+                # Collect RAG answer first
+                rag_answer_buffer = []
+                async for chunk in rag.aquery_stream(
+                    query=request.query,
+                    mode=request.mode,
+                    enable_verification=request.enable_verification_check,
+                    enable_query_improvement=request.enable_query_improvement,
+                    top_k=top_k,
+                    chunk_top_k=chunk_top_k,
+                    max_entity_tokens=max_entity_tokens,
+                    max_relation_tokens=max_relation_tokens,
+                    max_total_tokens=max_total_tokens
+                ):
+                    chunk_type = chunk.get("type", "token")
+                    content = chunk.get("content", "")
+                    done = chunk.get("done", False)
+
+                    if chunk_type == "token":
+                        # Stream token and collect it
+                        rag_answer_buffer.append(content)
+                        data = {"type": "token", "content": content, "done": False}
+                        yield f"event: token\ndata: {json.dumps(data)}\n\n"
+
+                    elif chunk_type == "verification":
+                        # Send verification metadata
+                        data = {"type": "verification", "content": content, "done": done}
+                        yield f"event: verification\ndata: {json.dumps(data)}\n\n"
+
+                    elif chunk_type == "error":
+                        # Send error
+                        data = {"type": "error", "content": content, "done": True}
+                        yield f"event: error\ndata: {json.dumps(data)}\n\n"
+                        return
+
+                # Now perform web search and synthesis
+                try:
+                    rag_answer = "".join(rag_answer_buffer)
+                    logger.info("Performing web search to augment RAG answer...")
+                    web_results = await web_searcher.search(request.query, max_results=5)
+
+                    if web_results.get("results"):
+                        web_context = web_searcher.format_results_for_llm(web_results)
+
+                        # Synthesize combined answer
+                        logger.info("Synthesizing RAG + web results with Gemini")
+                        synthesized_answer = await synthesize_web_results_with_gemini(
+                            query=request.query,
+                            web_context=web_context,
+                            rag_context=rag_answer
+                        )
+
+                        # Clear the previous RAG answer and stream the synthesized one
+                        # Send a newline separator
+                        data = {"type": "token", "content": "\n\n---\n\n**Enhanced with Web Search:**\n\n", "done": False}
+                        yield f"event: token\ndata: {json.dumps(data)}\n\n"
+
+                        # Stream synthesized answer
+                        words = synthesized_answer.split()
+                        for i, word in enumerate(words):
+                            token = word + " " if i < len(words) - 1 else word
+                            data = {"type": "token", "content": token, "done": False}
+                            yield f"event: token\ndata: {json.dumps(data)}\n\n"
+                            await asyncio.sleep(0.01)
+
+                except Exception as e:
+                    logger.error(f"Web search augmentation error: {e}", exc_info=True)
+                    # Continue without web augmentation
+                    pass
+
+                # Send completion event
+                yield f"event: done\ndata: {json.dumps({'message': 'Stream complete', 'conversation_id': conversation_id})}\n\n"
+
+            else:
+                # Standard RAG streaming without web search
+                async for chunk in rag.aquery_stream(
+                    query=request.query,
+                    mode=request.mode,
+                    enable_verification=request.enable_verification_check,
+                    enable_query_improvement=request.enable_query_improvement,
+                    top_k=top_k,
+                    chunk_top_k=chunk_top_k,
+                    max_entity_tokens=max_entity_tokens,
+                    max_relation_tokens=max_relation_tokens,
+                    max_total_tokens=max_total_tokens
+                ):
+                    chunk_type = chunk.get("type", "token")
+                    content = chunk.get("content", "")
+                    done = chunk.get("done", False)
+
+                    if chunk_type == "token":
+                        # Stream token
+                        data = {"type": "token", "content": content, "done": done}
+                        yield f"event: token\ndata: {json.dumps(data)}\n\n"
+
+                    elif chunk_type == "verification":
+                        # Send verification metadata
+                        data = {"type": "verification", "content": content, "done": done}
+                        yield f"event: verification\ndata: {json.dumps(data)}\n\n"
+
+                    elif chunk_type == "error":
+                        # Send error
+                        data = {"type": "error", "content": content, "done": True}
+                        yield f"event: error\ndata: {json.dumps(data)}\n\n"
+                        break
+
+                # Send completion event
+                yield f"event: done\ndata: {json.dumps({'message': 'Stream complete', 'conversation_id': conversation_id})}\n\n"
+
+        except Exception as e:
+            logger.error(f"Streaming error: {e}", exc_info=True)
+            error_data = {"type": "error", "content": {"message": str(e)}, "done": True}
+            yield f"event: error\ndata: {json.dumps(error_data)}\n\n"
+
+    return StreamingResponse(
+        generate_sse(),
+        media_type="text/event-stream",
+        headers={
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "X-Accel-Buffering": "no"
+        }
+    )
+
+
+@app.post("/query", response_model=QueryResponse)
+async def query_documents(request: QueryRequest):
+    """Query documents with enhanced RAG capabilities and optional web search."""
+    start_time = time.time()
+    logger.info(f"Query request: '{request.query[:50]}...' in domain: {request.domain}, mode: {request.mode}, fast_mode: {request.fast_mode}")
+
+    try:
+        conversation_id = request.conversation_id or f"conv_{uuid.uuid4()}"
+        conversation_history = conversation_histories.get(conversation_id, [])
+
+        # Generate cache key for non-web-search queries
+        cache_key = None
+        if request.enable_cache and not request.web_search_only and not request.enable_web_search:
+            cache_data = f"{request.query}:{request.domain}:{request.mode}:{request.fast_mode}:{request.enable_verification}"
+            cache_key = hashlib.md5(cache_data.encode()).hexdigest()
+
+            # Check cache
+            if cache_key in query_cache:
+                cached_response = query_cache[cache_key]
+                logger.info(f"✓ Cache hit for query (saved {time.time() - start_time:.2f}s)")
+                # Update conversation ID in cached response
+                cached_response.conversation_id = conversation_id
+                return cached_response
+
+        # Handle web search only mode
+        if request.web_search_only:
+            if not web_searcher:
+                raise HTTPException(503, "Web search not available. Set TAVILY_API_KEY.")
+
+            logger.info("Using web search only mode")
+            web_results = await web_searcher.search(request.query, max_results=5)
+
+            # Format results for LLM processing
+            web_context = web_searcher.format_results_for_llm(web_results)
+
+            # Synthesize answer using Gemini
+            logger.info("Synthesizing web search results with Gemini")
+            answer = await synthesize_web_results_with_gemini(
+                query=request.query,
+                web_context=web_context,
+                rag_context=None
+            )
+
+            result = {
+                "answer": answer,
+                "original_query": request.query,
+                "improved_query": request.query,
+                "verification_passed": False,
+                "verification_score": 0,
+                "web_search_performed": True,
+                "sources": [{"url": r.get("url"), "title": r.get("title")} for r in web_results.get("results", [])]
+            }
+        else:
+            # Standard RAG query with optimized parameters
+            rag = await get_rag_instance(request.domain)
+
+            # Determine optimal parameters based on fast_mode
+            if request.fast_mode:
+                # Optimized parameters for 2-3x speedup
+                top_k = request.top_k if request.top_k is not None else 20
+                chunk_top_k = 10
+                max_entity_tokens = 4000
+                max_relation_tokens = 6000
+                max_total_tokens = 20000
+                logger.info(f"⚡ Fast mode enabled: top_k={top_k}, chunk_top_k={chunk_top_k}")
+            else:
+                # Default parameters (higher quality, slower)
+                top_k = request.top_k if request.top_k is not None else 40
+                chunk_top_k = 20
+                max_entity_tokens = 6000
+                max_relation_tokens = 8000
+                max_total_tokens = 30000
+
+            # Build query parameters
+            from lightrag import QueryParam
+            query_kwargs = {
+                "top_k": top_k,
+                "chunk_top_k": chunk_top_k,
+                "max_entity_tokens": max_entity_tokens,
+                "max_relation_tokens": max_relation_tokens,
+                "max_total_tokens": max_total_tokens,
+            }
+
+            # Log toggle settings
+            logger.info(f"Query settings - improvement: {request.enable_query_improvement}, verification: {request.enable_verification_check}")
+
+            result = await rag.aquery(
+                query=request.query,
+                mode=request.mode,
+                enable_query_improvement=request.enable_query_improvement,  # Use toggle instead of always true
+                enable_verification=request.enable_verification_check,  # Use toggle instead of always request.enable_verification
+                return_verification_info=request.return_metadata,
+                **query_kwargs
+            )
+
+            # Augment with web search if requested
+            if request.enable_web_search and web_searcher:
+                logger.info("Augmenting RAG results with web search")
+                try:
+                    rag_answer = result.get("answer") if isinstance(result, dict) else str(result)
+                    web_results = await web_searcher.search(request.query, max_results=5)
+
+                    if web_results.get("results"):
+                        # Format web results for LLM
+                        web_context = web_searcher.format_results_for_llm(web_results)
+
+                        # Synthesize combined answer using Gemini
+                        logger.info("Synthesizing RAG + web results with Gemini")
+                        synthesized_answer = await synthesize_web_results_with_gemini(
+                            query=request.query,
+                            web_context=web_context,
+                            rag_context=rag_answer
+                        )
+
+                        if isinstance(result, dict):
+                            result["answer"] = synthesized_answer
+                            result["web_search_performed"] = True
+                            result["web_sources"] = [{"url": r.get("url"), "title": r.get("title")} for r in web_results.get("results", [])]
+                        else:
+                            result = synthesized_answer
+                except Exception as e:
+                    logger.error(f"Web search augmentation error: {e}")
+                    # Continue with RAG-only result
+
+        # Handle None result
+        if result is None:
+            answer = "I couldn't find any relevant information in the knowledge base to answer your question. Please try rephrasing your question or ensure that relevant documents have been uploaded."
+            metadata = {
+                "original_query": request.query,
+                "improved_query": request.query,
+                "verification_passed": False,
+                "verification_score": 0,
+            }
+            query_improved = False
+            verification_performed = False
+            confidence = 0.0
+        elif isinstance(result, dict):
+            answer = result.get("answer", "No answer found.")
+            metadata = {
+                "original_query": result.get("original_query", request.query),
+                "improved_query": result.get("improved_query", request.query),
+                "verification_passed": result.get("verification_passed", False),
+                "verification_score": result.get("verification_score", 0),
+            }
+            query_improved = result.get("improved_query") != result.get("original_query")
+            verification_performed = result.get("verification_passed", False)
+            confidence = result.get("verification_score", 0) / 10.0
+        else:
+            answer = str(result) if result else "No answer found."
+            metadata = {}
+            query_improved = False
+            verification_performed = False
+            confidence = 1.0
+
+        conversation_history.extend([
+            {"role": "user", "content": request.query},
+            {"role": "assistant", "content": answer}
+        ])
+        conversation_histories[conversation_id] = conversation_history
+
+        response = QueryResponse(
+            answer=answer,
+            sources=[],  # TODO: Extract from result if available
+            confidence_score=confidence,
+            query_improved=query_improved,
+            verification_performed=verification_performed,
+            conversation_id=conversation_id,
+            metadata=metadata if request.return_metadata else None
+        )
+
+        # Store in cache if enabled (non-web search queries only)
+        if cache_key and request.enable_cache:
+            query_cache[cache_key] = response
+            logger.info(f"✓ Cached query result (key: {cache_key[:16]}...)")
+
+        # Track performance metrics
+        query_time = time.time() - start_time
+        performance_metrics["query_times"].append(query_time)
+        # Keep only last 100 metrics
+        if len(performance_metrics["query_times"]) > 100:
+            performance_metrics["query_times"] = performance_metrics["query_times"][-100:]
+
+        logger.info(f"Query completed in {query_time:.2f}s (fast_mode: {request.fast_mode}, confidence: {confidence:.2f})")
+        return response
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Query error: {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail=f"Query failed: {str(e)}")
+
+
+@app.get("/conversation/{conversation_id}")
+async def get_conversation(conversation_id: str):
+    """Get conversation history by ID."""
+    if conversation_id not in conversation_histories:
+        raise HTTPException(status_code=404, detail="Conversation not found")
+    return {
+        "conversation_id": conversation_id,
+        "messages": conversation_histories[conversation_id],
+    }
+
+
+@app.delete("/conversation/{conversation_id}")
+async def clear_conversation(conversation_id: str):
+    """Clear conversation history."""
+    if conversation_id in conversation_histories:
+        del conversation_histories[conversation_id]
+        logger.info(f"Cleared conversation: {conversation_id}")
+        return {"success": True, "message": "Conversation cleared"}
+    raise HTTPException(status_code=404, detail="Conversation not found")
+
+
+@app.delete("/clear/{domain}")
+async def clear_domain_data(domain: str):
+    """WARNING: Deletes all processed documents and indices for the domain."""
+    logger.warning(f"Clear domain data request: {domain}")
+    try:
+        if domain not in DOMAIN_CONFIGS:
+            raise HTTPException(400, f"Invalid domain. Valid: {list(DOMAIN_CONFIGS.keys())}")
+        
+        if domain in rag_instances:
+            await rag_instances[domain].finalize_storages()
+            del rag_instances[domain]
+
+        domain_storage = STORAGE_DIR / domain
+        if domain_storage.exists():
+            import shutil
+            shutil.rmtree(domain_storage)
+            domain_storage.mkdir(parents=True, exist_ok=True)
+        
+        logger.info(f"Domain data cleared: {domain}")
+        return {"success": True, "message": f"All data cleared for domain '{domain}'"}
+    except Exception as e:
+        logger.error(f"Clear domain error: {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail=f"Failed to clear domain: {str(e)}")
+
+
+@app.get("/documents")
+async def list_documents(domain: str):
+    """
+    List all processed documents for a domain.
+
+    Only returns documents with status 'completed'. Documents still being
+    processed are excluded to avoid confusion.
+    """
+    try:
+        if domain not in DOMAIN_CONFIGS:
+            raise HTTPException(400, f"Invalid domain. Valid: {list(DOMAIN_CONFIGS.keys())}")
+
+        documents = []
+        domain_upload_dir = UPLOAD_DIR / domain
+
+        if domain_upload_dir.exists():
+            for file_path in domain_upload_dir.glob("*"):
+                if file_path.is_file():
+                    # Extract processing_id and filename
+                    filename = file_path.name
+                    parts = filename.split('_', 1)
+                    processing_id = parts[0] if len(parts) > 1 else ""
+                    display_name = parts[1] if len(parts) > 1 else filename
+
+                    # Check if document is actually completed
+                    # Skip if still processing, queued, or fetching
+                    if processing_id in processing_status:
+                        status = processing_status[processing_id].get('status')
+                        if status in ['processing', 'queued', 'fetching']:
+                            # Document is still being processed, skip it
+                            logger.debug(f"Skipping document {processing_id} - status: {status}")
+                            continue
+                        elif status == 'failed':
+                            # Optionally skip failed documents or include them
+                            # For now, skip them to only show successfully processed docs
+                            continue
+
+                    # Only include completed documents or legacy ones without status
+                    documents.append({
+                        "id": processing_id,
+                        "name": display_name,
+                        "domain": domain,
+                        "status": "processed",
+                        "uploadedAt": str(file_path.stat().st_mtime)
+                    })
+
+        return {"documents": documents}
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error listing documents: {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail=f"Failed to list documents: {str(e)}")
+
+
+@app.get("/performance-metrics")
+async def get_performance_metrics():
+    """Get performance metrics for queries and document processing."""
+    try:
+        query_times = performance_metrics.get("query_times", [])
+        processing_times = performance_metrics.get("processing_times", [])
+
+        # Calculate statistics
+        def calc_stats(times):
+            if not times:
+                return {"count": 0, "avg": 0, "min": 0, "max": 0}
+            return {
+                "count": len(times),
+                "avg": sum(times) / len(times),
+                "min": min(times),
+                "max": max(times)
+            }
+
+        return {
+            "query_metrics": calc_stats(query_times),
+            "processing_metrics": calc_stats(processing_times),
+            "cache_stats": {
+                "size": len(query_cache),
+                "max_size": query_cache.maxsize,
+                "ttl_seconds": query_cache.ttl
+            }
+        }
+    except Exception as e:
+        logger.error(f"Error getting performance metrics: {e}", exc_info=True)
+        return {
+            "query_metrics": {"count": 0, "avg": 0, "min": 0, "max": 0},
+            "processing_metrics": {"count": 0, "avg": 0, "min": 0, "max": 0},
+            "cache_stats": {"size": 0, "max_size": 100, "ttl_seconds": 300}
+        }
+
+
+@app.get("/status/{processing_id}")
+async def get_processing_status(processing_id: str):
+    """
+    Get the processing status of a document.
+
+    Now uses persistent status storage that survives backend restarts.
+    The status is loaded from disk on startup and kept in sync.
+    """
+    try:
+        # Check the persistent status tracker (loaded from disk on startup)
+        if processing_id in processing_status:
+            status_info = processing_status[processing_id]
+            logger.debug(f"Status check for {processing_id}: {status_info.get('status')}")
+            return {
+                "processing_id": processing_id,
+                **status_info
+            }
+
+        # If not in status tracker, check if this is a legacy upload
+        # (uploaded before persistent status was implemented)
+        for domain in DOMAIN_CONFIGS.keys():
+            domain_upload_dir = UPLOAD_DIR / domain
+            if domain_upload_dir.exists():
+                for file_path in domain_upload_dir.glob(f"{processing_id}_*"):
+                    if file_path.is_file():
+                        # Legacy upload - return completed status
+                        # but don't add to persistent status to avoid confusion
+                        return {
+                            "processing_id": processing_id,
+                            "status": "completed",
+                            "message": "Document processed successfully (legacy upload)"
+                        }
+
+        # If not found anywhere, status is unknown
+        # This typically means the processing_id is invalid
+        return {
+            "processing_id": processing_id,
+            "status": "unknown",
+            "message": "Processing ID not found. It may be invalid or expired."
+        }
+    except Exception as e:
+        logger.error(f"Error checking status: {e}", exc_info=True)
+        return {
+            "processing_id": processing_id,
+            "status": "error",
+            "message": f"Error checking status: {str(e)}",
+            "error": str(e)
+        }
+
+
+@app.delete("/documents/{doc_id}")
+async def delete_document(doc_id: str):
+    """
+    Delete a processed document completely including all RAG data.
+
+    This endpoint performs comprehensive deletion of:
+    - Knowledge graph entities and relationships
+    - Embedding vectors (chunks, entities, relationships)
+    - Text chunks and metadata
+    - Document status records
+    - Physical upload files
+    - Parser output files
+
+    Returns detailed deletion report with verification.
+    """
+    try:
+        from raganything.deletion_verifier import delete_document_complete
+
+        logger.info(f"Delete document request: {doc_id}")
+
+        # Step 1: Search for the document in all domains
+        found_domain = None
+        for domain in DOMAIN_CONFIGS.keys():
+            domain_upload_dir = UPLOAD_DIR / domain
+            if domain_upload_dir.exists():
+                for file_path in domain_upload_dir.glob(f"{doc_id}_*"):
+                    if file_path.is_file():
+                        found_domain = domain
+                        break
+            if found_domain:
+                break
+
+        if not found_domain:
+            logger.warning(f"Document {doc_id} not found in any domain")
+            raise HTTPException(status_code=404, detail="Document not found")
+
+        logger.info(f"Found document {doc_id} in domain: {found_domain}")
+
+        # Step 2: Get RAG instance and find the actual document ID in storage
+        rag = await get_rag_instance(found_domain)
+
+        # Find document in doc_status by processing_id prefix
+        doc_to_delete = None
+        doc_status_file = STORAGE_DIR / found_domain / "kv_store_doc_status.json"
+        if doc_status_file.exists():
+            import json
+            with open(doc_status_file, 'r') as f:
+                doc_status = json.load(f)
+
+            # Find document by file_path containing doc_id
+            for doc_key, doc_info in doc_status.items():
+                if 'file_path' in doc_info and doc_id in doc_info['file_path']:
+                    doc_to_delete = doc_key
+                    logger.info(f"Found document in storage: {doc_key}")
+                    break
+
+        if not doc_to_delete:
+            logger.warning(f"Document {doc_id} not found in doc_status")
+            # Still try to delete physical files
+            doc_to_delete = doc_id
+
+        # Step 3: Collect files and directories to delete
+        upload_files = list((UPLOAD_DIR / found_domain).glob(f"{doc_id}_*"))
+        output_dir = BASE_DIR / "backend" / "output"
+        output_paths = list(output_dir.glob(f"{doc_id}_*")) if output_dir.exists() else []
+
+        # Step 4: Perform complete deletion with verification
+        deletion_report = await delete_document_complete(
+            rag_instance=rag,
+            doc_id=doc_to_delete,
+            storage_dir=STORAGE_DIR / found_domain,
+            upload_files=upload_files,
+            output_dirs=output_paths
+        )
+
+        # Step 5: Return detailed report
+        if deletion_report.success:
+            logger.info(
+                f"Successfully deleted document {doc_id}: "
+                f"{deletion_report.chunks_deleted} chunks, "
+                f"{deletion_report.entities_deleted} entities, "
+                f"{deletion_report.relationships_deleted} relationships, "
+                f"{len(deletion_report.files_deleted)} files, "
+                f"{len(deletion_report.directories_deleted)} directories"
+            )
+            return {
+                "success": True,
+                "message": "Document deleted completely with verification",
+                "domain": found_domain,
+                "report": deletion_report.to_dict()
+            }
+        else:
+            logger.error(
+                f"Document deletion completed with errors for {doc_id}: "
+                f"{deletion_report.errors}"
+            )
+            return {
+                "success": False,
+                "message": "Document deletion completed with errors",
+                "domain": found_domain,
+                "report": deletion_report.to_dict()
+            }
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error deleting document {doc_id}: {e}", exc_info=True)
+        raise HTTPException(
+            status_code=500,
+            detail=f"Failed to delete document: {str(e)}"
+        )
+
+
+# =============================================================================
+# Main Entry Point
+# =============================================================================
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run("main:app", host="0.0.0.0", port=8000, reload=True, log_level="info")

backend/requirements.txt

@@ -0,0 +1,28 @@
+# FastAPI Backend Requirements - Python 3.12 compatible
+
+# Web Framework
+fastapi>=0.104.0
+uvicorn[standard]>=0.24.0
+python-multipart>=0.0.6
+
+# Google Gemini API
+google-generativeai>=0.8.0
+
+# Image Processing
+Pillow>=10.0.0
+
+# Environment Variables
+python-dotenv>=1.0.0
+
+# Web Search & URL Fetching
+tavily-python>=0.3.0
+requests>=2.31.0
+beautifulsoup4>=4.12.0
+markdownify>=0.11.0
+
+# Additional dependencies
+cachetools>=5.3.0
+aiofiles>=23.0.0
+
+# LightRAG - Using local modified version in /lightrag directory
+# lightrag-hku>=1.4.0

backend/reranker.py

@@ -0,0 +1,304 @@
+"""
+Reranking Module for RAG-Anything
+
+Provides reranking functionality using:
+1. Gemini-based LLM reranking (free tier compatible)
+2. Cross-encoder style scoring
+3. Relevance-based reordering
+
+Reranking is crucial for RAG systems because:
+- Vector search (embeddings) finds semantically similar text but may miss subtle context
+- LLMs can deeply understand query intent and document relevance
+- Reranking improves answer quality by promoting truly relevant chunks to the top
+
+Author: RAG-Anything Team
+Version: 1.0.0
+"""
+
+import asyncio
+import logging
+import re
+from typing import List, Dict, Any, Optional, Callable
+
+logger = logging.getLogger(__name__)
+
+
+class GeminiReranker:
+    """
+    Reranker using Gemini API for semantic relevance scoring
+
+    This reranker takes chunks from vector search and re-scores them
+    based on deep semantic understanding using an LLM.
+
+    Why reranking matters:
+    ---------------------
+    Vector embeddings alone can miss:
+    - Negations ("not effective" vs "effective")
+    - Context dependencies ("aspirin for elderly" vs "aspirin for children")
+    - Query intent ("what causes X" vs "how to prevent X")
+
+    LLM reranking provides:
+    - Contextual understanding of the query
+    - Semantic relevance beyond keyword matching
+    - Better handling of complex queries
+    """
+
+    def __init__(
+        self,
+        llm_func: Optional[Callable] = None,
+        model_name: str = "models/gemini-2.5-flash",
+        batch_size: int = 5,
+        temperature: float = 0.1
+    ):
+        """
+        Initialize Gemini-based reranker
+
+        Args:
+            llm_func: Optional LLM function to use for reranking
+            model_name: Gemini model to use (default: flash for speed)
+            batch_size: Number of chunks to process in parallel
+            temperature: Temperature for relevance scoring (low=consistent)
+        """
+        self.llm_func = llm_func
+        self.model_name = model_name
+        self.batch_size = batch_size
+        self.temperature = temperature
+
+    async def rerank(
+        self,
+        query: str,
+        chunks: List[Dict[str, Any]],
+        top_k: Optional[int] = None
+    ) -> List[Dict[str, Any]]:
+        """
+        Rerank chunks based on relevance to query
+
+        Process:
+        1. Take top chunks from vector search (e.g., top 50)
+        2. Score each chunk's relevance using LLM (0-10 scale)
+        3. Re-order by relevance score
+        4. Return top_k most relevant chunks
+
+        Args:
+            query: Search query
+            chunks: List of chunks with 'content' field
+            top_k: Return only top K results (None = return all, reranked)
+
+        Returns:
+            List of reranked chunks with 'relevance_score' field added
+        """
+        if not chunks:
+            logger.warning("No chunks to rerank")
+            return []
+
+        if len(chunks) == 1:
+            logger.debug("Only one chunk, skipping reranking")
+            chunks[0]['relevance_score'] = 1.0
+            return chunks
+
+        logger.info(f"Reranking {len(chunks)} chunks for query: {query[:50]}...")
+
+        try:
+            # Score all chunks in batches
+            scored_chunks = await self._score_chunks_batch(query, chunks)
+
+            # Sort by relevance score (highest first)
+            scored_chunks.sort(key=lambda x: x.get('relevance_score', 0), reverse=True)
+
+            # Return top_k if specified
+            if top_k:
+                scored_chunks = scored_chunks[:top_k]
+
+            logger.info(
+                f"Reranking complete. Top score: {scored_chunks[0].get('relevance_score', 0):.2f}, "
+                f"Bottom score: {scored_chunks[-1].get('relevance_score', 0):.2f}"
+            )
+
+            return scored_chunks
+
+        except Exception as e:
+            logger.error(f"Error during reranking: {e}", exc_info=True)
+            # Return original order on error
+            return chunks[:top_k] if top_k else chunks
+
+    async def _score_chunks_batch(
+        self,
+        query: str,
+        chunks: List[Dict[str, Any]]
+    ) -> List[Dict[str, Any]]:
+        """
+        Score chunks in batches for efficiency
+
+        Args:
+            query: Search query
+            chunks: List of chunks to score
+
+        Returns:
+            Chunks with relevance_score added
+        """
+        scored_chunks = []
+
+        # Process in batches to avoid rate limits
+        for i in range(0, len(chunks), self.batch_size):
+            batch = chunks[i:i + self.batch_size]
+
+            # Score batch concurrently
+            tasks = [self._score_chunk(query, chunk) for chunk in batch]
+            batch_scores = await asyncio.gather(*tasks, return_exceptions=True)
+
+            # Collect results
+            for chunk, score_result in zip(batch, batch_scores):
+                if isinstance(score_result, Exception):
+                    logger.warning(f"Failed to score chunk: {score_result}")
+                    chunk['relevance_score'] = 0.0
+                else:
+                    chunk['relevance_score'] = score_result
+
+                scored_chunks.append(chunk)
+
+        return scored_chunks
+
+    async def _score_chunk(
+        self,
+        query: str,
+        chunk: Dict[str, Any]
+    ) -> float:
+        """
+        Score a single chunk's relevance to the query using LLM
+
+        Prompt engineering approach:
+        - Ask LLM to act as a relevance judge
+        - Provide clear scoring criteria (0-10 scale)
+        - Extract numeric score from response
+
+        Args:
+            query: Search query
+            chunk: Chunk dictionary with 'content' field
+
+        Returns:
+            Relevance score (0-10)
+        """
+        content = chunk.get('content', '')
+        if not content:
+            return 0.0
+
+        # Truncate very long chunks to avoid token limits
+        max_content_length = 1000
+        if len(content) > max_content_length:
+            content = content[:max_content_length] + "..."
+
+        # Prompt for relevance scoring
+        prompt = f"""You are a relevance judge. Score how relevant the following passage is to answering the query.
+
+Query: {query}
+
+Passage:
+{content}
+
+Scoring criteria:
+10 = Directly answers the query with specific, relevant information
+8-9 = Highly relevant, provides useful context
+6-7 = Somewhat relevant, contains related information
+4-5 = Tangentially related, limited usefulness
+2-3 = Barely related, mostly off-topic
+0-1 = Completely irrelevant
+
+Respond with ONLY a number from 0-10. No explanation needed."""
+
+        try:
+            # Call LLM for scoring
+            if self.llm_func:
+                response = await self.llm_func(
+                    prompt=prompt,
+                    temperature=self.temperature,
+                    max_tokens=50  # Increased from 10 to allow for complete score responses
+                )
+            else:
+                # Fallback: no reranking
+                return 5.0
+
+            # Extract numeric score from response
+            score = self._extract_score(response)
+            return score
+
+        except Exception as e:
+            logger.error(f"Error scoring chunk: {e}")
+            return 5.0  # Default mid-range score on error
+
+    def _extract_score(self, response: str) -> float:
+        """
+        Extract numeric score from LLM response
+
+        Handles various response formats:
+        - "8.5"
+        - "Score: 9"
+        - "The relevance is 7/10"
+        - "8"
+
+        Args:
+            response: LLM response text
+
+        Returns:
+            Extracted score (0-10), defaults to 5.0 if parsing fails
+        """
+        try:
+            # Remove whitespace
+            response = response.strip()
+
+            # Try to find a number (int or float) in the response
+            # Pattern matches: "8", "8.5", "9/10", "Score: 7", etc.
+            number_pattern = r'(\d+\.?\d*)'
+            matches = re.findall(number_pattern, response)
+
+            if matches:
+                # Take the first number found
+                score = float(matches[0])
+
+                # Normalize to 0-10 range
+                score = max(0.0, min(10.0, score))
+
+                return score
+            else:
+                logger.warning(f"Could not extract score from response: {response}")
+                return 5.0
+
+        except Exception as e:
+            logger.error(f"Error extracting score: {e}")
+            return 5.0
+
+
+# Example usage
+async def main():
+    """Example demonstrating reranking"""
+    # Mock LLM function for testing
+    async def mock_llm(prompt: str, **kwargs) -> str:
+        # Simulate scoring based on keyword matching
+        if "directly" in prompt.lower():
+            return "9"
+        elif "somewhat" in prompt.lower():
+            return "6"
+        else:
+            return "3"
+
+    # Create reranker
+    reranker = GeminiReranker(llm_func=mock_llm)
+
+    # Example query and chunks
+    query = "What are the side effects of aspirin?"
+
+    chunks = [
+        {"content": "Aspirin can cause stomach bleeding in some patients..."},
+        {"content": "The history of aspirin dates back to ancient times..."},
+        {"content": "Common side effects include nausea and heartburn..."},
+    ]
+
+    # Rerank
+    reranked = await reranker.rerank(query, chunks, top_k=2)
+
+    print("Reranked results:")
+    for i, chunk in enumerate(reranked, 1):
+        print(f"{i}. Score: {chunk['relevance_score']:.1f} - {chunk['content'][:50]}...")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

backend/url_fetcher.py

@@ -0,0 +1,381 @@
+"""
+URL Document Fetcher for RAG-Anything
+
+Fetches and processes documents from URLs for ingestion into the RAG system.
+
+Features:
+- Web page scraping and parsing
+- PDF download from URLs
+- Markdown conversion
+- Content cleaning and preprocessing
+- Advanced parsing with text and image extraction
+- Integration with RAG pipeline
+
+Author: RAG-Anything Team
+Version: 2.0.0
+"""
+
+import os
+import asyncio
+import logging
+import tempfile
+from pathlib import Path
+from typing import Optional, Dict, Any, List
+from urllib.parse import urlparse
+import hashlib
+import base64
+
+logger = logging.getLogger(__name__)
+
+try:
+    import requests
+    from bs4 import BeautifulSoup
+    import markdownify
+    from urllib.parse import urljoin
+    DEPS_AVAILABLE = True
+except ImportError:
+    DEPS_AVAILABLE = False
+    logger.warning("URL fetcher dependencies not installed. Install with: pip install requests beautifulsoup4 markdownify")
+
+
+class URLFetcher:
+    """Fetch and process documents from URLs"""
+
+    def __init__(
+        self,
+        download_dir: Optional[str] = None,
+        timeout: int = 30,
+        user_agent: str = "RAG-Anything/1.0"
+    ):
+        """
+        Initialize URL fetcher
+
+        Args:
+            download_dir: Directory to save downloaded files
+            timeout: Request timeout in seconds
+            user_agent: User agent string for requests
+        """
+        if not DEPS_AVAILABLE:
+            raise ImportError("Required dependencies not installed. Run: pip install requests beautifulsoup4 markdownify")
+
+        self.download_dir = download_dir or tempfile.gettempdir()
+        self.timeout = timeout
+        self.headers = {"User-Agent": user_agent}
+
+        Path(self.download_dir).mkdir(parents=True, exist_ok=True)
+        logger.info(f"URLFetcher initialized (download_dir={self.download_dir})")
+
+    def _create_content_list(self, title: str, text_content: str, images: List[Dict]) -> List[Dict[str, Any]]:
+        """
+        Create a structured content list compatible with RAG pipeline
+
+        Args:
+            title: Document title
+            text_content: Extracted text content
+            images: List of extracted images with metadata
+
+        Returns:
+            List of content blocks for RAG processing
+        """
+        content_list = []
+
+        # Add title as first text block
+        if title:
+            content_list.append({
+                "type": "text",
+                "text": f"# {title}",
+                "page_idx": 0
+            })
+
+        # Split text into paragraphs and add as text blocks
+        paragraphs = [p.strip() for p in text_content.split("\n\n") if p.strip()]
+        for idx, paragraph in enumerate(paragraphs[:50]):  # Limit to first 50 paragraphs
+            if paragraph:
+                content_list.append({
+                    "type": "text",
+                    "text": paragraph,
+                    "page_idx": idx // 10  # Group every 10 paragraphs as a "page"
+                })
+
+        # Add images as image blocks
+        for idx, img_info in enumerate(images):
+            content_list.append({
+                "type": "image",
+                "img_path": img_info["path"],
+                "image_caption": img_info.get("alt", "") or img_info.get("title", ""),
+                "page_idx": (len(paragraphs) + idx) // 10
+            })
+
+        return content_list
+
+    async def fetch_url(
+        self,
+        url: str,
+        save_as_pdf: bool = False,
+        convert_to_markdown: bool = True
+    ) -> Dict[str, Any]:
+        """
+        Fetch and process content from URL
+
+        Args:
+            url: URL to fetch
+            save_as_pdf: Whether to save as PDF (for PDF URLs)
+            convert_to_markdown: Convert HTML to markdown
+
+        Returns:
+            Dictionary with file_path, content, metadata
+        """
+        try:
+            logger.info(f"Fetching URL: {url}")
+
+            # Validate URL
+            parsed = urlparse(url)
+            if not parsed.scheme or not parsed.netloc:
+                raise ValueError(f"Invalid URL: {url}")
+
+            # Determine content type
+            response = await asyncio.to_thread(
+                requests.head, url, headers=self.headers, timeout=self.timeout, allow_redirects=True
+            )
+            content_type = response.headers.get("Content-Type", "").lower()
+
+            # Handle PDF files
+            if "pdf" in content_type or url.lower().endswith(".pdf"):
+                return await self._fetch_pdf(url)
+
+            # Handle HTML/web pages
+            elif "html" in content_type or not content_type:
+                return await self._fetch_html(url, convert_to_markdown)
+
+            # Handle other file types
+            else:
+                return await self._fetch_generic(url, content_type)
+
+        except Exception as e:
+            logger.error(f"Error fetching URL {url}: {e}", exc_info=True)
+            return {
+                "success": False,
+                "error": str(e),
+                "url": url,
+            }
+
+    async def _fetch_pdf(self, url: str) -> Dict[str, Any]:
+        """Fetch PDF from URL"""
+        try:
+            response = await asyncio.to_thread(
+                requests.get, url, headers=self.headers, timeout=self.timeout
+            )
+            response.raise_for_status()
+
+            # Generate filename from URL
+            url_hash = hashlib.md5(url.encode()).hexdigest()[:8]
+            filename = f"url_{url_hash}.pdf"
+            file_path = Path(self.download_dir) / filename
+
+            # Save PDF
+            with open(file_path, "wb") as f:
+                f.write(response.content)
+
+            logger.info(f"PDF downloaded: {file_path}")
+
+            return {
+                "success": True,
+                "file_path": str(file_path),
+                "url": url,
+                "content_type": "pdf",
+                "size_bytes": len(response.content),
+            }
+
+        except Exception as e:
+            logger.error(f"Error fetching PDF: {e}")
+            raise
+
+    async def _fetch_html(self, url: str, convert_to_markdown: bool = True) -> Dict[str, Any]:
+        """Fetch and parse HTML page with advanced content extraction"""
+        try:
+            response = await asyncio.to_thread(
+                requests.get, url, headers=self.headers, timeout=self.timeout
+            )
+            response.raise_for_status()
+
+            # Parse HTML
+            soup = BeautifulSoup(response.content, "html.parser")
+
+            # Remove unwanted elements
+            for tag in soup(["script", "style", "nav", "footer", "header", "aside", "iframe", "noscript"]):
+                tag.decompose()
+
+            # Extract title
+            title = soup.find("title")
+            title_text = title.get_text().strip() if title else "Untitled"
+
+            # Extract main content
+            main_content = soup.find("main") or soup.find("article") or soup.find("body")
+
+            # Extract images before converting to markdown (limit to first 10 images)
+            images = []
+            url_hash = hashlib.md5(url.encode()).hexdigest()[:8]
+            images_dir = Path(self.download_dir) / f"url_{url_hash}_images"
+            images_dir.mkdir(parents=True, exist_ok=True)
+
+            all_images = main_content.find_all("img")
+            max_images = min(10, len(all_images))  # Limit to 10 images
+            logger.info(f"Found {len(all_images)} images, downloading first {max_images}")
+
+            for idx, img in enumerate(all_images[:max_images]):
+                try:
+                    img_url = img.get("src")
+                    if not img_url:
+                        continue
+
+                    # Skip data URIs and very small images
+                    if img_url.startswith("data:"):
+                        continue
+
+                    # Handle relative URLs
+                    if img_url.startswith("//"):
+                        img_url = "https:" + img_url
+                    elif img_url.startswith("/"):
+                        parsed_base = urlparse(url)
+                        img_url = f"{parsed_base.scheme}://{parsed_base.netloc}{img_url}"
+                    elif not img_url.startswith("http"):
+                        img_url = urljoin(url, img_url)
+
+                    # Download image with timeout
+                    img_response = await asyncio.to_thread(
+                        requests.get, img_url, headers=self.headers, timeout=5, stream=True
+                    )
+
+                    if img_response.status_code == 200:
+                        # Check content size (skip if too large > 10MB)
+                        content_length = img_response.headers.get('content-length')
+                        if content_length and int(content_length) > 10 * 1024 * 1024:
+                            logger.debug(f"Skipping large image {idx}: {content_length} bytes")
+                            continue
+
+                        # Determine file extension
+                        content_type = img_response.headers.get("Content-Type", "")
+                        ext = ".jpg"
+                        if "png" in content_type:
+                            ext = ".png"
+                        elif "gif" in content_type:
+                            ext = ".gif"
+                        elif "webp" in content_type:
+                            ext = ".webp"
+
+                        img_path = images_dir / f"image_{idx}{ext}"
+                        with open(img_path, "wb") as f:
+                            f.write(img_response.content)
+
+                        images.append({
+                            "path": str(img_path),
+                            "alt": img.get("alt", ""),
+                            "title": img.get("title", ""),
+                            "url": img_url
+                        })
+                        logger.debug(f"Downloaded image {idx+1}/{max_images}: {img_path.name}")
+                except Exception as img_error:
+                    logger.debug(f"Failed to download image {idx}: {img_error}")
+                    continue
+
+            if convert_to_markdown:
+                # Convert to markdown
+                content = markdownify.markdownify(
+                    str(main_content),
+                    heading_style="ATX",
+                    bullets="-"
+                )
+            else:
+                # Extract plain text
+                content = main_content.get_text(separator="\n", strip=True)
+
+            # Create content list with structured data
+            content_list = self._create_content_list(title_text, content, images)
+
+            # Save to file
+            ext = ".md" if convert_to_markdown else ".txt"
+            filename = f"url_{url_hash}{ext}"
+            file_path = Path(self.download_dir) / filename
+
+            with open(file_path, "w", encoding="utf-8") as f:
+                f.write(f"# {title_text}\n\n")
+                f.write(f"Source: {url}\n\n")
+                f.write(content)
+
+            # Save content list as JSON for RAG processing
+            import json
+            json_path = Path(self.download_dir) / f"url_{url_hash}_content_list.json"
+            with open(json_path, "w", encoding="utf-8") as f:
+                json.dump(content_list, f, indent=2, ensure_ascii=False)
+
+            logger.info(f"HTML content saved: {file_path}")
+            logger.info(f"Extracted {len(images)} images from web page")
+
+            return {
+                "success": True,
+                "file_path": str(file_path),
+                "content_list_path": str(json_path),
+                "url": url,
+                "content_type": "html",
+                "title": title_text,
+                "content_preview": content[:500],
+                "images_count": len(images),
+                "content_list": content_list
+            }
+
+        except Exception as e:
+            logger.error(f"Error fetching HTML: {e}")
+            raise
+
+    async def _fetch_generic(self, url: str, content_type: str) -> Dict[str, Any]:
+        """Fetch generic file"""
+        try:
+            response = await asyncio.to_thread(
+                requests.get, url, headers=self.headers, timeout=self.timeout
+            )
+            response.raise_for_status()
+
+            # Determine extension from content type
+            ext_map = {
+                "text/plain": ".txt",
+                "text/markdown": ".md",
+                "application/msword": ".doc",
+                "application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx",
+            }
+            ext = ext_map.get(content_type, ".bin")
+
+            # Save file
+            url_hash = hashlib.md5(url.encode()).hexdigest()[:8]
+            filename = f"url_{url_hash}{ext}"
+            file_path = Path(self.download_dir) / filename
+
+            with open(file_path, "wb") as f:
+                f.write(response.content)
+
+            logger.info(f"File downloaded: {file_path}")
+
+            return {
+                "success": True,
+                "file_path": str(file_path),
+                "url": url,
+                "content_type": content_type,
+                "size_bytes": len(response.content),
+            }
+
+        except Exception as e:
+            logger.error(f"Error fetching file: {e}")
+            raise
+
+
+def create_url_fetcher(download_dir: Optional[str] = None, **kwargs) -> URLFetcher:
+    """
+    Factory function to create a URL fetcher
+
+    Args:
+        download_dir: Directory to save downloaded files
+        **kwargs: Additional URLFetcher parameters
+
+    Returns:
+        Configured URLFetcher instance
+    """
+    return URLFetcher(download_dir=download_dir, **kwargs)

backend/web_search.py

@@ -0,0 +1,295 @@
+"""
+Web Search Module for RAG-Anything using Tavily API
+
+Provides intelligent web search capabilities to augment RAG with real-time information.
+
+Features:
+- Tavily API integration for high-quality search results
+- Context-aware search query generation
+- Result filtering and ranking
+- Hybrid RAG + Web search mode
+
+Author: RAG-Anything Team
+Version: 1.0.0
+"""
+
+import os
+import asyncio
+import logging
+from typing import List, Dict, Any, Optional
+from datetime import datetime
+
+logger = logging.getLogger(__name__)
+
+try:
+    from tavily import TavilyClient, AsyncTavilyClient
+    TAVILY_AVAILABLE = True
+except ImportError:
+    TAVILY_AVAILABLE = False
+    logger.warning("Tavily not installed. Install with: pip install tavily-python")
+
+
+class WebSearcher:
+    """Web search integration using Tavily API"""
+
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        max_results: int = 5,
+        search_depth: str = "advanced",
+        include_raw_content: bool = True
+    ):
+        """
+        Initialize web searcher
+
+        Args:
+            api_key: Tavily API key (from env if not provided)
+            max_results: Maximum number of search results to return
+            search_depth: "basic" or "advanced" (advanced is more thorough)
+            include_raw_content: Whether to include full page content
+        """
+        if not TAVILY_AVAILABLE:
+            raise ImportError("Tavily is not installed. Install with: pip install tavily-python")
+
+        self.api_key = api_key or os.getenv("TAVILY_API_KEY")
+        if not self.api_key:
+            raise ValueError("Tavily API key not found. Set TAVILY_API_KEY environment variable.")
+
+        self.max_results = max_results
+        self.search_depth = search_depth
+        self.include_raw_content = include_raw_content
+
+        # Initialize async client
+        self.client = AsyncTavilyClient(api_key=self.api_key)
+
+        logger.info(f"WebSearcher initialized (max_results={max_results}, depth={search_depth})")
+
+    async def search(
+        self,
+        query: str,
+        max_results: Optional[int] = None,
+        include_domains: Optional[List[str]] = None,
+        exclude_domains: Optional[List[str]] = None,
+        search_depth: Optional[str] = None
+    ) -> Dict[str, Any]:
+        """
+        Perform web search
+
+        Args:
+            query: Search query
+            max_results: Override default max results
+            include_domains: Only search these domains
+            exclude_domains: Exclude these domains
+            search_depth: Override default search depth
+
+        Returns:
+            Dictionary with search results and metadata
+        """
+        try:
+            logger.info(f"Searching web: {query[:100]}...")
+
+            # Build search parameters
+            search_params = {
+                "query": query,
+                "max_results": max_results or self.max_results,
+                "search_depth": search_depth or self.search_depth,
+                "include_raw_content": self.include_raw_content,
+            }
+
+            if include_domains:
+                search_params["include_domains"] = include_domains
+            if exclude_domains:
+                search_params["exclude_domains"] = exclude_domains
+
+            # Perform search
+            response = await self.client.search(**search_params)
+
+            # Process results
+            results = {
+                "query": query,
+                "results": response.get("results", []),
+                "answer": response.get("answer", ""),  # Tavily's AI-generated answer
+                "search_metadata": {
+                    "total_results": len(response.get("results", [])),
+                    "search_depth": search_params["search_depth"],
+                    "timestamp": datetime.now().isoformat(),
+                }
+            }
+
+            logger.info(f"Web search complete: {len(results['results'])} results found")
+            return results
+
+        except Exception as e:
+            logger.error(f"Web search error: {e}", exc_info=True)
+            return {
+                "query": query,
+                "results": [],
+                "answer": "",
+                "error": str(e),
+                "search_metadata": {
+                    "total_results": 0,
+                    "error": str(e),
+                    "timestamp": datetime.now().isoformat(),
+                }
+            }
+
+    async def search_with_context(
+        self,
+        query: str,
+        context: Optional[str] = None,
+        **kwargs
+    ) -> Dict[str, Any]:
+        """
+        Search with additional context to refine query
+
+        Args:
+            query: Base search query
+            context: Additional context to help refine search
+            **kwargs: Additional search parameters
+
+        Returns:
+            Search results dictionary
+        """
+        # If context provided, enhance query
+        if context:
+            enhanced_query = f"{query} {context}"
+        else:
+            enhanced_query = query
+
+        return await self.search(enhanced_query, **kwargs)
+
+    def format_results_for_rag(self, search_results: Dict[str, Any]) -> str:
+        """
+        Format web search results for RAG context
+
+        Args:
+            search_results: Results from search()
+
+        Returns:
+            Formatted string for RAG context
+        """
+        if not search_results.get("results"):
+            return "No web search results found."
+
+        formatted = ["=== Web Search Results ===\n"]
+
+        # Add Tavily's answer if available
+        if search_results.get("answer"):
+            formatted.append(f"Quick Answer: {search_results['answer']}\n")
+
+        # Add individual results
+        for idx, result in enumerate(search_results["results"], 1):
+            formatted.append(f"\n[Source {idx}] {result.get('title', 'Untitled')}")
+            formatted.append(f"URL: {result.get('url', 'N/A')}")
+            formatted.append(f"Content: {result.get('content', 'No content')[:500]}...")
+            if result.get("score"):
+                formatted.append(f"Relevance: {result['score']:.2f}")
+
+        formatted.append(f"\n=== End of Web Results ({len(search_results['results'])} sources) ===")
+        return "\n".join(formatted)
+
+    def format_results_for_llm(self, search_results: Dict[str, Any]) -> str:
+        """
+        Format web search results optimally for LLM processing
+
+        Args:
+            search_results: Results from search()
+
+        Returns:
+            Structured string optimized for LLM comprehension
+        """
+        if not search_results.get("results"):
+            return "No relevant web search results were found for this query."
+
+        formatted = []
+
+        # Add Tavily's AI-generated answer first (if available)
+        if search_results.get("answer"):
+            formatted.append("### AI-Generated Summary:")
+            formatted.append(search_results['answer'])
+            formatted.append("")
+
+        # Add detailed source information
+        formatted.append("### Detailed Sources:")
+        formatted.append("")
+
+        for idx, result in enumerate(search_results["results"], 1):
+            formatted.append(f"**Source {idx}: {result.get('title', 'Untitled')}**")
+            formatted.append(f"- URL: {result.get('url', 'N/A')}")
+            formatted.append(f"- Published: {result.get('published_date', 'Unknown date')}")
+
+            # Get content (full or truncated based on availability)
+            content = result.get('content', '')
+            if result.get('raw_content') and len(result.get('raw_content', '')) > len(content):
+                content = result['raw_content'][:2000]  # Use more detailed content
+
+            formatted.append(f"- Content: {content}")
+
+            if result.get("score"):
+                formatted.append(f"- Relevance Score: {result['score']:.2%}")
+
+            formatted.append("")
+
+        formatted.append(f"*Total sources: {len(search_results['results'])}*")
+        return "\n".join(formatted)
+
+    async def hybrid_search(
+        self,
+        query: str,
+        rag_results: Optional[str] = None,
+        combine_results: bool = True,
+        **kwargs
+    ) -> Dict[str, Any]:
+        """
+        Hybrid search: combine RAG results with web search
+
+        Args:
+            query: Search query
+            rag_results: Results from RAG system
+            combine_results: Whether to combine RAG and web results
+            **kwargs: Additional search parameters
+
+        Returns:
+            Dictionary with combined results
+        """
+        # Perform web search
+        web_results = await self.search(query, **kwargs)
+
+        if not combine_results:
+            return web_results
+
+        # Combine RAG and web results
+        combined_context = []
+
+        if rag_results:
+            combined_context.append("=== Knowledge Base Results ===")
+            combined_context.append(rag_results)
+            combined_context.append("")
+
+        combined_context.append(self.format_results_for_rag(web_results))
+
+        return {
+            "query": query,
+            "combined_context": "\n".join(combined_context),
+            "rag_results": rag_results,
+            "web_results": web_results,
+            "metadata": {
+                "has_rag_results": bool(rag_results),
+                "has_web_results": len(web_results.get("results", [])) > 0,
+                "timestamp": datetime.now().isoformat(),
+            }
+        }
+
+
+def create_web_searcher(api_key: Optional[str] = None, **kwargs) -> WebSearcher:
+    """
+    Factory function to create a web searcher
+
+    Args:
+        api_key: Tavily API key
+        **kwargs: Additional WebSearcher parameters
+
+    Returns:
+        Configured WebSearcher instance
+    """
+    return WebSearcher(api_key=api_key, **kwargs)

docker-compose.yml

@@ -0,0 +1,58 @@
+services:
+  backend:
+    build:
+      context: .
+      dockerfile: backend/Dockerfile
+    container_name: agentic-rag-backend
+    restart: unless-stopped
+    ports:
+      - "8000:8000"
+    environment:
+      - GEMINI_API_KEY=${GEMINI_API_KEY}
+      - GEMINI_TEXT_MODEL=${GEMINI_TEXT_MODEL:-models/gemini-flash-latest}
+      - GEMINI_VERIFIER_MODEL=${GEMINI_VERIFIER_MODEL:-models/gemini-pro-latest}
+      - GEMINI_VISION_MODEL=${GEMINI_VISION_MODEL:-models/gemini-flash-latest}
+      - GEMINI_EMBEDDING_MODEL=${GEMINI_EMBEDDING_MODEL:-models/text-embedding-004}
+      - TAVILY_API_KEY=${TAVILY_API_KEY}
+      - PYTHONUNBUFFERED=1
+    volumes:
+      - ./storage:/app/storage
+      - ./uploads:/app/uploads
+      - ./backend/output:/app/backend/output
+    networks:
+      - rag-network
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 40s
+
+  frontend:
+    build:
+      context: ./frontend
+      dockerfile: Dockerfile
+    container_name: agentic-rag-frontend
+    restart: unless-stopped
+    ports:
+      - "3000:80"
+    depends_on:
+      backend:
+        condition: service_healthy
+    networks:
+      - rag-network
+    healthcheck:
+      test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost/"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 10s
+
+networks:
+  rag-network:
+    driver: bridge
+
+volumes:
+  storage:
+  uploads:
+  output:

frontend/.env.example

@@ -0,0 +1,2 @@
+# API Configuration
+REACT_APP_API_URL=http://localhost:8000

frontend/Dockerfile

@@ -0,0 +1,33 @@
+FROM node:20-alpine AS builder
+
+WORKDIR /app
+
+# Copy package files
+COPY package*.json ./
+
+# Install dependencies
+RUN npm ci --silent
+
+# Copy source code
+COPY . .
+
+# Build the application
+RUN npm run build
+
+FROM nginx:alpine
+
+# Copy custom nginx configuration
+COPY nginx.conf /etc/nginx/conf.d/default.conf
+
+# Copy built application from builder stage
+COPY --from=builder /app/build /usr/share/nginx/html
+
+# Expose port
+EXPOSE 80
+
+# Health check
+HEALTHCHECK --interval=30s --timeout=10s --start-period=10s --retries=3 \
+    CMD wget --quiet --tries=1 --spider http://localhost/ || exit 1
+
+# Start nginx
+CMD ["nginx", "-g", "daemon off;"]

frontend/nginx.conf

@@ -0,0 +1,46 @@
+server {
+    listen 80;
+    server_name localhost;
+    root /usr/share/nginx/html;
+    index index.html;
+
+    # Gzip compression
+    gzip on;
+    gzip_vary on;
+    gzip_min_length 1000;
+    gzip_types text/plain text/css text/xml text/javascript application/x-javascript application/xml+rss application/json;
+
+    # Serve static files with cache headers
+    location ~* \.(js|css|png|jpg|jpeg|gif|ico|svg|woff|woff2|ttf|eot)$ {
+        expires 1y;
+        add_header Cache-Control "public, immutable";
+    }
+
+    # API proxy to backend
+    location /api {
+        proxy_pass http://backend:8000;
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection 'upgrade';
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_cache_bypass $http_upgrade;
+
+        # Increase timeout for long-running queries
+        proxy_read_timeout 300s;
+        proxy_connect_timeout 75s;
+    }
+
+    # Serve React app - all routes go to index.html
+    location / {
+        try_files $uri $uri/ /index.html;
+        add_header Cache-Control "no-cache";
+    }
+
+    # Security headers
+    add_header X-Frame-Options "SAMEORIGIN" always;
+    add_header X-Content-Type-Options "nosniff" always;
+    add_header X-XSS-Protection "1; mode=block" always;
+}

frontend/package.json

@@ -0,0 +1,43 @@
+{
+  "name": "enhanced-rag-frontend",
+  "version": "1.0.0",
+  "description": "Enhanced RAG-Anything Frontend with Multi-Domain Support",
+  "private": true,
+  "dependencies": {
+    "lucide-react": "^0.294.0",
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0",
+    "react-markdown": "^10.1.0",
+    "react-scripts": "5.0.1",
+    "rehype-highlight": "^7.0.2",
+    "remark-gfm": "^4.0.1"
+  },
+  "scripts": {
+    "start": "react-scripts start",
+    "build": "react-scripts build",
+    "test": "react-scripts test",
+    "eject": "react-scripts eject"
+  },
+  "eslintConfig": {
+    "extends": [
+      "react-app"
+    ]
+  },
+  "browserslist": {
+    "production": [
+      ">0.2%",
+      "not dead",
+      "not op_mini all"
+    ],
+    "development": [
+      "last 1 chrome version",
+      "last 1 firefox version",
+      "last 1 safari version"
+    ]
+  },
+  "devDependencies": {
+    "autoprefixer": "^10.4.16",
+    "postcss": "^8.4.31",
+    "tailwindcss": "^3.3.5"
+  }
+}

frontend/postcss.config.js

@@ -0,0 +1,6 @@
+module.exports = {
+  plugins: {
+    tailwindcss: {},
+    autoprefixer: {},
+  },
+}

frontend/public/index.html

@@ -0,0 +1,17 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <meta name="theme-color" content="#000000" />
+    <meta
+      name="description"
+      content="Enhanced RAG System with Multi-Domain Support"
+    />
+    <title>Enhanced RAG System</title>
+  </head>
+  <body>
+    <noscript>You need to enable JavaScript to run this app.</noscript>
+    <div id="root"></div>
+  </body>
+</html>

frontend/src/App.js

@@ -0,0 +1,1268 @@
+/**
+ * TheTruthSchool - Professional AI Assistant Interface
+ *
+ * Features:
+ * - Dark mode with elegant theme switching
+ * - Claude/ChatGPT-inspired professional design
+ * - Multi-domain RAG with TheTruthSchool branding
+ * - Smooth animations and modern UI
+ */
+
+import React, { useState, useEffect, useRef, useCallback } from 'react';
+import ReactMarkdown from 'react-markdown';
+import remarkGfm from 'remark-gfm';
+import rehypeHighlight from 'rehype-highlight';
+import 'highlight.js/styles/atom-one-dark.css';
+import {
+  Send,
+  Upload,
+  FileText,
+  CheckCircle,
+  XCircle,
+  Menu,
+  X,
+  Loader2,
+  Trash2,
+  FolderOpen,
+  RefreshCw,
+  Moon,
+  Sun,
+  Sparkles
+} from 'lucide-react';
+
+// =============================================================================
+// Domain Configurations
+// =============================================================================
+
+const DOMAIN_CONFIGS = {
+  medical: {
+    name: 'Medical & Healthcare',
+    description: 'Medical documents, research papers, clinical guidelines',
+    color: '#3b82f6',
+    fileTypes: ['.pdf', '.docx', '.xml', '.txt', '.doc', '.csv', '.xlsx'],
+    icon: '🏥'
+  },
+  legal: {
+    name: 'Legal & Compliance',
+    description: 'Legal documents, contracts, regulations, case law',
+    color: '#8b5cf6',
+    fileTypes: ['.pdf', '.docx', '.txt', '.doc', '.csv', '.xlsx'],
+    icon: '⚖️'
+  },
+  financial: {
+    name: 'Financial & Analytics',
+    description: 'Financial reports, analysis, market research',
+    color: '#10b981',
+    fileTypes: ['.pdf', '.xlsx', '.csv', '.json', '.xls'],
+    icon: '💰'
+  },
+  technical: {
+    name: 'Technical Documentation',
+    description: 'Technical docs, APIs, code, system architecture',
+    color: '#f97316',
+    fileTypes: ['.pdf', '.md', '.docx', '.json', '.txt', '.rst', '.csv', '.xlsx'],
+    icon: '⚙️'
+  },
+  academic: {
+    name: 'Academic Research',
+    description: 'Research papers, academic publications, studies',
+    color: '#6366f1',
+    fileTypes: ['.pdf', '.docx', '.tex', '.bib', '.txt', '.csv', '.xlsx'],
+    icon: '🎓'
+  }
+};
+
+const API_BASE_URL = process.env.REACT_APP_API_URL || 'http://localhost:8000';
+
+// =============================================================================
+// Main Component
+// =============================================================================
+
+export default function TheTruthSchoolAI() {
+  const getFromLocalStorage = (key, defaultValue) => {
+    try {
+      const item = window.localStorage.getItem(key);
+      return item ? JSON.parse(item) : defaultValue;
+    } catch (error) {
+      console.error(`Error reading localStorage key "${key}":`, error);
+      return defaultValue;
+    }
+  };
+
+  // State Management
+  const [darkMode, setDarkMode] = useState(() => getFromLocalStorage('darkMode', true));
+  const [selectedDomain, setSelectedDomain] = useState(() => getFromLocalStorage('selectedDomain', 'medical'));
+  const [currentView, setCurrentView] = useState('app');
+  const [processingDocs, setProcessingDocs] = useState(() => getFromLocalStorage('processingDocs', []));
+  const [processedDocs, setProcessedDocs] = useState([]);
+  const [query, setQuery] = useState('');
+  const [messages, setMessages] = useState(() => getFromLocalStorage('chatMessages', []));
+  const [isQuerying, setIsQuerying] = useState(false);
+  const [error, setError] = useState(null);
+  const [showUploadModal, setShowUploadModal] = useState(false);
+  const [isDragging, setIsDragging] = useState(false);
+  const [showSidebar, setShowSidebar] = useState(true);
+  const [enableWebSearch, setEnableWebSearch] = useState(() => getFromLocalStorage('enableWebSearch', false));
+  const [webSearchOnly, setWebSearchOnly] = useState(() => getFromLocalStorage('webSearchOnly', false));
+  const [urlInput, setUrlInput] = useState('');
+  const [uploadMode, setUploadMode] = useState('file');
+  const [fastMode, setFastMode] = useState(() => getFromLocalStorage('fastMode', false));
+  const [enableCache, setEnableCache] = useState(() => getFromLocalStorage('enableCache', true));
+  const [enableQueryImprovement, setEnableQueryImprovement] = useState(() => getFromLocalStorage('enableQueryImprovement', true));
+  const [enableVerification, setEnableVerification] = useState(() => getFromLocalStorage('enableVerification', true));
+  const [typingSpeed] = useState(0);
+
+  const messagesEndRef = useRef(null);
+  const fileInputRef = useRef(null);
+  const typingIntervalRef = useRef(null);
+
+  // Theme classes based on dark mode
+  const theme = {
+    bg: darkMode ? 'bg-[#0D0D0D]' : 'bg-white',
+    bgSecondary: darkMode ? 'bg-[#171717]' : 'bg-gray-50',
+    bgTertiary: darkMode ? 'bg-[#252525]' : 'bg-white',
+    text: darkMode ? 'text-gray-100' : 'text-gray-900',
+    textSecondary: darkMode ? 'text-gray-400' : 'text-gray-600',
+    textMuted: darkMode ? 'text-gray-500' : 'text-gray-500',
+    border: darkMode ? 'border-gray-800' : 'border-gray-200',
+    borderLight: darkMode ? 'border-gray-700' : 'border-gray-300',
+    hover: darkMode ? 'hover:bg-[#252525]' : 'hover:bg-gray-100',
+    active: darkMode ? 'bg-[#252525]' : 'bg-blue-50',
+    userMessage: darkMode ? 'bg-blue-600' : 'bg-blue-600',
+    assistantMessage: darkMode ? 'bg-[#252525]' : 'bg-gray-100',
+    input: darkMode ? 'bg-[#171717] border-gray-700 text-gray-100' : 'bg-white border-gray-300 text-gray-900',
+    button: darkMode ? 'bg-[#252525] hover:bg-[#2D2D2D]' : 'bg-gray-100 hover:bg-gray-200'
+  };
+
+  const scrollToBottom = () => {
+    messagesEndRef.current?.scrollIntoView({ behavior: 'smooth' });
+  };
+
+  useEffect(() => {
+    scrollToBottom();
+  }, [messages]);
+
+  // Persist to localStorage
+  useEffect(() => {
+    try {
+      window.localStorage.setItem('darkMode', JSON.stringify(darkMode));
+      window.localStorage.setItem('chatMessages', JSON.stringify(messages));
+      window.localStorage.setItem('selectedDomain', JSON.stringify(selectedDomain));
+      window.localStorage.setItem('processingDocs', JSON.stringify(processingDocs));
+      window.localStorage.setItem('enableWebSearch', JSON.stringify(enableWebSearch));
+      window.localStorage.setItem('webSearchOnly', JSON.stringify(webSearchOnly));
+      window.localStorage.setItem('fastMode', JSON.stringify(fastMode));
+      window.localStorage.setItem('enableCache', JSON.stringify(enableCache));
+      window.localStorage.setItem('enableQueryImprovement', JSON.stringify(enableQueryImprovement));
+      window.localStorage.setItem('enableVerification', JSON.stringify(enableVerification));
+    } catch (error) {
+      console.error('Error saving to localStorage:', error);
+    }
+  }, [darkMode, messages, selectedDomain, processingDocs, enableWebSearch, webSearchOnly, fastMode, enableCache, enableQueryImprovement, enableVerification]);
+
+  // Fetch processed documents
+  const fetchProcessedDocuments = useCallback(async () => {
+    try {
+      const response = await fetch(`${API_BASE_URL}/documents?domain=${selectedDomain}`);
+      if (response.ok) {
+        const data = await response.json();
+        const fetchedDocs = data.documents || [];
+        setProcessedDocs(prev => {
+          const fetchedIds = new Set(fetchedDocs.map(d => d.id));
+          const recentlyAdded = prev.filter(d => d.id && !fetchedIds.has(d.id));
+          return [...fetchedDocs, ...recentlyAdded];
+        });
+      }
+    } catch (err) {
+      console.error('Error fetching documents:', err);
+    }
+  }, [selectedDomain]);
+
+  // Check processing status
+  const checkProcessingStatus = useCallback(async () => {
+    const updatedProcessing = [];
+    for (const doc of processingDocs) {
+      try {
+        const response = await fetch(`${API_BASE_URL}/status/${doc.processingId}`);
+        if (response.ok) {
+          const status = await response.json();
+          if (status.status === 'completed') {
+            setProcessedDocs(prev => [...prev, { ...doc, id: doc.processingId, status: 'completed' }]);
+          } else if (status.status === 'failed') {
+            setError(`Processing failed for ${doc.name}: ${status.error}`);
+          } else {
+            updatedProcessing.push({ ...doc, status: status.status });
+          }
+        }
+      } catch (err) {
+        console.error('Error checking status:', err);
+      }
+    }
+    setProcessingDocs(updatedProcessing);
+  }, [processingDocs]);
+
+  useEffect(() => {
+    fetchProcessedDocuments();
+  }, [selectedDomain, fetchProcessedDocuments]);
+
+  useEffect(() => {
+    const interval = setInterval(() => {
+      if (processingDocs.length > 0) {
+        checkProcessingStatus();
+      }
+    }, 3000);
+    return () => clearInterval(interval);
+  }, [processingDocs, checkProcessingStatus]);
+
+  // API Functions
+  const handleFileUpload = async (files) => {
+    if (!files || files.length === 0) return;
+    setError(null);
+    const newProcessingDocs = [];
+
+    for (const file of files) {
+      const fileExt = '.' + file.name.split('.').pop().toLowerCase();
+      const allowedTypes = DOMAIN_CONFIGS[selectedDomain].fileTypes;
+
+      if (!allowedTypes.includes(fileExt)) {
+        setError(`File type ${fileExt} not supported for ${selectedDomain} domain.`);
+        continue;
+      }
+
+      const formData = new FormData();
+      formData.append('file', file);
+      formData.append('domain', selectedDomain);
+
+      try {
+        const response = await fetch(`${API_BASE_URL}/upload`, {
+          method: 'POST',
+          body: formData
+        });
+
+        const data = await response.json();
+        if (response.ok) {
+          newProcessingDocs.push({
+            name: file.name,
+            domain: selectedDomain,
+            processingId: data.processing_id,
+            status: 'processing',
+            uploadedAt: new Date().toISOString()
+          });
+        } else {
+          setError(data.detail || 'Upload failed');
+        }
+      } catch (err) {
+        console.error('Upload error:', err);
+        setError(`Failed to upload ${file.name}: ${err.message}`);
+      }
+    }
+
+    setProcessingDocs(prev => [...prev, ...newProcessingDocs]);
+    setShowUploadModal(false);
+  };
+
+  const handleUrlUpload = async () => {
+    if (!urlInput.trim()) {
+      setError('Please enter a valid URL');
+      return;
+    }
+
+    setError(null);
+
+    try {
+      const response = await fetch(`${API_BASE_URL}/upload-url`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({
+          url: urlInput,
+          domain: selectedDomain,
+          convert_to_markdown: true
+        })
+      });
+
+      const data = await response.json();
+      if (response.ok) {
+        setProcessingDocs(prev => [...prev, {
+          name: urlInput,
+          domain: selectedDomain,
+          processingId: data.processing_id,
+          status: 'processing',
+          uploadedAt: new Date().toISOString()
+        }]);
+        setUrlInput('');
+        setShowUploadModal(false);
+      } else {
+        setError(data.detail || 'URL upload failed');
+      }
+    } catch (err) {
+      console.error('URL upload error:', err);
+      setError(`Failed to upload URL: ${err.message}`);
+    }
+  };
+
+  const startTypingEffect = useCallback((messageIndex, targetTextRef, isStreamingRef) => {
+    if (typingIntervalRef.current) {
+      clearInterval(typingIntervalRef.current);
+    }
+
+    let displayedLength = 0;
+
+    typingIntervalRef.current = setInterval(() => {
+      const targetText = targetTextRef.current || '';
+      const isStillStreaming = isStreamingRef.current;
+
+      if (displayedLength < targetText.length) {
+        const charsToAdd = Math.max(1, Math.floor(typingSpeed / 10));
+        displayedLength = Math.min(displayedLength + charsToAdd, targetText.length);
+
+        setMessages(prev => {
+          const newMessages = [...prev];
+          if (newMessages[messageIndex]) {
+            newMessages[messageIndex] = {
+              ...newMessages[messageIndex],
+              content: targetText.substring(0, displayedLength)
+            };
+          }
+          return newMessages;
+        });
+      } else if (!isStillStreaming && displayedLength >= targetText.length) {
+        clearInterval(typingIntervalRef.current);
+        typingIntervalRef.current = null;
+      }
+    }, 30);
+  }, [typingSpeed]);
+
+  useEffect(() => {
+    return () => {
+      if (typingIntervalRef.current) {
+        clearInterval(typingIntervalRef.current);
+      }
+    };
+  }, []);
+
+  const handleQuery = async () => {
+    if (!query.trim()) return;
+
+    setError(null);
+    setIsQuerying(true);
+
+    const userMessage = { role: 'user', content: query };
+    setMessages(prev => [...prev, userMessage]);
+    const currentQuery = query;
+    setQuery('');
+
+    const assistantMessageIndex = messages.length + 1;
+    setMessages(prev => [...prev, {
+      role: 'assistant',
+      content: '',
+      streaming: true,
+      verification: null
+    }]);
+
+    const fullTextBufferRef = { current: '' };
+    const isStreamingRef = { current: true };
+    let typingStarted = false;
+
+    try {
+      const response = await fetch(`${API_BASE_URL}/query/stream`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({
+          query: currentQuery,
+          domain: selectedDomain,
+          enable_verification: true,
+          enable_web_search: enableWebSearch,
+          web_search_only: webSearchOnly,
+          fast_mode: fastMode,
+          enable_cache: enableCache,
+          enable_query_improvement: enableQueryImprovement,
+          enable_verification_check: enableVerification
+        })
+      });
+
+      if (!response.ok) {
+        throw new Error(`HTTP error! status: ${response.status}`);
+      }
+
+      const reader = response.body.getReader();
+      const decoder = new TextDecoder();
+      let buffer = '';
+
+      while (true) {
+        const { done, value } = await reader.read();
+
+        if (done) {
+          break;
+        }
+
+        buffer += decoder.decode(value, { stream: true });
+
+        const events = buffer.split('\n\n');
+        buffer = events.pop() || '';
+
+        for (const event of events) {
+          if (!event.trim()) continue;
+
+          const lines = event.split('\n');
+          let eventType = 'message';
+          let eventData = '';
+
+          for (const line of lines) {
+            if (line.startsWith('event:')) {
+              eventType = line.substring(6).trim();
+            } else if (line.startsWith('data:')) {
+              eventData = line.substring(5).trim();
+            }
+          }
+
+          if (eventData) {
+            const data = JSON.parse(eventData);
+
+            if (eventType === 'token') {
+              fullTextBufferRef.current += data.content;
+
+              if (!typingStarted && typingSpeed > 0) {
+                typingStarted = true;
+                startTypingEffect(assistantMessageIndex, fullTextBufferRef, isStreamingRef);
+              } else if (typingSpeed === 0) {
+                setMessages(prev => {
+                  const newMessages = [...prev];
+                  newMessages[assistantMessageIndex] = {
+                    ...newMessages[assistantMessageIndex],
+                    content: fullTextBufferRef.current
+                  };
+                  return newMessages;
+                });
+              }
+
+            } else if (eventType === 'verification') {
+              setMessages(prev => {
+                const newMessages = [...prev];
+                newMessages[assistantMessageIndex] = {
+                  ...newMessages[assistantMessageIndex],
+                  verification: data.content,
+                  streaming: false
+                };
+                return newMessages;
+              });
+
+            } else if (eventType === 'done') {
+              isStreamingRef.current = false;
+
+              setTimeout(() => {
+                if (typingIntervalRef.current) {
+                  clearInterval(typingIntervalRef.current);
+                  typingIntervalRef.current = null;
+                }
+
+                setMessages(prev => {
+                  const newMessages = [...prev];
+                  newMessages[assistantMessageIndex] = {
+                    ...newMessages[assistantMessageIndex],
+                    streaming: false,
+                    content: fullTextBufferRef.current
+                  };
+                  return newMessages;
+                });
+              }, typingSpeed === 0 ? 0 : 500);
+
+            } else if (eventType === 'error') {
+              const errorMessage = data.content.message || 'An error occurred';
+              const errorSuggestion = data.content.suggestion || '';
+              setError(errorSuggestion ? `${errorMessage}\n\n${errorSuggestion}` : errorMessage);
+
+              isStreamingRef.current = false;
+
+              if (typingIntervalRef.current) {
+                clearInterval(typingIntervalRef.current);
+                typingIntervalRef.current = null;
+              }
+
+              setMessages(prev => {
+                const newMessages = [...prev];
+                newMessages[assistantMessageIndex] = {
+                  ...newMessages[assistantMessageIndex],
+                  content: fullTextBufferRef.current || errorMessage,
+                  streaming: false,
+                  error: true
+                };
+                return newMessages;
+              });
+              break;
+            }
+          }
+        }
+      }
+
+    } catch (err) {
+      console.error('Query error:', err);
+      setError(`Query failed: ${err.message}`);
+
+      if (typingIntervalRef.current) {
+        clearInterval(typingIntervalRef.current);
+        typingIntervalRef.current = null;
+      }
+
+      setMessages(prev => {
+        const newMessages = [...prev];
+        if (newMessages[assistantMessageIndex]) {
+          newMessages[assistantMessageIndex] = {
+            ...newMessages[assistantMessageIndex],
+            content: newMessages[assistantMessageIndex].content || '[Error occurred]',
+            streaming: false,
+            error: true
+          };
+        }
+        return newMessages;
+      });
+    } finally {
+      setIsQuerying(false);
+    }
+  };
+
+  const handleKeyPress = (e) => {
+    if (e.key === 'Enter' && !e.shiftKey) {
+      e.preventDefault();
+      handleQuery();
+    }
+  };
+
+  const handleDeleteDocument = async (docId, docName) => {
+    if (!docId) {
+      console.error('Document ID is undefined');
+      setError('Cannot delete document: ID is missing');
+      return;
+    }
+
+    const confirmed = window.confirm(
+      `Are you sure you want to delete "${docName || 'this document'}"?\n\nThis action cannot be undone.`
+    );
+
+    if (!confirmed) {
+      return;
+    }
+
+    try {
+      const response = await fetch(`${API_BASE_URL}/documents/${docId}`, {
+        method: 'DELETE'
+      });
+
+      const data = await response.json();
+
+      if (response.ok && data.success) {
+        setProcessedDocs(prev => prev.filter(doc => doc.id !== docId));
+        await fetchProcessedDocuments();
+      } else {
+        const errorMsg = data.message || data.detail || 'Failed to delete document';
+        setError(errorMsg);
+      }
+    } catch (err) {
+      console.error('Error deleting document:', err);
+      setError('Failed to delete document: ' + err.message);
+    }
+  };
+
+  const handleDragOver = (e) => {
+    e.preventDefault();
+    setIsDragging(true);
+  };
+
+  const handleDragLeave = (e) => {
+    e.preventDefault();
+    setIsDragging(false);
+  };
+
+  const handleDrop = (e) => {
+    e.preventDefault();
+    setIsDragging(false);
+    handleFileUpload(e.dataTransfer.files);
+  };
+
+  // =============================================================================
+  // Render Functions
+  // =============================================================================
+
+  const renderNavigation = () => (
+    <nav className={`${theme.bgTertiary} ${theme.border} border-b px-6 py-3`}>
+      <div className="flex items-center justify-between max-w-7xl mx-auto">
+        <div className="flex items-center space-x-8">
+          <div className="flex items-center space-x-3">
+            <div className="flex items-center space-x-2">
+              <div className={`w-8 h-8 ${darkMode ? 'bg-gradient-to-br from-purple-500 to-blue-500' : 'bg-gradient-to-br from-blue-600 to-purple-600'} rounded-lg flex items-center justify-center`}>
+                <Sparkles className="w-5 h-5 text-white" />
+              </div>
+              <h1 className={`text-xl font-bold ${theme.text}`}>TheTruthSchool</h1>
+            </div>
+            <span className={`text-sm ${theme.textMuted}`}>/ {DOMAIN_CONFIGS[selectedDomain].name}</span>
+          </div>
+
+          <div className="flex items-center space-x-1">
+            <button
+              onClick={() => setCurrentView('app')}
+              className={`px-4 py-2 text-sm font-medium rounded-md transition-colors ${
+                currentView === 'app'
+                  ? `${darkMode ? 'text-blue-400 bg-blue-900/30' : 'text-blue-600 bg-blue-50'}`
+                  : `${theme.textSecondary} ${theme.hover}`
+              }`}
+            >
+              Chat
+            </button>
+            <button
+              onClick={() => setCurrentView('files')}
+              className={`px-4 py-2 text-sm font-medium rounded-md transition-colors ${
+                currentView === 'files'
+                  ? `${darkMode ? 'text-blue-400 bg-blue-900/30' : 'text-blue-600 bg-blue-50'}`
+                  : `${theme.textSecondary} ${theme.hover}`
+              }`}
+            >
+              Files
+            </button>
+            <button
+              onClick={() => setCurrentView('settings')}
+              className={`px-4 py-2 text-sm font-medium rounded-md transition-colors ${
+                currentView === 'settings'
+                  ? `${darkMode ? 'text-blue-400 bg-blue-900/30' : 'text-blue-600 bg-blue-50'}`
+                  : `${theme.textSecondary} ${theme.hover}`
+              }`}
+            >
+              Settings
+            </button>
+          </div>
+        </div>
+
+        <div className="flex items-center space-x-2">
+          <button
+            onClick={() => setDarkMode(!darkMode)}
+            className={`p-2 ${theme.textSecondary} ${theme.hover} rounded-md transition-colors`}
+          >
+            {darkMode ? <Sun className="w-5 h-5" /> : <Moon className="w-5 h-5" />}
+          </button>
+          <button
+            onClick={() => setShowSidebar(!showSidebar)}
+            className={`p-2 ${theme.textSecondary} ${theme.hover} rounded-md transition-colors`}
+          >
+            {showSidebar ? <X className="w-5 h-5" /> : <Menu className="w-5 h-5" />}
+          </button>
+        </div>
+      </div>
+    </nav>
+  );
+
+  const renderSidebar = () => (
+    <div className={`${showSidebar ? 'w-64' : 'w-0'} transition-all duration-300 ${theme.bgSecondary} ${theme.border} border-r overflow-hidden`}>
+      <div className="p-4 space-y-4">
+        <div>
+          <h3 className={`text-xs font-semibold ${theme.textMuted} uppercase mb-3`}>Domains</h3>
+          <div className="space-y-1">
+            {Object.entries(DOMAIN_CONFIGS).map(([key, config]) => (
+              <button
+                key={key}
+                onClick={() => setSelectedDomain(key)}
+                className={`w-full flex items-center space-x-3 px-3 py-2 rounded-lg text-sm transition-colors ${
+                  selectedDomain === key
+                    ? `${darkMode ? 'bg-blue-900/30 text-blue-400' : 'bg-blue-50 text-blue-700'} font-medium`
+                    : `${theme.textSecondary} ${theme.hover}`
+                }`}
+              >
+                <span className="text-lg">{config.icon}</span>
+                <span className="flex-1 text-left truncate">{config.name}</span>
+              </button>
+            ))}
+          </div>
+        </div>
+
+        {processingDocs.length > 0 && (
+          <div>
+            <h3 className={`text-xs font-semibold ${theme.textMuted} uppercase mb-3`}>Processing</h3>
+            <div className="space-y-2">
+              {processingDocs.map((doc, idx) => (
+                <div key={idx} className={`flex items-center space-x-2 px-3 py-2 ${darkMode ? 'bg-yellow-900/20' : 'bg-yellow-50'} rounded-lg`}>
+                  <Loader2 className={`w-4 h-4 ${darkMode ? 'text-yellow-400' : 'text-yellow-600'} animate-spin`} />
+                  <span className={`text-xs ${darkMode ? 'text-yellow-300' : 'text-yellow-800'} truncate flex-1`}>{doc.name}</span>
+                </div>
+              ))}
+            </div>
+          </div>
+        )}
+
+        {processedDocs.length > 0 && (
+          <div>
+            <h3 className={`text-xs font-semibold ${theme.textMuted} uppercase mb-3`}>
+              Documents ({processedDocs.length})
+            </h3>
+            <div className="space-y-1 max-h-64 overflow-y-auto">
+              {processedDocs.map((doc, idx) => (
+                <div key={idx} className={`flex items-center space-x-2 px-3 py-2 ${theme.bgTertiary} rounded-lg ${theme.border} border group`}>
+                  <FileText className={`w-4 h-4 ${theme.textMuted}`} />
+                  <span className={`text-xs ${theme.textSecondary} truncate flex-1`}>{doc.name || `Document ${idx + 1}`}</span>
+                  <button
+                    onClick={() => handleDeleteDocument(doc.id, doc.name)}
+                    className="opacity-0 group-hover:opacity-100 transition-opacity"
+                  >
+                    <Trash2 className={`w-3 h-3 ${theme.textMuted} hover:text-red-600`} />
+                  </button>
+                </div>
+              ))}
+            </div>
+          </div>
+        )}
+
+        {messages.length > 0 && (
+          <div className={`pt-4 ${theme.border} border-t`}>
+            <button
+              onClick={() => {
+                if (window.confirm('Clear all chat history? This cannot be undone.')) {
+                  setMessages([]);
+                  window.localStorage.removeItem('chatMessages');
+                }
+              }}
+              className={`w-full flex items-center justify-center space-x-2 px-3 py-2 text-sm text-red-500 hover:${darkMode ? 'bg-red-900/20' : 'bg-red-50'} rounded-lg transition-colors`}
+            >
+              <Trash2 className="w-4 h-4" />
+              <span>Clear Chat</span>
+            </button>
+          </div>
+        )}
+      </div>
+    </div>
+  );
+
+  const renderAppView = () => (
+    <div className={`flex-1 flex flex-col ${theme.bg}`}>
+      {messages.length === 0 ? (
+        <div className="flex-1 flex flex-col items-center justify-center px-4">
+          <div className="text-center max-w-2xl">
+            <div className={`w-20 h-20 ${darkMode ? 'bg-gradient-to-br from-purple-500 to-blue-500' : 'bg-gradient-to-br from-blue-600 to-purple-600'} rounded-2xl flex items-center justify-center mx-auto mb-6 shadow-lg`}>
+              <Sparkles className="w-10 h-10 text-white" />
+            </div>
+            <h2 className={`text-4xl font-bold ${theme.text} mb-3`}>TheTruthSchool AI</h2>
+            <p className={`${theme.textSecondary} mb-8 text-lg`}>
+              Your intelligent assistant for document analysis and knowledge discovery
+            </p>
+
+            <div className="grid grid-cols-3 gap-4 text-left">
+              <div className={`p-5 ${theme.bgSecondary} rounded-xl ${theme.border} border`}>
+                <div className="text-3xl mb-3">📚</div>
+                <h3 className={`font-semibold ${theme.text} mb-2`}>Smart Upload</h3>
+                <p className={`text-sm ${theme.textSecondary}`}>Process PDFs, documents, and web content</p>
+              </div>
+              <div className={`p-5 ${theme.bgSecondary} rounded-xl ${theme.border} border`}>
+                <div className="text-3xl mb-3">🧠</div>
+                <h3 className={`font-semibold ${theme.text} mb-2`}>Deep Understanding</h3>
+                <p className={`text-sm ${theme.textSecondary}`}>Advanced RAG with knowledge graphs</p>
+              </div>
+              <div className={`p-5 ${theme.bgSecondary} rounded-xl ${theme.border} border`}>
+                <div className="text-3xl mb-3">✨</div>
+                <h3 className={`font-semibold ${theme.text} mb-2`}>Multi-Domain</h3>
+                <p className={`text-sm ${theme.textSecondary}`}>Optimized for healthcare, legal, finance & more</p>
+              </div>
+            </div>
+          </div>
+        </div>
+      ) : (
+        <div className="flex-1 overflow-y-auto px-4 py-6">
+          <div className="max-w-3xl mx-auto space-y-6">
+            {messages.map((msg, idx) => (
+              <div key={idx} className={`flex ${msg.role === 'user' ? 'justify-end' : 'justify-start'}`}>
+                <div className={`max-w-[80%] ${msg.role === 'user' ? 'bg-blue-600 text-white' : `${theme.assistantMessage} ${theme.text}`} rounded-2xl px-5 py-4 shadow-sm`}>
+                  {msg.role === 'user' ? (
+                    <p className="text-sm whitespace-pre-wrap">{msg.content}</p>
+                  ) : (
+                    <div className={`text-sm prose prose-sm max-w-none ${darkMode ? 'prose-invert' : ''}`}>
+                      <ReactMarkdown
+                        remarkPlugins={[remarkGfm]}
+                        rehypePlugins={[rehypeHighlight]}
+                        components={{
+                          code({ node, inline, className, children, ...props }) {
+                            return inline ? (
+                              <code className={`${darkMode ? 'bg-gray-700 text-gray-100' : 'bg-gray-200 text-gray-800'} px-1.5 py-0.5 rounded text-xs font-mono`} {...props}>
+                                {children}
+                              </code>
+                            ) : (
+                              <code className={className} {...props}>
+                                {children}
+                              </code>
+                            );
+                          },
+                          a({ node, children, ...props }) {
+                            return (
+                              <a className={`${darkMode ? 'text-blue-400' : 'text-blue-600'} hover:underline`} target="_blank" rel="noopener noreferrer" {...props}>
+                                {children}
+                              </a>
+                            );
+                          },
+                          table: ({ node, ...props }) => (
+                            <div className="overflow-x-auto my-4">
+                              <table className={`min-w-full divide-y ${darkMode ? 'divide-gray-700 border-gray-700' : 'divide-gray-300 border-gray-300'} border rounded-lg`} {...props} />
+                            </div>
+                          ),
+                          thead: ({ node, ...props }) => (
+                            <thead className={darkMode ? 'bg-gray-800' : 'bg-gray-100'} {...props} />
+                          ),
+                          tbody: ({ node, ...props }) => (
+                            <tbody className={`divide-y ${darkMode ? 'divide-gray-700 bg-gray-900' : 'divide-gray-200 bg-white'}`} {...props} />
+                          ),
+                          th: ({ node, ...props }) => (
+                            <th className={`px-4 py-3 text-left text-xs font-bold uppercase tracking-wider ${darkMode ? 'text-gray-300 border-gray-700' : 'text-gray-700 border-gray-300'} border-r last:border-r-0`} {...props} />
+                          ),
+                          td: ({ node, ...props }) => (
+                            <td className={`px-4 py-3 text-sm ${darkMode ? 'text-gray-300 border-gray-700' : 'text-gray-900 border-gray-200'} border-r last:border-r-0`} {...props} />
+                          ),
+                          tr: ({ node, ...props }) => (
+                            <tr className={darkMode ? 'hover:bg-gray-800' : 'hover:bg-gray-50'} {...props} />
+                          ),
+                        }}
+                      >
+                        {msg.content}
+                      </ReactMarkdown>
+                    </div>
+                  )}
+                  {msg.streaming && msg.role === 'assistant' && (
+                    <div className={`flex items-center space-x-1 ${theme.textMuted} text-sm mt-2`}>
+                      <span>Thinking</span>
+                      <span className="animate-pulse">...</span>
+                    </div>
+                  )}
+                </div>
+              </div>
+            ))}
+            <div ref={messagesEndRef} />
+          </div>
+        </div>
+      )}
+
+      {/* Bottom Input Bar */}
+      <div className={`${theme.border} border-t ${theme.bgTertiary} px-4 py-4`}>
+        <div className="max-w-3xl mx-auto">
+          <div className="flex items-end space-x-3">
+            <button
+              onClick={() => setShowUploadModal(true)}
+              className={`px-4 py-3 ${darkMode ? 'bg-blue-600 hover:bg-blue-700' : 'bg-blue-600 hover:bg-blue-700'} text-white rounded-xl transition-colors flex items-center space-x-2`}
+            >
+              <Upload className="w-4 h-4" />
+              <span className="text-sm font-medium">Upload</span>
+            </button>
+
+            <textarea
+              value={query}
+              onChange={(e) => setQuery(e.target.value)}
+              onKeyDown={handleKeyPress}
+              placeholder="Message TheTruthSchool..."
+              className={`flex-1 px-4 py-3 ${theme.input} rounded-xl focus:outline-none focus:ring-2 focus:ring-blue-500 resize-none`}
+              disabled={isQuerying}
+              rows={1}
+              style={{ minHeight: '48px', maxHeight: '200px' }}
+            />
+
+            <button
+              onClick={handleQuery}
+              disabled={isQuerying || !query.trim()}
+              className={`p-3 ${darkMode ? 'bg-blue-600 hover:bg-blue-700' : 'bg-blue-600 hover:bg-blue-700'} text-white rounded-xl transition-colors disabled:opacity-50 disabled:cursor-not-allowed`}
+            >
+              <Send className="w-5 h-5" />
+            </button>
+          </div>
+
+          <div className="flex items-center justify-center space-x-6 mt-3">
+            <label className="flex items-center space-x-2 cursor-pointer">
+              <input
+                type="checkbox"
+                checked={enableWebSearch}
+                onChange={(e) => {
+                  setEnableWebSearch(e.target.checked);
+                  if (e.target.checked && webSearchOnly) {
+                    setWebSearchOnly(false);
+                  }
+                }}
+                className="w-4 h-4 text-blue-600 rounded focus:ring-blue-500"
+              />
+              <span className={`text-sm ${theme.textSecondary}`}>Enhance with Web Search</span>
+            </label>
+            <label className="flex items-center space-x-2 cursor-pointer">
+              <input
+                type="checkbox"
+                checked={webSearchOnly}
+                onChange={(e) => {
+                  setWebSearchOnly(e.target.checked);
+                  if (e.target.checked) {
+                    setEnableWebSearch(false);
+                  }
+                }}
+                className="w-4 h-4 text-blue-600 rounded focus:ring-blue-500"
+              />
+              <span className={`text-sm ${theme.textSecondary}`}>Web Search Only</span>
+            </label>
+          </div>
+
+          <p className={`text-xs ${theme.textMuted} mt-2 text-center`}>
+            Press Enter to send • Shift+Enter for new line
+          </p>
+        </div>
+      </div>
+    </div>
+  );
+
+  const renderFilesView = () => (
+    <div className={`flex-1 overflow-y-auto p-6 ${theme.bg}`}>
+      <div className="max-w-5xl mx-auto">
+        <div className="flex items-center justify-between mb-6">
+          <div>
+            <h2 className={`text-2xl font-bold ${theme.text}`}>Document Management</h2>
+            <p className={theme.textSecondary}>Manage your uploaded and processed documents</p>
+          </div>
+          <div className="flex space-x-3">
+            <button
+              onClick={fetchProcessedDocuments}
+              className={`flex items-center space-x-2 px-4 py-2 ${theme.button} ${theme.text} rounded-lg transition-colors`}
+            >
+              <RefreshCw className="w-4 h-4" />
+              <span>Refresh</span>
+            </button>
+            <button
+              onClick={() => setShowUploadModal(true)}
+              className="flex items-center space-x-2 px-4 py-2 bg-blue-600 text-white rounded-lg hover:bg-blue-700 transition-colors"
+            >
+              <Upload className="w-4 h-4" />
+              <span>Upload Documents</span>
+            </button>
+          </div>
+        </div>
+
+        {processingDocs.length > 0 && (
+          <div className="mb-6">
+            <h3 className={`text-lg font-semibold ${theme.text} mb-3`}>Processing Documents</h3>
+            <div className="grid grid-cols-1 md:grid-cols-2 gap-4">
+              {processingDocs.map((doc, idx) => (
+                <div key={idx} className={`flex items-center space-x-4 p-4 ${darkMode ? 'bg-yellow-900/20 border-yellow-800' : 'bg-yellow-50 border-yellow-200'} border rounded-lg`}>
+                  <Loader2 className={`w-8 h-8 ${darkMode ? 'text-yellow-400' : 'text-yellow-600'} animate-spin`} />
+                  <div className="flex-1">
+                    <p className={`font-medium ${theme.text}`}>{doc.name}</p>
+                    <p className={`text-sm ${theme.textSecondary}`}>Processing...</p>
+                  </div>
+                </div>
+              ))}
+            </div>
+          </div>
+        )}
+
+        <div>
+          <h3 className={`text-lg font-semibold ${theme.text} mb-3`}>
+            Processed Documents ({processedDocs.length})
+          </h3>
+          {processedDocs.length === 0 ? (
+            <div className={`text-center py-12 ${theme.bgSecondary} rounded-lg`}>
+              <FolderOpen className={`w-16 h-16 ${theme.textMuted} mx-auto mb-4`} />
+              <p className={theme.textSecondary}>No documents processed yet</p>
+              <button
+                onClick={() => setShowUploadModal(true)}
+                className="mt-4 px-6 py-2 bg-blue-600 text-white rounded-lg hover:bg-blue-700 transition-colors"
+              >
+                Upload Your First Document
+              </button>
+            </div>
+          ) : (
+            <div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4">
+              {processedDocs.map((doc, idx) => (
+                <div key={idx} className={`p-4 ${theme.bgTertiary} ${theme.border} border rounded-lg hover:shadow-lg transition-all group`}>
+                  <div className="flex items-start justify-between mb-3">
+                    <FileText className="w-8 h-8 text-blue-600" />
+                    <button
+                      onClick={() => handleDeleteDocument(doc.id, doc.name)}
+                      className="opacity-0 group-hover:opacity-100 transition-opacity p-1 hover:bg-gray-700 rounded"
+                    >
+                      <Trash2 className={`w-4 h-4 ${theme.textMuted} hover:text-red-500`} />
+                    </button>
+                  </div>
+                  <p className={`font-medium ${theme.text} mb-1 truncate`} title={doc.name}>{doc.name || `Document ${idx + 1}`}</p>
+                  <p className={`text-sm ${theme.textSecondary} mb-2`}>{DOMAIN_CONFIGS[doc.domain]?.name || selectedDomain}</p>
+                  <div className="flex items-center space-x-2">
+                    <CheckCircle className="w-4 h-4 text-green-500" />
+                    <span className={`text-xs ${theme.textSecondary}`}>Processed</span>
+                  </div>
+                </div>
+              ))}
+            </div>
+          )}
+        </div>
+      </div>
+    </div>
+  );
+
+  const renderSettingsView = () => (
+    <div className={`flex-1 overflow-y-auto p-6 ${theme.bg}`}>
+      <div className="max-w-3xl mx-auto">
+        <h2 className={`text-2xl font-bold ${theme.text} mb-6`}>Settings</h2>
+
+        <div className="space-y-6">
+          <div className={`${theme.bgTertiary} ${theme.border} border rounded-lg p-6`}>
+            <h3 className={`text-lg font-semibold ${theme.text} mb-4`}>Appearance</h3>
+            <div className="flex items-center justify-between">
+              <div>
+                <label className={`block text-sm font-medium ${theme.text}`}>Theme</label>
+                <p className={`text-xs ${theme.textSecondary} mt-1`}>Choose your preferred interface theme</p>
+              </div>
+              <button
+                onClick={() => setDarkMode(!darkMode)}
+                className={`px-4 py-2 ${theme.button} ${theme.text} rounded-lg transition-colors flex items-center space-x-2`}
+              >
+                {darkMode ? (
+                  <>
+                    <Sun className="w-4 h-4" />
+                    <span>Light Mode</span>
+                  </>
+                ) : (
+                  <>
+                    <Moon className="w-4 h-4" />
+                    <span>Dark Mode</span>
+                  </>
+                )}
+              </button>
+            </div>
+          </div>
+
+          <div className={`${theme.bgTertiary} ${theme.border} border rounded-lg p-6`}>
+            <h3 className={`text-lg font-semibold ${theme.text} mb-4`}>Domain Configuration</h3>
+            <div className="space-y-3">
+              <div>
+                <label className={`block text-sm font-medium ${theme.text} mb-2`}>Current Domain</label>
+                <select
+                  value={selectedDomain}
+                  onChange={(e) => setSelectedDomain(e.target.value)}
+                  className={`w-full px-4 py-2 ${theme.input} rounded-lg focus:outline-none focus:ring-2 focus:ring-blue-500`}
+                >
+                  {Object.entries(DOMAIN_CONFIGS).map(([key, config]) => (
+                    <option key={key} value={key}>{config.name}</option>
+                  ))}
+                </select>
+              </div>
+            </div>
+          </div>
+
+          <div className={`${theme.bgTertiary} ${theme.border} border rounded-lg p-6`}>
+            <h3 className={`text-lg font-semibold ${theme.text} mb-4`}>Performance Settings</h3>
+            <div className="space-y-4">
+              <div className="flex items-start space-x-3">
+                <input
+                  type="checkbox"
+                  id="fastMode"
+                  checked={fastMode}
+                  onChange={(e) => setFastMode(e.target.checked)}
+                  className="w-5 h-5 text-blue-600 rounded focus:ring-blue-500 mt-0.5"
+                />
+                <div className="flex-1">
+                  <label htmlFor="fastMode" className={`block text-sm font-medium ${theme.text} cursor-pointer`}>
+                    Fast Mode
+                  </label>
+                  <p className={`text-xs ${theme.textSecondary} mt-1`}>
+                    Use optimized parameters for 2-3x faster queries
+                  </p>
+                </div>
+              </div>
+
+              <div className="flex items-start space-x-3">
+                <input
+                  type="checkbox"
+                  id="enableCache"
+                  checked={enableCache}
+                  onChange={(e) => setEnableCache(e.target.checked)}
+                  className="w-5 h-5 text-blue-600 rounded focus:ring-blue-500 mt-0.5"
+                />
+                <div className="flex-1">
+                  <label htmlFor="enableCache" className={`block text-sm font-medium ${theme.text} cursor-pointer`}>
+                    Enable Query Caching
+                  </label>
+                  <p className={`text-xs ${theme.textSecondary} mt-1`}>
+                    Cache results for faster repeated queries
+                  </p>
+                </div>
+              </div>
+
+              <div className="flex items-start space-x-3">
+                <input
+                  type="checkbox"
+                  id="enableQueryImprovement"
+                  checked={enableQueryImprovement}
+                  onChange={(e) => setEnableQueryImprovement(e.target.checked)}
+                  className="w-5 h-5 text-blue-600 rounded focus:ring-blue-500 mt-0.5"
+                />
+                <div className="flex-1">
+                  <label htmlFor="enableQueryImprovement" className={`block text-sm font-medium ${theme.text} cursor-pointer`}>
+                    Enable Query Improvement
+                  </label>
+                  <p className={`text-xs ${theme.textSecondary} mt-1`}>
+                    Automatically enhance queries for better results
+                  </p>
+                </div>
+              </div>
+
+              <div className="flex items-start space-x-3">
+                <input
+                  type="checkbox"
+                  id="enableVerification"
+                  checked={enableVerification}
+                  onChange={(e) => setEnableVerification(e.target.checked)}
+                  className="w-5 h-5 text-blue-600 rounded focus:ring-blue-500 mt-0.5"
+                />
+                <div className="flex-1">
+                  <label htmlFor="enableVerification" className={`block text-sm font-medium ${theme.text} cursor-pointer`}>
+                    Enable Answer Verification
+                  </label>
+                  <p className={`text-xs ${theme.textSecondary} mt-1`}>
+                    Verify answer quality and accuracy with dual-LLM
+                  </p>
+                </div>
+              </div>
+            </div>
+          </div>
+        </div>
+      </div>
+    </div>
+  );
+
+  const renderUploadModal = () => {
+    if (!showUploadModal) return null;
+
+    return (
+      <div className="fixed inset-0 bg-black bg-opacity-70 flex items-center justify-center z-50 p-4 backdrop-blur-sm">
+        <div className={`${theme.bgTertiary} rounded-2xl max-w-2xl w-full p-6 shadow-2xl`}>
+          <div className="flex items-center justify-between mb-6">
+            <h2 className={`text-2xl font-bold ${theme.text}`}>Upload Documents</h2>
+            <button
+              onClick={() => {
+                setShowUploadModal(false);
+                setUploadMode('file');
+                setUrlInput('');
+              }}
+              className={`p-2 ${theme.hover} rounded-lg`}
+            >
+              <X className={`w-5 h-5 ${theme.textSecondary}`} />
+            </button>
+          </div>
+
+          <div className="flex items-center space-x-2 mb-6">
+            <button
+              onClick={() => setUploadMode('file')}
+              className={`flex-1 px-4 py-2 rounded-lg font-medium transition-colors ${
+                uploadMode === 'file'
+                  ? 'bg-blue-600 text-white'
+                  : `${theme.button} ${theme.text}`
+              }`}
+            >
+              Upload File
+            </button>
+            <button
+              onClick={() => setUploadMode('url')}
+              className={`flex-1 px-4 py-2 rounded-lg font-medium transition-colors ${
+                uploadMode === 'url'
+                  ? 'bg-blue-600 text-white'
+                  : `${theme.button} ${theme.text}`
+              }`}
+            >
+              Upload from URL
+            </button>
+          </div>
+
+          {uploadMode === 'file' ? (
+            <div
+              onDragOver={handleDragOver}
+              onDragLeave={handleDragLeave}
+              onDrop={handleDrop}
+              className={`border-2 border-dashed rounded-xl p-12 text-center transition-colors ${
+                isDragging
+                  ? 'border-blue-500 bg-blue-500/10'
+                  : `${theme.borderLight}`
+              }`}
+            >
+              <Upload className={`w-16 h-16 ${theme.textMuted} mx-auto mb-4`} />
+              <h3 className={`text-lg font-semibold ${theme.text} mb-2`}>
+                Drop files here or click to browse
+              </h3>
+              <p className={`${theme.textSecondary} mb-4`}>
+                Supported: {DOMAIN_CONFIGS[selectedDomain].fileTypes.join(', ')}
+              </p>
+              <input
+                ref={fileInputRef}
+                type="file"
+                multiple
+                accept={DOMAIN_CONFIGS[selectedDomain].fileTypes.join(',')}
+                onChange={(e) => handleFileUpload(e.target.files)}
+                className="hidden"
+              />
+              <button
+                onClick={() => fileInputRef.current?.click()}
+                className="px-6 py-2 bg-blue-600 text-white rounded-lg hover:bg-blue-700 transition-colors"
+              >
+                Select Files
+              </button>
+            </div>
+          ) : (
+            <div className="space-y-4">
+              <div>
+                <label className={`block text-sm font-medium ${theme.text} mb-2`}>
+                  Enter URL to fetch and process
+                </label>
+                <input
+                  type="url"
+                  value={urlInput}
+                  onChange={(e) => setUrlInput(e.target.value)}
+                  placeholder="https://example.com/document.pdf"
+                  className={`w-full px-4 py-3 ${theme.input} rounded-lg focus:outline-none focus:ring-2 focus:ring-blue-500`}
+                  onKeyDown={(e) => {
+                    if (e.key === 'Enter') {
+                      handleUrlUpload();
+                    }
+                  }}
+                />
+              </div>
+              <button
+                onClick={handleUrlUpload}
+                disabled={!urlInput.trim()}
+                className="w-full px-6 py-3 bg-blue-600 text-white rounded-lg hover:bg-blue-700 transition-colors disabled:opacity-50 disabled:cursor-not-allowed"
+              >
+                Fetch and Process URL
+              </button>
+            </div>
+          )}
+        </div>
+      </div>
+    );
+  };
+
+  const renderError = () => {
+    if (!error) return null;
+
+    return (
+      <div className={`fixed bottom-4 right-4 ${darkMode ? 'bg-red-900/90 border-red-800' : 'bg-red-50 border-red-200'} border rounded-lg p-4 max-w-md shadow-2xl backdrop-blur-sm`}>
+        <div className="flex items-start space-x-3">
+          <XCircle className="w-5 h-5 text-red-500 flex-shrink-0 mt-0.5" />
+          <div className="flex-1">
+            <p className={`text-sm ${darkMode ? 'text-red-200' : 'text-red-800'}`}>{error}</p>
+          </div>
+          <button
+            onClick={() => setError(null)}
+            className="text-red-500 hover:text-red-600"
+          >
+            <X className="w-4 h-4" />
+          </button>
+        </div>
+      </div>
+    );
+  };
+
+  return (
+    <div className={`h-screen flex flex-col ${theme.bg}`}>
+      {renderNavigation()}
+
+      <div className="flex-1 flex overflow-hidden">
+        {renderSidebar()}
+
+        {currentView === 'app' && renderAppView()}
+        {currentView === 'files' && renderFilesView()}
+        {currentView === 'settings' && renderSettingsView()}
+      </div>
+
+      {renderUploadModal()}
+      {renderError()}
+    </div>
+  );
+}

frontend/src/index.css

@@ -0,0 +1,79 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;
+
+@keyframes blink {
+  0%, 80%, 100% {
+    opacity: 0;
+  }
+  40% {
+    opacity: 1;
+  }
+}
+.animate-blink {
+  animation: blink 1.4s infinite;
+  animation-fill-mode: both;
+}
+
+* {
+  margin: 0;
+  padding: 0;
+  box-sizing: border-box;
+}
+
+body {
+  font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen',
+    'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue',
+    sans-serif;
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+}
+
+code {
+  font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New',
+    monospace;
+}
+
+
+/* Custom scrollbar for webkit browsers */
+::-webkit-scrollbar {
+  width: 8px;
+  height: 8px;
+}
+
+::-webkit-scrollbar-track {
+  background: #1f2937;
+  border-radius: 4px;
+}
+
+::-webkit-scrollbar-thumb {
+  background: #4b5563;
+  border-radius: 4px;
+}
+
+::-webkit-scrollbar-thumb:hover {
+  background: #6b7280;
+}
+
+/* Smooth transitions */
+* {
+  transition-property: background-color, border-color, color, fill, stroke;
+  transition-timing-function: cubic-bezier(0.4, 0, 0.2, 1);
+  transition-duration: 150ms;
+}
+
+/* Animations */
+@keyframes fadeIn {
+  from {
+    opacity: 0;
+    transform: translateY(10px);
+  }
+  to {
+    opacity: 1;
+    transform: translateY(0);
+  }
+}
+
+.animate-fadeIn {
+  animation: fadeIn 0.3s ease-out;
+}

frontend/src/index.js

@@ -0,0 +1,11 @@
+import React from 'react';
+import ReactDOM from 'react-dom/client';
+import './index.css';
+import App from './App';
+
+const root = ReactDOM.createRoot(document.getElementById('root'));
+root.render(
+  <React.StrictMode>
+    <App />
+  </React.StrictMode>
+);

frontend/tailwind.config.js

@@ -0,0 +1,27 @@
+/** @type {import('tailwindcss').Config} */
+module.exports = {
+  content: [
+    "./src/**/*.{js,jsx,ts,tsx}",
+  ],
+  theme: {
+    extend: {
+      colors: {
+        gray: {
+          650: '#4b5563',
+          750: '#2d3748',
+          850: '#1a202c',
+        }
+      },
+      animation: {
+        'fadeIn': 'fadeIn 0.3s ease-out',
+      },
+      keyframes: {
+        fadeIn: {
+          '0%': { opacity: '0', transform: 'translateY(10px)' },
+          '100%': { opacity: '1', transform: 'translateY(0)' },
+        }
+      }
+    },
+  },
+  plugins: [],
+}

rag_anything_smaranika/.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,61 @@
+name: Bug Report
+description: File a bug report
+title: "[Bug]:"
+labels: ["bug", "triage"]
+
+body:
+  - type: checkboxes
+    id: existingcheck
+    attributes:
+      label: Do you need to file an issue?
+      description: Please help us manage our time by avoiding duplicates and common bugs with the steps below.
+      options:
+        - label: I have searched the existing issues and this bug is not already filed.
+        - label: I believe this is a legitimate bug, not just a question or feature request.
+  - type: textarea
+    id: description
+    attributes:
+      label: Describe the bug
+      description: A clear and concise description of what the bug is.
+      placeholder: What went wrong?
+  - type: textarea
+    id: reproduce
+    attributes:
+      label: Steps to reproduce
+      description: Steps to reproduce the behavior.
+      placeholder: How can we replicate the issue?
+  - type: textarea
+    id: expected_behavior
+    attributes:
+      label: Expected Behavior
+      description: A clear and concise description of what you expected to happen.
+      placeholder: What should have happened?
+  - type: textarea
+    id: configused
+    attributes:
+      label: LightRAG Config Used
+      description: The LightRAG configuration used for the run.
+      placeholder: The settings content or LightRAG configuration
+      value: |
+        # Paste your config here
+  - type: textarea
+    id: screenshotslogs
+    attributes:
+      label: Logs and screenshots
+      description: If applicable, add screenshots and logs to help explain your problem.
+      placeholder: Add logs and screenshots here
+  - type: textarea
+    id: additional_information
+    attributes:
+      label: Additional Information
+      description: |
+        - LightRAG Version: e.g., v0.1.1
+        - Operating System: e.g., Windows 10, Ubuntu 20.04
+        - Python Version: e.g., 3.8
+        - Related Issues: e.g., #1
+        - Any other relevant information.
+      value: |
+        - LightRAG Version:
+        - Operating System:
+        - Python Version:
+        - Related Issues:

rag_anything_smaranika/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1 @@
+blank_issues_enabled: false

rag_anything_smaranika/.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,26 @@
+name: Feature Request
+description: File a feature request
+labels: ["enhancement"]
+title: "[Feature Request]:"
+
+body:
+  - type: checkboxes
+    id: existingcheck
+    attributes:
+      label: Do you need to file a feature request?
+      description: Please help us manage our time by avoiding duplicates and common feature request with the steps below.
+      options:
+        - label: I have searched the existing feature request and this feature request is not already filed.
+        - label: I believe this is a legitimate feature request, not just a question or bug.
+  - type: textarea
+    id: feature_request_description
+    attributes:
+      label: Feature Request Description
+      description: A clear and concise description of the feature request you would like.
+      placeholder: What this feature request add more or improve?
+  - type: textarea
+    id: additional_context
+    attributes:
+      label: Additional Context
+      description: Add any other context or screenshots about the feature request here.
+      placeholder: Any additional information

rag_anything_smaranika/.github/ISSUE_TEMPLATE/question.yml

@@ -0,0 +1,26 @@
+name: Question
+description: Ask a general question
+labels: ["question"]
+title: "[Question]:"
+
+body:
+  - type: checkboxes
+    id: existingcheck
+    attributes:
+      label: Do you need to ask a question?
+      description: Please help us manage our time by avoiding duplicates and common questions with the steps below.
+      options:
+        - label: I have searched the existing question and discussions and this question is not already answered.
+        - label: I believe this is a legitimate question, not just a bug or feature request.
+  - type: textarea
+    id: question
+    attributes:
+      label: Your Question
+      description: A clear and concise description of your question.
+      placeholder: What is your question?
+  - type: textarea
+    id: context
+    attributes:
+      label: Additional Context
+      description: Provide any additional context or details that might help us understand your question better.
+      placeholder: Add any relevant information here

rag_anything_smaranika/.github/dependabot.yml

@@ -0,0 +1,11 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  - package-ecosystem: "pip" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "weekly"

rag_anything_smaranika/.github/pull_request_template.md

@@ -0,0 +1,32 @@
+<!--
+Thanks for contributing to RAGAnything!
+
+Please ensure your pull request is ready for review before submitting.
+
+About this template
+
+This template helps contributors provide a clear and concise description of their changes. Feel free to adjust it as needed.
+-->
+
+## Description
+
+[Briefly describe the changes made in this pull request.]
+
+## Related Issues
+
+[Reference any related issues or tasks addressed by this pull request.]
+
+## Changes Made
+
+[List the specific changes made in this pull request.]
+
+## Checklist
+
+- [ ] Changes tested locally
+- [ ] Code reviewed
+- [ ] Documentation updated (if necessary)
+- [ ] Unit tests added (if applicable)
+
+## Additional Notes
+
+[Add any additional notes or context for the reviewer(s).]

rag_anything_smaranika/.github/workflows/linting.yaml

@@ -0,0 +1,30 @@
+name: Linting and Formatting
+
+on:
+    push:
+        branches:
+            - main
+    pull_request:
+        branches:
+            - main
+
+jobs:
+    lint-and-format:
+        runs-on: ubuntu-latest
+
+        steps:
+            - name: Checkout code
+              uses: actions/checkout@v2
+
+            - name: Set up Python
+              uses: actions/setup-python@v2
+              with:
+                python-version: '3.x'
+
+            - name: Install dependencies
+              run: |
+                python -m pip install --upgrade pip
+                pip install pre-commit
+
+            - name: Run pre-commit
+              run: pre-commit run --all-files --show-diff-on-failure

rag_anything_smaranika/.github/workflows/pypi-publish.yml

@@ -0,0 +1,52 @@
+name: Upload RAGAnything Package
+
+on:
+  release:
+    types: [published]
+
+permissions:
+  contents: read
+
+jobs:
+  release-build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+
+      - name: Build release distributions
+        run: |
+          python -m pip install build
+          python -m build
+
+      - name: Upload distributions
+        uses: actions/upload-artifact@v4
+        with:
+          name: release-dists
+          path: dist/
+
+  pypi-publish:
+    runs-on: ubuntu-latest
+    needs:
+      - release-build
+    permissions:
+      id-token: write
+
+    environment:
+      name: pypi
+
+    steps:
+      - name: Retrieve release distributions
+        uses: actions/download-artifact@v4
+        with:
+          name: release-dists
+          path: dist/
+
+      - name: Publish release distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: dist/

rag_anything_smaranika/.gitignore

@@ -0,0 +1,79 @@
+# Python-related files
+__pycache__/
+*.py[cod]
+*.egg-info/
+.eggs/
+*.tgz
+*.tar.gz
+*.ini
+
+# Virtual Environment
+.venv/
+env/
+venv/
+
+*.env*
+.env_example
+
+# Build / Distribution
+dist/
+build/
+site/
+
+# Logs / Reports
+*.log
+*.log.*
+*.logfire
+*.coverage/
+log/
+
+# Caches
+.cache/
+.mypy_cache/
+.pytest_cache/
+.ruff_cache/
+.gradio/
+.history/
+temp/
+
+# IDE / Editor Files
+.idea/
+.vscode/
+.vscode/settings.json
+
+# Framework-specific files
+local_neo4jWorkDir/
+neo4jWorkDir/
+
+# Data & Storage
+inputs/
+rag_storage*/
+examples/input/
+examples/output/
+output*/
+
+# Miscellaneous
+.DS_Store
+TODO.md
+ignore_this.txt
+*.ignore.*
+
+# Project-specific files
+dickens*/
+book.txt
+LightRAG.pdf
+LightRAG_2-4.pdf
+download_models_hf.py
+lightrag-dev/
+gui/
+
+# unit-test files
+test_*
+
+# Cline files
+memory-bank/
+
+# AI
+.claude/
+.cursor/
+CLAUDE.md

rag_anything_smaranika/.pre-commit-config.yaml

@@ -0,0 +1,28 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: trailing-whitespace
+        exclude: ^lightrag/api/webui/
+      - id: end-of-file-fixer
+        exclude: ^lightrag/api/webui/
+      - id: requirements-txt-fixer
+        exclude: ^lightrag/api/webui/
+
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.6.4
+    hooks:
+      - id: ruff-format
+        exclude: ^lightrag/api/webui/
+      - id: ruff
+        args: [--fix, --ignore=E402]
+        exclude: ^lightrag/api/webui/
+
+
+  - repo: https://github.com/mgedmin/check-manifest
+    rev: "0.49"
+    hooks:
+      - id: check-manifest
+        stages: [manual]
+        exclude: ^lightrag/api/webui/

rag_anything_smaranika/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 ✨Data Intelligence Lab@HKU✨
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

rag_anything_smaranika/MANIFEST.in

@@ -0,0 +1,9 @@
+include requirements.txt
+include README.md
+include README_zh.md
+include LICENSE
+recursive-include raganything *.py
+recursive-include examples *.py
+global-exclude *.pyc
+global-exclude __pycache__
+global-exclude *.egg-info

rag_anything_smaranika/README.md

@@ -0,0 +1,1260 @@
+<div align="center">
+
+<div style="margin: 20px 0;">
+  <img src="./assets/logo.png" width="120" height="120" alt="RAG-Anything Logo" style="border-radius: 20px; box-shadow: 0 8px 32px rgba(0, 217, 255, 0.3);">
+</div>
+
+# 🚀 RAG-Anything: All-in-One RAG Framework
+
+<a href="https://trendshift.io/repositories/14959" target="_blank"><img src="https://trendshift.io/api/badge/repositories/14959" alt="HKUDS%2FRAG-Anything | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
+
+<div align="center">
+  <img src="https://readme-typing-svg.herokuapp.com?font=Orbitron&size=24&duration=3000&pause=1000&color=00D9FF&center=true&vCenter=true&width=600&lines=Welcome+to+RAG-Anything;Next-Gen+Multimodal+RAG+System;Powered+by+Advanced+AI+Technology" alt="Typing Animation" />
+</div>
+
+<div align="center">
+  <div style="background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); border-radius: 15px; padding: 25px; text-align: center;">
+    <p>
+      <a href='https://github.com/HKUDS/RAG-Anything'><img src='https://img.shields.io/badge/🔥Project-Page-00d9ff?style=for-the-badge&logo=github&logoColor=white&labelColor=1a1a2e'></a>
+      <a href='https://arxiv.org/abs/2410.05779'><img src='https://img.shields.io/badge/📄arXiv-2410.05779-ff6b6b?style=for-the-badge&logo=arxiv&logoColor=white&labelColor=1a1a2e'></a>
+      <a href='https://github.com/HKUDS/LightRAG'><img src='https://img.shields.io/badge/⚡Based%20on-LightRAG-4ecdc4?style=for-the-badge&logo=lightning&logoColor=white&labelColor=1a1a2e'></a>
+    </p>
+    <p>
+      <a href="https://github.com/HKUDS/RAG-Anything/stargazers"><img src='https://img.shields.io/github/stars/HKUDS/RAG-Anything?color=00d9ff&style=for-the-badge&logo=star&logoColor=white&labelColor=1a1a2e' /></a>
+      <img src="https://img.shields.io/badge/🐍Python-3.10-4ecdc4?style=for-the-badge&logo=python&logoColor=white&labelColor=1a1a2e">
+      <a href="https://pypi.org/project/raganything/"><img src="https://img.shields.io/pypi/v/raganything.svg?style=for-the-badge&logo=pypi&logoColor=white&labelColor=1a1a2e&color=ff6b6b"></a>
+      <a href="https://github.com/astral-sh/uv"><img src="https://img.shields.io/badge/⚡uv-Ready-ff6b6b?style=for-the-badge&logo=python&logoColor=white&labelColor=1a1a2e"></a>
+    </p>
+    <p>
+      <a href="https://discord.gg/yF2MmDJyGJ"><img src="https://img.shields.io/badge/💬Discord-Community-7289da?style=for-the-badge&logo=discord&logoColor=white&labelColor=1a1a2e"></a>
+      <a href="https://github.com/HKUDS/RAG-Anything/issues/7"><img src="https://img.shields.io/badge/💬WeChat-Group-07c160?style=for-the-badge&logo=wechat&logoColor=white&labelColor=1a1a2e"></a>
+    </p>
+    <p>
+      <a href="README_zh.md"><img src="https://img.shields.io/badge/🇨🇳中文版-1a1a2e?style=for-the-badge"></a>
+      <a href="README.md"><img src="https://img.shields.io/badge/🇺🇸English-1a1a2e?style=for-the-badge"></a>
+    </p>
+  </div>
+</div>
+
+</div>
+
+<div align="center">
+  <div style="width: 100%; height: 2px; margin: 20px 0; background: linear-gradient(90deg, transparent, #00d9ff, transparent);"></div>
+</div>
+
+<div align="center">
+  <a href="#-quick-start" style="text-decoration: none;">
+    <img src="https://img.shields.io/badge/Quick%20Start-Get%20Started%20Now-00d9ff?style=for-the-badge&logo=rocket&logoColor=white&labelColor=1a1a2e">
+  </a>
+</div>
+
+---
+
+## 🎉 News
+- [X] [2025.08.12]🎯📢 🔍 RAG-Anything now features **VLM-Enhanced Query** mode! When documents include images, the system seamlessly integrates them into VLM for advanced multimodal analysis, combining visual and textual context for deeper insights.
+- [X] [2025.07.05]🎯📢 RAG-Anything now features a [context configuration module](docs/context_aware_processing.md), enabling intelligent integration of relevant contextual information to enhance multimodal content processing.
+- [X] [2025.07.04]🎯📢 🚀 RAG-Anything now supports multimodal query capabilities, enabling enhanced RAG with seamless processing of text, images, tables, and equations.
+- [X] [2025.07.03]🎯📢 🎉 RAG-Anything has reached 1k🌟 stars on GitHub! Thank you for your incredible support and valuable contributions to the project.
+
+---
+
+## 🌟 System Overview
+
+*Next-Generation Multimodal Intelligence*
+
+<div style="background: linear-gradient(135deg, #1a1a2e 0%, #16213e 50%, #0f3460 100%); border-radius: 15px; padding: 25px; margin: 20px 0; border: 2px solid #00d9ff; box-shadow: 0 0 30px rgba(0, 217, 255, 0.3);">
+
+Modern documents increasingly contain diverse multimodal content—text, images, tables, equations, charts, and multimedia—that traditional text-focused RAG systems cannot effectively process. **RAG-Anything** addresses this challenge as a comprehensive **All-in-One Multimodal Document Processing RAG system** built on [LightRAG](https://github.com/HKUDS/LightRAG).
+
+As a unified solution, RAG-Anything **eliminates the need for multiple specialized tools**. It provides **seamless processing and querying across all content modalities** within a single integrated framework. Unlike conventional RAG approaches that struggle with non-textual elements, our all-in-one system delivers **comprehensive multimodal retrieval capabilities**.
+
+Users can query documents containing **interleaved text**, **visual diagrams**, **structured tables**, and **mathematical formulations** through **one cohesive interface**. This consolidated approach makes RAG-Anything particularly valuable for academic research, technical documentation, financial reports, and enterprise knowledge management where rich, mixed-content documents demand a **unified processing framework**.
+
+<img src="assets/rag_anything_framework.png" alt="RAG-Anything" />
+
+</div>
+
+### 🎯 Key Features
+
+<div style="background: linear-gradient(135deg, #1a1a2e 0%, #16213e 100%); border-radius: 15px; padding: 25px; margin: 20px 0;">
+
+- **🔄 End-to-End Multimodal Pipeline** - Complete workflow from document ingestion and parsing to intelligent multimodal query answering
+- **📄 Universal Document Support** - Seamless processing of PDFs, Office documents, images, and diverse file formats
+- **🧠 Specialized Content Analysis** - Dedicated processors for images, tables, mathematical equations, and heterogeneous content types
+- **🔗 Multimodal Knowledge Graph** - Automatic entity extraction and cross-modal relationship discovery for enhanced understanding
+- **⚡ Adaptive Processing Modes** - Flexible MinerU-based parsing or direct multimodal content injection workflows
+- **📋 Direct Content List Insertion** - Bypass document parsing by directly inserting pre-parsed content lists from external sources
+- **🎯 Hybrid Intelligent Retrieval** - Advanced search capabilities spanning textual and multimodal content with contextual understanding
+
+</div>
+
+---
+
+## 🏗️ Algorithm & Architecture
+
+<div style="background: linear-gradient(135deg, #0f0f23 0%, #1a1a2e 100%); border-radius: 15px; padding: 25px; margin: 20px 0; border-left: 5px solid #00d9ff;">
+
+### Core Algorithm
+
+**RAG-Anything** implements an effective **multi-stage multimodal pipeline** that fundamentally extends traditional RAG architectures to seamlessly handle diverse content modalities through intelligent orchestration and cross-modal understanding.
+
+</div>
+
+<div align="center">
+  <div style="width: 100%; max-width: 600px; margin: 20px auto; padding: 20px; background: linear-gradient(135deg, rgba(0, 217, 255, 0.1) 0%, rgba(0, 217, 255, 0.05) 100%); border-radius: 15px; border: 1px solid rgba(0, 217, 255, 0.2);">
+    <div style="display: flex; justify-content: space-around; align-items: center; flex-wrap: wrap; gap: 20px;">
+      <div style="text-align: center;">
+        <div style="font-size: 24px; margin-bottom: 10px;">📄</div>
+        <div style="font-size: 14px; color: #00d9ff;">Document Parsing</div>
+      </div>
+      <div style="font-size: 20px; color: #00d9ff;">→</div>
+      <div style="text-align: center;">
+        <div style="font-size: 24px; margin-bottom: 10px;">🧠</div>
+        <div style="font-size: 14px; color: #00d9ff;">Content Analysis</div>
+      </div>
+      <div style="font-size: 20px; color: #00d9ff;">→</div>
+      <div style="text-align: center;">
+        <div style="font-size: 24px; margin-bottom: 10px;">🔍</div>
+        <div style="font-size: 14px; color: #00d9ff;">Knowledge Graph</div>
+      </div>
+      <div style="font-size: 20px; color: #00d9ff;">→</div>
+      <div style="text-align: center;">
+        <div style="font-size: 24px; margin-bottom: 10px;">🎯</div>
+        <div style="font-size: 14px; color: #00d9ff;">Intelligent Retrieval</div>
+      </div>
+    </div>
+  </div>
+</div>
+
+### 1. Document Parsing Stage
+
+<div style="background: linear-gradient(90deg, #1a1a2e 0%, #16213e 100%); border-radius: 10px; padding: 20px; margin: 15px 0; border-left: 4px solid #4ecdc4;">
+
+The system provides high-fidelity document extraction through adaptive content decomposition. It intelligently segments heterogeneous elements while preserving contextual relationships. Universal format compatibility is achieved via specialized optimized parsers.
+
+**Key Components:**
+
+- **⚙️ MinerU Integration**: Leverages [MinerU](https://github.com/opendatalab/MinerU) for high-fidelity document structure extraction and semantic preservation across complex layouts.
+
+- **🧩 Adaptive Content Decomposition**: Automatically segments documents into coherent text blocks, visual elements, structured tables, mathematical equations, and specialized content types while preserving contextual relationships.
+
+- **📁 Universal Format Support**: Provides comprehensive handling of PDFs, Office documents (DOC/DOCX/PPT/PPTX/XLS/XLSX), images, and emerging formats through specialized parsers with format-specific optimization.
+
+</div>
+
+### 2. Multi-Modal Content Understanding & Processing
+
+<div style="background: linear-gradient(90deg, #16213e 0%, #0f3460 100%); border-radius: 10px; padding: 20px; margin: 15px 0; border-left: 4px solid #ff6b6b;">
+
+The system automatically categorizes and routes content through optimized channels. It uses concurrent pipelines for parallel text and multimodal processing. Document hierarchy and relationships are preserved during transformation.
+
+**Key Components:**
+
+- **🎯 Autonomous Content Categorization and Routing**: Automatically identify, categorize, and route different content types through optimized execution channels.
+
+- **⚡ Concurrent Multi-Pipeline Architecture**: Implements concurrent execution of textual and multimodal content through dedicated processing pipelines. This approach maximizes throughput efficiency while preserving content integrity.
+
+- **🏗️ Document Hierarchy Extraction**: Extracts and preserves original document hierarchy and inter-element relationships during content transformation.
+
+</div>
+
+### 3. Multimodal Analysis Engine
+
+<div style="background: linear-gradient(90deg, #0f3460 0%, #1a1a2e 100%); border-radius: 10px; padding: 20px; margin: 15px 0; border-left: 4px solid #00d9ff;">
+
+The system deploys modality-aware processing units for heterogeneous data modalities:
+
+**Specialized Analyzers:**
+
+- **🔍 Visual Content Analyzer**:
+  - Integrate vision model for image analysis.
+  - Generates context-aware descriptive captions based on visual semantics.
+  - Extracts spatial relationships and hierarchical structures between visual elements.
+
+- **📊 Structured Data Interpreter**:
+  - Performs systematic interpretation of tabular and structured data formats.
+  - Implements statistical pattern recognition algorithms for data trend analysis.
+  - Identifies semantic relationships and dependencies across multiple tabular datasets.
+
+- **📐 Mathematical Expression Parser**:
+  - Parses complex mathematical expressions and formulas with high accuracy.
+  - Provides native LaTeX format support for seamless integration with academic workflows.
+  - Establishes conceptual mappings between mathematical equations and domain-specific knowledge bases.
+
+- **🔧 Extensible Modality Handler**:
+  - Provides configurable processing framework for custom and emerging content types.
+  - Enables dynamic integration of new modality processors through plugin architecture.
+  - Supports runtime configuration of processing pipelines for specialized use cases.
+
+</div>
+
+### 4. Multimodal Knowledge Graph Index
+
+<div style="background: linear-gradient(90deg, #1a1a2e 0%, #16213e 100%); border-radius: 10px; padding: 20px; margin: 15px 0; border-left: 4px solid #4ecdc4;">
+
+The multi-modal knowledge graph construction module transforms document content into structured semantic representations. It extracts multimodal entities, establishes cross-modal relationships, and preserves hierarchical organization. The system applies weighted relevance scoring for optimized knowledge retrieval.
+
+**Core Functions:**
+
+- **🔍 Multi-Modal Entity Extraction**: Transforms significant multimodal elements into structured knowledge graph entities. The process includes semantic annotations and metadata preservation.
+
+- **🔗 Cross-Modal Relationship Mapping**: Establishes semantic connections and dependencies between textual entities and multimodal components. This is achieved through automated relationship inference algorithms.
+
+- **🏗️ Hierarchical Structure Preservation**: Maintains original document organization through "belongs_to" relationship chains. These chains preserve logical content hierarchy and sectional dependencies.
+
+- **⚖️ Weighted Relationship Scoring**: Assigns quantitative relevance scores to relationship types. Scoring is based on semantic proximity and contextual significance within the document structure.
+
+</div>
+
+### 5. Modality-Aware Retrieval
+
+<div style="background: linear-gradient(90deg, #16213e 0%, #0f3460 100%); border-radius: 10px; padding: 20px; margin: 15px 0; border-left: 4px solid #ff6b6b;">
+
+The hybrid retrieval system combines vector similarity search with graph traversal algorithms for comprehensive content retrieval. It implements modality-aware ranking mechanisms and maintains relational coherence between retrieved elements to ensure contextually integrated information delivery.
+
+**Retrieval Mechanisms:**
+
+- **🔀 Vector-Graph Fusion**: Integrates vector similarity search with graph traversal algorithms. This approach leverages both semantic embeddings and structural relationships for comprehensive content retrieval.
+
+- **📊 Modality-Aware Ranking**: Implements adaptive scoring mechanisms that weight retrieval results based on content type relevance. The system adjusts rankings according to query-specific modality preferences.
+
+- **🔗 Relational Coherence Maintenance**: Maintains semantic and structural relationships between retrieved elements. This ensures coherent information delivery and contextual integrity.
+
+</div>
+
+---
+
+## 🚀 Quick Start
+
+*Initialize Your AI Journey*
+
+<div align="center">
+  <img src="https://user-images.githubusercontent.com/74038190/212284158-e840e285-664b-44d7-b79b-e264b5e54825.gif" width="400">
+</div>
+
+### Installation
+
+#### Option 1: Install from PyPI (Recommended)
+
+```bash
+# Basic installation
+pip install raganything
+
+# With optional dependencies for extended format support:
+pip install 'raganything[all]'              # All optional features
+pip install 'raganything[image]'            # Image format conversion (BMP, TIFF, GIF, WebP)
+pip install 'raganything[text]'             # Text file processing (TXT, MD)
+pip install 'raganything[image,text]'       # Multiple features
+```
+
+#### Option 2: Install from Source
+```bash
+# Install uv (if not already installed)
+curl -LsSf https://astral.sh/uv/install.sh | sh
+
+# Clone and setup the project with uv
+git clone https://github.com/HKUDS/RAG-Anything.git
+cd RAG-Anything
+
+# Install the package and dependencies in a virtual environment
+uv sync
+
+# If you encounter network timeouts (especially for opencv packages):
+# UV_HTTP_TIMEOUT=120 uv sync
+
+# Run commands directly with uv (recommended approach)
+uv run python examples/raganything_example.py --help
+
+# Install with optional dependencies
+uv sync --extra image --extra text  # Specific extras
+uv sync --all-extras                 # All optional features
+```
+
+#### Optional Dependencies
+
+- **`[image]`** - Enables processing of BMP, TIFF, GIF, WebP image formats (requires Pillow)
+- **`[text]`** - Enables processing of TXT and MD files (requires ReportLab)
+- **`[all]`** - Includes all Python optional dependencies
+
+> **⚠️ Office Document Processing Requirements:**
+> - Office documents (.doc, .docx, .ppt, .pptx, .xls, .xlsx) require **LibreOffice** installation
+> - Download from [LibreOffice official website](https://www.libreoffice.org/download/download/)
+> - **Windows**: Download installer from official website
+> - **macOS**: `brew install --cask libreoffice`
+> - **Ubuntu/Debian**: `sudo apt-get install libreoffice`
+> - **CentOS/RHEL**: `sudo yum install libreoffice`
+
+**Check MinerU installation:**
+
+```bash
+# Verify installation
+mineru --version
+
+# Check if properly configured
+python -c "from raganything import RAGAnything; rag = RAGAnything(); print('✅ MinerU installed properly' if rag.check_parser_installation() else '❌ MinerU installation issue')"
+```
+
+Models are downloaded automatically on first use. For manual download, refer to [MinerU Model Source Configuration](https://github.com/opendatalab/MinerU/blob/master/README.md#22-model-source-configuration).
+
+### Usage Examples
+
+#### 1. End-to-End Document Processing
+
+```python
+import asyncio
+from raganything import RAGAnything, RAGAnythingConfig
+from lightrag.llm.openai import openai_complete_if_cache, openai_embed
+from lightrag.utils import EmbeddingFunc
+
+async def main():
+    # Set up API configuration
+    api_key = "your-api-key"
+    base_url = "your-base-url"  # Optional
+
+    # Create RAGAnything configuration
+    config = RAGAnythingConfig(
+        working_dir="./rag_storage",
+        parser="mineru",  # Parser selection: mineru or docling
+        parse_method="auto",  # Parse method: auto, ocr, or txt
+        enable_image_processing=True,
+        enable_table_processing=True,
+        enable_equation_processing=True,
+    )
+
+    # Define LLM model function
+    def llm_model_func(prompt, system_prompt=None, history_messages=[], **kwargs):
+        return openai_complete_if_cache(
+            "gpt-4o-mini",
+            prompt,
+            system_prompt=system_prompt,
+            history_messages=history_messages,
+            api_key=api_key,
+            base_url=base_url,
+            **kwargs,
+        )
+
+    # Define vision model function for image processing
+    def vision_model_func(
+        prompt, system_prompt=None, history_messages=[], image_data=None, messages=None, **kwargs
+    ):
+        # If messages format is provided (for multimodal VLM enhanced query), use it directly
+        if messages:
+            return openai_complete_if_cache(
+                "gpt-4o",
+                "",
+                system_prompt=None,
+                history_messages=[],
+                messages=messages,
+                api_key=api_key,
+                base_url=base_url,
+                **kwargs,
+            )
+        # Traditional single image format
+        elif image_data:
+            return openai_complete_if_cache(
+                "gpt-4o",
+                "",
+                system_prompt=None,
+                history_messages=[],
+                messages=[
+                    {"role": "system", "content": system_prompt}
+                    if system_prompt
+                    else None,
+                    {
+                        "role": "user",
+                        "content": [
+                            {"type": "text", "text": prompt},
+                            {
+                                "type": "image_url",
+                                "image_url": {
+                                    "url": f"data:image/jpeg;base64,{image_data}"
+                                },
+                            },
+                        ],
+                    }
+                    if image_data
+                    else {"role": "user", "content": prompt},
+                ],
+                api_key=api_key,
+                base_url=base_url,
+                **kwargs,
+            )
+        # Pure text format
+        else:
+            return llm_model_func(prompt, system_prompt, history_messages, **kwargs)
+
+    # Define embedding function
+    embedding_func = EmbeddingFunc(
+        embedding_dim=3072,
+        max_token_size=8192,
+        func=lambda texts: openai_embed(
+            texts,
+            model="text-embedding-3-large",
+            api_key=api_key,
+            base_url=base_url,
+        ),
+    )
+
+    # Initialize RAGAnything
+    rag = RAGAnything(
+        config=config,
+        llm_model_func=llm_model_func,
+        vision_model_func=vision_model_func,
+        embedding_func=embedding_func,
+    )
+
+    # Process a document
+    await rag.process_document_complete(
+        file_path="path/to/your/document.pdf",
+        output_dir="./output",
+        parse_method="auto"
+    )
+
+    # Query the processed content
+    # Pure text query - for basic knowledge base search
+    text_result = await rag.aquery(
+        "What are the main findings shown in the figures and tables?",
+        mode="hybrid"
+    )
+    print("Text query result:", text_result)
+
+    # Multimodal query with specific multimodal content
+    multimodal_result = await rag.aquery_with_multimodal(
+    "Explain this formula and its relevance to the document content",
+    multimodal_content=[{
+        "type": "equation",
+        "latex": "P(d|q) = \\frac{P(q|d) \\cdot P(d)}{P(q)}",
+        "equation_caption": "Document relevance probability"
+    }],
+    mode="hybrid"
+)
+    print("Multimodal query result:", multimodal_result)
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+#### 2. Direct Multimodal Content Processing
+
+```python
+import asyncio
+from lightrag import LightRAG
+from lightrag.llm.openai import openai_complete_if_cache, openai_embed
+from lightrag.utils import EmbeddingFunc
+from raganything.modalprocessors import ImageModalProcessor, TableModalProcessor
+
+async def process_multimodal_content():
+    # Set up API configuration
+    api_key = "your-api-key"
+    base_url = "your-base-url"  # Optional
+
+    # Initialize LightRAG
+    rag = LightRAG(
+        working_dir="./rag_storage",
+        llm_model_func=lambda prompt, system_prompt=None, history_messages=[], **kwargs: openai_complete_if_cache(
+            "gpt-4o-mini",
+            prompt,
+            system_prompt=system_prompt,
+            history_messages=history_messages,
+            api_key=api_key,
+            base_url=base_url,
+            **kwargs,
+        ),
+        embedding_func=EmbeddingFunc(
+            embedding_dim=3072,
+            max_token_size=8192,
+            func=lambda texts: openai_embed(
+                texts,
+                model="text-embedding-3-large",
+                api_key=api_key,
+                base_url=base_url,
+            ),
+        )
+    )
+    await rag.initialize_storages()
+
+    # Process an image
+    image_processor = ImageModalProcessor(
+        lightrag=rag,
+        modal_caption_func=lambda prompt, system_prompt=None, history_messages=[], image_data=None, **kwargs: openai_complete_if_cache(
+            "gpt-4o",
+            "",
+            system_prompt=None,
+            history_messages=[],
+            messages=[
+                {"role": "system", "content": system_prompt} if system_prompt else None,
+                {"role": "user", "content": [
+                    {"type": "text", "text": prompt},
+                    {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{image_data}"}}
+                ]} if image_data else {"role": "user", "content": prompt}
+            ],
+            api_key=api_key,
+            base_url=base_url,
+            **kwargs,
+        ) if image_data else openai_complete_if_cache(
+            "gpt-4o-mini",
+            prompt,
+            system_prompt=system_prompt,
+            history_messages=history_messages,
+            api_key=api_key,
+            base_url=base_url,
+            **kwargs,
+        )
+    )
+
+    image_content = {
+        "img_path": "path/to/image.jpg",
+        "image_caption": ["Figure 1: Experimental results"],
+        "image_footnote": ["Data collected in 2024"]
+    }
+
+    description, entity_info = await image_processor.process_multimodal_content(
+        modal_content=image_content,
+        content_type="image",
+        file_path="research_paper.pdf",
+        entity_name="Experimental Results Figure"
+    )
+
+    # Process a table
+    table_processor = TableModalProcessor(
+        lightrag=rag,
+        modal_caption_func=lambda prompt, system_prompt=None, history_messages=[], **kwargs: openai_complete_if_cache(
+            "gpt-4o-mini",
+            prompt,
+            system_prompt=system_prompt,
+            history_messages=history_messages,
+            api_key=api_key,
+            base_url=base_url,
+            **kwargs,
+        )
+    )
+
+    table_content = {
+        "table_body": """
+        | Method | Accuracy | F1-Score |
+        |--------|----------|----------|
+        | RAGAnything | 95.2% | 0.94 |
+        | Baseline | 87.3% | 0.85 |
+        """,
+        "table_caption": ["Performance Comparison"],
+        "table_footnote": ["Results on test dataset"]
+    }
+
+    description, entity_info = await table_processor.process_multimodal_content(
+        modal_content=table_content,
+        content_type="table",
+        file_path="research_paper.pdf",
+        entity_name="Performance Results Table"
+    )
+
+if __name__ == "__main__":
+    asyncio.run(process_multimodal_content())
+```
+
+#### 3. Batch Processing
+
+```python
+# Process multiple documents
+await rag.process_folder_complete(
+    folder_path="./documents",
+    output_dir="./output",
+    file_extensions=[".pdf", ".docx", ".pptx"],
+    recursive=True,
+    max_workers=4
+)
+```
+
+#### 4. Custom Modal Processors
+
+```python
+from raganything.modalprocessors import GenericModalProcessor
+
+class CustomModalProcessor(GenericModalProcessor):
+    async def process_multimodal_content(self, modal_content, content_type, file_path, entity_name):
+        # Your custom processing logic
+        enhanced_description = await self.analyze_custom_content(modal_content)
+        entity_info = self.create_custom_entity(enhanced_description, entity_name)
+        return await self._create_entity_and_chunk(enhanced_description, entity_info, file_path)
+```
+
+#### 5. Query Options
+
+RAG-Anything provides three types of query methods:
+
+**Pure Text Queries** - Direct knowledge base search using LightRAG:
+```python
+# Different query modes for text queries
+text_result_hybrid = await rag.aquery("Your question", mode="hybrid")
+text_result_local = await rag.aquery("Your question", mode="local")
+text_result_global = await rag.aquery("Your question", mode="global")
+text_result_naive = await rag.aquery("Your question", mode="naive")
+
+# Synchronous version
+sync_text_result = rag.query("Your question", mode="hybrid")
+```
+
+**VLM Enhanced Queries** - Automatically analyze images in retrieved context using VLM:
+```python
+# VLM enhanced query (automatically enabled when vision_model_func is provided)
+vlm_result = await rag.aquery(
+    "Analyze the charts and figures in the document",
+    mode="hybrid"
+    # vlm_enhanced=True is automatically set when vision_model_func is available
+)
+
+# Manually control VLM enhancement
+vlm_enabled = await rag.aquery(
+    "What do the images show in this document?",
+    mode="hybrid",
+    vlm_enhanced=True  # Force enable VLM enhancement
+)
+
+vlm_disabled = await rag.aquery(
+    "What do the images show in this document?",
+    mode="hybrid",
+    vlm_enhanced=False  # Force disable VLM enhancement
+)
+
+# When documents contain images, VLM can see and analyze them directly
+# The system will automatically:
+# 1. Retrieve relevant context containing image paths
+# 2. Load and encode images as base64
+# 3. Send both text context and images to VLM for comprehensive analysis
+```
+
+**Multimodal Queries** - Enhanced queries with specific multimodal content analysis:
+```python
+# Query with table data
+table_result = await rag.aquery_with_multimodal(
+    "Compare these performance metrics with the document content",
+    multimodal_content=[{
+        "type": "table",
+        "table_data": """Method,Accuracy,Speed
+                        RAGAnything,95.2%,120ms
+                        Traditional,87.3%,180ms""",
+        "table_caption": "Performance comparison"
+    }],
+    mode="hybrid"
+)
+
+# Query with equation content
+equation_result = await rag.aquery_with_multimodal(
+    "Explain this formula and its relevance to the document content",
+    multimodal_content=[{
+        "type": "equation",
+        "latex": "P(d|q) = \\frac{P(q|d) \\cdot P(d)}{P(q)}",
+        "equation_caption": "Document relevance probability"
+    }],
+    mode="hybrid"
+)
+```
+
+#### 6. Loading Existing LightRAG Instance
+
+```python
+import asyncio
+from raganything import RAGAnything, RAGAnythingConfig
+from lightrag import LightRAG
+from lightrag.llm.openai import openai_complete_if_cache, openai_embed
+from lightrag.kg.shared_storage import initialize_pipeline_status
+from lightrag.utils import EmbeddingFunc
+import os
+
+async def load_existing_lightrag():
+    # Set up API configuration
+    api_key = "your-api-key"
+    base_url = "your-base-url"  # Optional
+
+    # First, create or load existing LightRAG instance
+    lightrag_working_dir = "./existing_lightrag_storage"
+
+    # Check if previous LightRAG instance exists
+    if os.path.exists(lightrag_working_dir) and os.listdir(lightrag_working_dir):
+        print("✅ Found existing LightRAG instance, loading...")
+    else:
+        print("❌ No existing LightRAG instance found, will create new one")
+
+    # Create/load LightRAG instance with your configuration
+    lightrag_instance = LightRAG(
+        working_dir=lightrag_working_dir,
+        llm_model_func=lambda prompt, system_prompt=None, history_messages=[], **kwargs: openai_complete_if_cache(
+            "gpt-4o-mini",
+            prompt,
+            system_prompt=system_prompt,
+            history_messages=history_messages,
+            api_key=api_key,
+            base_url=base_url,
+            **kwargs,
+        ),
+        embedding_func=EmbeddingFunc(
+            embedding_dim=3072,
+            max_token_size=8192,
+            func=lambda texts: openai_embed(
+                texts,
+                model="text-embedding-3-large",
+                api_key=api_key,
+                base_url=base_url,
+            ),
+        )
+    )
+
+    # Initialize storage (this will load existing data if available)
+    await lightrag_instance.initialize_storages()
+    await initialize_pipeline_status()
+
+    # Define vision model function for image processing
+    def vision_model_func(
+        prompt, system_prompt=None, history_messages=[], image_data=None, messages=None, **kwargs
+    ):
+        # If messages format is provided (for multimodal VLM enhanced query), use it directly
+        if messages:
+            return openai_complete_if_cache(
+                "gpt-4o",
+                "",
+                system_prompt=None,
+                history_messages=[],
+                messages=messages,
+                api_key=api_key,
+                base_url=base_url,
+                **kwargs,
+            )
+        # Traditional single image format
+        elif image_data:
+            return openai_complete_if_cache(
+                "gpt-4o",
+                "",
+                system_prompt=None,
+                history_messages=[],
+                messages=[
+                    {"role": "system", "content": system_prompt}
+                    if system_prompt
+                    else None,
+                    {
+                        "role": "user",
+                        "content": [
+                            {"type": "text", "text": prompt},
+                            {
+                                "type": "image_url",
+                                "image_url": {
+                                    "url": f"data:image/jpeg;base64,{image_data}"
+                                },
+                            },
+                        ],
+                    }
+                    if image_data
+                    else {"role": "user", "content": prompt},
+                ],
+                api_key=api_key,
+                base_url=base_url,
+                **kwargs,
+            )
+        # Pure text format
+        else:
+            return lightrag_instance.llm_model_func(prompt, system_prompt, history_messages, **kwargs)
+
+    # Now use existing LightRAG instance to initialize RAGAnything
+    rag = RAGAnything(
+        lightrag=lightrag_instance,  # Pass existing LightRAG instance
+        vision_model_func=vision_model_func,
+        # Note: working_dir, llm_model_func, embedding_func, etc. are inherited from lightrag_instance
+    )
+
+    # Query existing knowledge base
+    result = await rag.aquery(
+        "What data has been processed in this LightRAG instance?",
+        mode="hybrid"
+    )
+    print("Query result:", result)
+
+    # Add new multimodal document to existing LightRAG instance
+    await rag.process_document_complete(
+        file_path="path/to/new/multimodal_document.pdf",
+        output_dir="./output"
+    )
+
+if __name__ == "__main__":
+    asyncio.run(load_existing_lightrag())
+```
+
+#### 7. Direct Content List Insertion
+
+For scenarios where you already have a pre-parsed content list (e.g., from external parsers or previous processing), you can directly insert it into RAGAnything without document parsing:
+
+```python
+import asyncio
+from raganything import RAGAnything, RAGAnythingConfig
+from lightrag.llm.openai import openai_complete_if_cache, openai_embed
+from lightrag.utils import EmbeddingFunc
+
+async def insert_content_list_example():
+    # Set up API configuration
+    api_key = "your-api-key"
+    base_url = "your-base-url"  # Optional
+
+    # Create RAGAnything configuration
+    config = RAGAnythingConfig(
+        working_dir="./rag_storage",
+        enable_image_processing=True,
+        enable_table_processing=True,
+        enable_equation_processing=True,
+    )
+
+    # Define model functions
+    def llm_model_func(prompt, system_prompt=None, history_messages=[], **kwargs):
+        return openai_complete_if_cache(
+            "gpt-4o-mini",
+            prompt,
+            system_prompt=system_prompt,
+            history_messages=history_messages,
+            api_key=api_key,
+            base_url=base_url,
+            **kwargs,
+        )
+
+    def vision_model_func(prompt, system_prompt=None, history_messages=[], image_data=None, messages=None, **kwargs):
+        # If messages format is provided (for multimodal VLM enhanced query), use it directly
+        if messages:
+            return openai_complete_if_cache(
+                "gpt-4o",
+                "",
+                system_prompt=None,
+                history_messages=[],
+                messages=messages,
+                api_key=api_key,
+                base_url=base_url,
+                **kwargs,
+            )
+        # Traditional single image format
+        elif image_data:
+            return openai_complete_if_cache(
+                "gpt-4o",
+                "",
+                system_prompt=None,
+                history_messages=[],
+                messages=[
+                    {"role": "system", "content": system_prompt} if system_prompt else None,
+                    {
+                        "role": "user",
+                        "content": [
+                            {"type": "text", "text": prompt},
+                            {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{image_data}"}}
+                        ],
+                    } if image_data else {"role": "user", "content": prompt},
+                ],
+                api_key=api_key,
+                base_url=base_url,
+                **kwargs,
+            )
+        # Pure text format
+        else:
+            return llm_model_func(prompt, system_prompt, history_messages, **kwargs)
+
+    embedding_func = EmbeddingFunc(
+        embedding_dim=3072,
+        max_token_size=8192,
+        func=lambda texts: openai_embed(
+            texts,
+            model="text-embedding-3-large",
+            api_key=api_key,
+            base_url=base_url,
+        ),
+    )
+
+    # Initialize RAGAnything
+    rag = RAGAnything(
+        config=config,
+        llm_model_func=llm_model_func,
+        vision_model_func=vision_model_func,
+        embedding_func=embedding_func,
+    )
+
+    # Example: Pre-parsed content list from external source
+    content_list = [
+        {
+            "type": "text",
+            "text": "This is the introduction section of our research paper.",
+            "page_idx": 0  # Page number where this content appears
+        },
+        {
+            "type": "image",
+            "img_path": "/absolute/path/to/figure1.jpg",  # IMPORTANT: Use absolute path
+            "image_caption": ["Figure 1: System Architecture"],
+            "image_footnote": ["Source: Authors' original design"],
+            "page_idx": 1  # Page number where this image appears
+        },
+        {
+            "type": "table",
+            "table_body": "| Method | Accuracy | F1-Score |\n|--------|----------|----------|\n| Ours | 95.2% | 0.94 |\n| Baseline | 87.3% | 0.85 |",
+            "table_caption": ["Table 1: Performance Comparison"],
+            "table_footnote": ["Results on test dataset"],
+            "page_idx": 2  # Page number where this table appears
+        },
+        {
+            "type": "equation",
+            "latex": "P(d|q) = \\frac{P(q|d) \\cdot P(d)}{P(q)}",
+            "text": "Document relevance probability formula",
+            "page_idx": 3  # Page number where this equation appears
+        },
+        {
+            "type": "text",
+            "text": "In conclusion, our method demonstrates superior performance across all metrics.",
+            "page_idx": 4  # Page number where this content appears
+        }
+    ]
+
+    # Insert the content list directly
+    await rag.insert_content_list(
+        content_list=content_list,
+        file_path="research_paper.pdf",  # Reference file name for citation
+        split_by_character=None,         # Optional text splitting
+        split_by_character_only=False,   # Optional text splitting mode
+        doc_id=None,                     # Optional custom document ID (will be auto-generated if not provided)
+        display_stats=True               # Show content statistics
+    )
+
+    # Query the inserted content
+    result = await rag.aquery(
+        "What are the key findings and performance metrics mentioned in the research?",
+        mode="hybrid"
+    )
+    print("Query result:", result)
+
+    # You can also insert multiple content lists with different document IDs
+    another_content_list = [
+        {
+            "type": "text",
+            "text": "This is content from another document.",
+            "page_idx": 0  # Page number where this content appears
+        },
+        {
+            "type": "table",
+            "table_body": "| Feature | Value |\n|---------|-------|\n| Speed | Fast |\n| Accuracy | High |",
+            "table_caption": ["Feature Comparison"],
+            "page_idx": 1  # Page number where this table appears
+        }
+    ]
+
+    await rag.insert_content_list(
+        content_list=another_content_list,
+        file_path="another_document.pdf",
+        doc_id="custom-doc-id-123"  # Custom document ID
+    )
+
+if __name__ == "__main__":
+    asyncio.run(insert_content_list_example())
+```
+
+**Content List Format:**
+
+The `content_list` should follow the standard format with each item being a dictionary containing:
+
+- **Text content**: `{"type": "text", "text": "content text", "page_idx": 0}`
+- **Image content**: `{"type": "image", "img_path": "/absolute/path/to/image.jpg", "image_caption": ["caption"], "image_footnote": ["note"], "page_idx": 1}`
+- **Table content**: `{"type": "table", "table_body": "markdown table", "table_caption": ["caption"], "table_footnote": ["note"], "page_idx": 2}`
+- **Equation content**: `{"type": "equation", "latex": "LaTeX formula", "text": "description", "page_idx": 3}`
+- **Generic content**: `{"type": "custom_type", "content": "any content", "page_idx": 4}`
+
+**Important Notes:**
+- **`img_path`**: Must be an absolute path to the image file (e.g., `/home/user/images/chart.jpg` or `C:\Users\user\images\chart.jpg`)
+- **`page_idx`**: Represents the page number where the content appears in the original document (0-based indexing)
+- **Content ordering**: Items are processed in the order they appear in the list
+
+This method is particularly useful when:
+- You have content from external parsers (non-MinerU/Docling)
+- You want to process programmatically generated content
+- You need to insert content from multiple sources into a single knowledge base
+- You have cached parsing results that you want to reuse
+
+---
+
+## 🛠️ Examples
+
+*Practical Implementation Demos*
+
+<div align="center">
+  <img src="https://user-images.githubusercontent.com/74038190/212257455-13e3e01e-d6a6-45dc-bb92-3ab87b12dfc1.gif" width="300">
+</div>
+
+The `examples/` directory contains comprehensive usage examples:
+
+- **`raganything_example.py`**: End-to-end document processing with MinerU
+- **`modalprocessors_example.py`**: Direct multimodal content processing
+- **`office_document_test.py`**: Office document parsing test with MinerU (no API key required)
+- **`image_format_test.py`**: Image format parsing test with MinerU (no API key required)
+- **`text_format_test.py`**: Text format parsing test with MinerU (no API key required)
+
+**Run examples:**
+
+```bash
+# End-to-end processing with parser selection
+python examples/raganything_example.py path/to/document.pdf --api-key YOUR_API_KEY --parser mineru
+
+# Direct modal processing
+python examples/modalprocessors_example.py --api-key YOUR_API_KEY
+
+# Office document parsing test (MinerU only)
+python examples/office_document_test.py --file path/to/document.docx
+
+# Image format parsing test (MinerU only)
+python examples/image_format_test.py --file path/to/image.bmp
+
+# Text format parsing test (MinerU only)
+python examples/text_format_test.py --file path/to/document.md
+
+# Check LibreOffice installation
+python examples/office_document_test.py --check-libreoffice --file dummy
+
+# Check PIL/Pillow installation
+python examples/image_format_test.py --check-pillow --file dummy
+
+# Check ReportLab installation
+python examples/text_format_test.py --check-reportlab --file dummy
+```
+
+---
+
+## 🔧 Configuration
+
+*System Optimization Parameters*
+
+### Environment Variables
+
+Create a `.env` file (refer to `.env.example`):
+
+```bash
+OPENAI_API_KEY=your_openai_api_key
+OPENAI_BASE_URL=your_base_url  # Optional
+OUTPUT_DIR=./output             # Default output directory for parsed documents
+PARSER=mineru                   # Parser selection: mineru or docling
+PARSE_METHOD=auto              # Parse method: auto, ocr, or txt
+```
+
+**Note:** For backward compatibility, legacy environment variable names are still supported:
+- `MINERU_PARSE_METHOD` is deprecated, please use `PARSE_METHOD`
+
+> **Note**: API keys are only required for full RAG processing with LLM integration. The parsing test files (`office_document_test.py` and `image_format_test.py`) only test parser functionality and do not require API keys.
+
+### Parser Configuration
+
+RAGAnything now supports multiple parsers, each with specific advantages:
+
+#### MinerU Parser
+- Supports PDF, images, Office documents, and more formats
+- Powerful OCR and table extraction capabilities
+- GPU acceleration support
+
+#### Docling Parser
+- Optimized for Office documents and HTML files
+- Better document structure preservation
+- Native support for multiple Office formats
+
+### MinerU Configuration
+
+```bash
+# MinerU 2.0 uses command-line parameters instead of config files
+# Check available options:
+mineru --help
+
+# Common configurations:
+mineru -p input.pdf -o output_dir -m auto    # Automatic parsing mode
+mineru -p input.pdf -o output_dir -m ocr     # OCR-focused parsing
+mineru -p input.pdf -o output_dir -b pipeline --device cuda  # GPU acceleration
+```
+
+You can also configure parsing through RAGAnything parameters:
+
+```python
+# Basic parsing configuration with parser selection
+await rag.process_document_complete(
+    file_path="document.pdf",
+    output_dir="./output/",
+    parse_method="auto",          # or "ocr", "txt"
+    parser="mineru"               # Optional: "mineru" or "docling"
+)
+
+# Advanced parsing configuration with special parameters
+await rag.process_document_complete(
+    file_path="document.pdf",
+    output_dir="./output/",
+    parse_method="auto",          # Parsing method: "auto", "ocr", "txt"
+    parser="mineru",              # Parser selection: "mineru" or "docling"
+
+    # MinerU special parameters - all supported kwargs:
+    lang="ch",                   # Document language for OCR optimization (e.g., "ch", "en", "ja")
+    device="cuda:0",             # Inference device: "cpu", "cuda", "cuda:0", "npu", "mps"
+    start_page=0,                # Starting page number (0-based, for PDF)
+    end_page=10,                 # Ending page number (0-based, for PDF)
+    formula=True,                # Enable formula parsing
+    table=True,                  # Enable table parsing
+    backend="pipeline",          # Parsing backend: pipeline|vlm-transformers|vlm-sglang-engine|vlm-sglang-client.
+    source="huggingface",        # Model source: "huggingface", "modelscope", "local"
+    # vlm_url="http://127.0.0.1:3000" # Service address when using backend=vlm-sglang-client
+
+    # Standard RAGAnything parameters
+    display_stats=True,          # Display content statistics
+    split_by_character=None,     # Optional character to split text by
+    doc_id=None                  # Optional document ID
+)
+```
+
+> **Note**: MinerU 2.0 no longer uses the `magic-pdf.json` configuration file. All settings are now passed as command-line parameters or function arguments. RAG-Anything now supports multiple document parsers - you can choose between MinerU and Docling based on your needs.
+
+### Processing Requirements
+
+Different content types require specific optional dependencies:
+
+- **Office Documents** (.doc, .docx, .ppt, .pptx, .xls, .xlsx): Install [LibreOffice](https://www.libreoffice.org/download/download/)
+- **Extended Image Formats** (.bmp, .tiff, .gif, .webp): Install with `pip install raganything[image]`
+- **Text Files** (.txt, .md): Install with `pip install raganything[text]`
+
+> **📋 Quick Install**: Use `pip install raganything[all]` to enable all format support (Python dependencies only - LibreOffice still needs separate installation)
+
+---
+
+## 🧪 Supported Content Types
+
+### Document Formats
+
+- **PDFs** - Research papers, reports, presentations
+- **Office Documents** - DOC, DOCX, PPT, PPTX, XLS, XLSX
+- **Images** - JPG, PNG, BMP, TIFF, GIF, WebP
+- **Text Files** - TXT, MD
+
+### Multimodal Elements
+
+- **Images** - Photographs, diagrams, charts, screenshots
+- **Tables** - Data tables, comparison charts, statistical summaries
+- **Equations** - Mathematical formulas in LaTeX format
+- **Generic Content** - Custom content types via extensible processors
+
+*For installation of format-specific dependencies, see the [Configuration](#-configuration) section.*
+
+---
+
+## 📖 Citation
+
+*Academic Reference*
+
+<div align="center">
+  <div style="width: 60px; height: 60px; margin: 20px auto; position: relative;">
+    <div style="width: 100%; height: 100%; border: 2px solid #00d9ff; border-radius: 50%; position: relative;">
+      <div style="position: absolute; top: 50%; left: 50%; transform: translate(-50%, -50%); font-size: 24px; color: #00d9ff;">📖</div>
+    </div>
+    <div style="position: absolute; bottom: -5px; left: 50%; transform: translateX(-50%); width: 20px; height: 20px; background: white; border-right: 2px solid #00d9ff; border-bottom: 2px solid #00d9ff; transform: rotate(45deg);"></div>
+  </div>
+</div>
+
+If you find RAG-Anything useful in your research, please cite our paper:
+
+```bibtex
+@article{guo2024lightrag,
+  title={LightRAG: Simple and Fast Retrieval-Augmented Generation},
+  author={Zirui Guo and Lianghao Xia and Yanhua Yu and Tu Ao and Chao Huang},
+  year={2024},
+  eprint={2410.05779},
+  archivePrefix={arXiv},
+  primaryClass={cs.IR}
+}
+```
+
+---
+
+## 🔗 Related Projects
+
+*Ecosystem & Extensions*
+
+<div align="center">
+  <table>
+    <tr>
+      <td align="center">
+        <a href="https://github.com/HKUDS/LightRAG">
+          <div style="width: 100px; height: 100px; background: linear-gradient(135deg, rgba(0, 217, 255, 0.1) 0%, rgba(0, 217, 255, 0.05) 100%); border-radius: 15px; border: 1px solid rgba(0, 217, 255, 0.2); display: flex; align-items: center; justify-content: center; margin-bottom: 10px;">
+            <span style="font-size: 32px;">⚡</span>
+          </div>
+          <b>LightRAG</b><br>
+          <sub>Simple and Fast RAG</sub>
+        </a>
+      </td>
+      <td align="center">
+        <a href="https://github.com/HKUDS/VideoRAG">
+          <div style="width: 100px; height: 100px; background: linear-gradient(135deg, rgba(0, 217, 255, 0.1) 0%, rgba(0, 217, 255, 0.05) 100%); border-radius: 15px; border: 1px solid rgba(0, 217, 255, 0.2); display: flex; align-items: center; justify-content: center; margin-bottom: 10px;">
+            <span style="font-size: 32px;">🎥</span>
+          </div>
+          <b>VideoRAG</b><br>
+          <sub>Extreme Long-Context Video RAG</sub>
+        </a>
+      </td>
+      <td align="center">
+        <a href="https://github.com/HKUDS/MiniRAG">
+          <div style="width: 100px; height: 100px; background: linear-gradient(135deg, rgba(0, 217, 255, 0.1) 0%, rgba(0, 217, 255, 0.05) 100%); border-radius: 15px; border: 1px solid rgba(0, 217, 255, 0.2); display: flex; align-items: center; justify-content: center; margin-bottom: 10px;">
+            <span style="font-size: 32px;">✨</span>
+          </div>
+          <b>MiniRAG</b><br>
+          <sub>Extremely Simple RAG</sub>
+        </a>
+      </td>
+    </tr>
+  </table>
+</div>
+
+---
+
+## ⭐ Star History
+
+*Community Growth Trajectory*
+
+<div align="center">
+  <a href="https://star-history.com/#HKUDS/RAG-Anything&Date">
+    <picture>
+      <source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=HKUDS/RAG-Anything&type=Date&theme=dark" />
+      <source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/svg?repos=HKUDS/RAG-Anything&type=Date" />
+      <img alt="Star History Chart" src="https://api.star-history.com/svg?repos=HKUDS/RAG-Anything&type=Date" style="border-radius: 15px; box-shadow: 0 0 30px rgba(0, 217, 255, 0.3);" />
+    </picture>
+  </a>
+</div>
+
+---
+
+## 🤝 Contribution
+
+*Join the Innovation*
+
+<div align="center">
+  We thank all our contributors for their valuable contributions.
+</div>
+
+<div align="center">
+  <a href="https://github.com/HKUDS/RAG-Anything/graphs/contributors">
+    <img src="https://contrib.rocks/image?repo=HKUDS/RAG-Anything" style="border-radius: 15px; box-shadow: 0 0 20px rgba(0, 217, 255, 0.3);" />
+  </a>
+</div>
+
+---
+
+<div align="center" style="background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); border-radius: 15px; padding: 30px; margin: 30px 0;">
+  <div>
+    <img src="https://user-images.githubusercontent.com/74038190/212284100-561aa473-3905-4a80-b561-0d28506553ee.gif" width="500">
+  </div>
+  <div style="margin-top: 20px;">
+    <a href="https://github.com/HKUDS/RAG-Anything" style="text-decoration: none;">
+      <img src="https://img.shields.io/badge/⭐%20Star%20us%20on%20GitHub-1a1a2e?style=for-the-badge&logo=github&logoColor=white">
+    </a>
+    <a href="https://github.com/HKUDS/RAG-Anything/issues" style="text-decoration: none;">
+      <img src="https://img.shields.io/badge/🐛%20Report%20Issues-ff6b6b?style=for-the-badge&logo=github&logoColor=white">
+    </a>
+    <a href="https://github.com/HKUDS/RAG-Anything/discussions" style="text-decoration: none;">
+      <img src="https://img.shields.io/badge/💬%20Discussions-4ecdc4?style=for-the-badge&logo=github&logoColor=white">
+    </a>
+  </div>
+</div>
+
+<div align="center">
+  <div style="width: 100%; max-width: 600px; margin: 20px auto; padding: 20px; background: linear-gradient(135deg, rgba(0, 217, 255, 0.1) 0%, rgba(0, 217, 255, 0.05) 100%); border-radius: 15px; border: 1px solid rgba(0, 217, 255, 0.2);">
+    <div style="display: flex; justify-content: center; align-items: center; gap: 15px;">
+      <span style="font-size: 24px;">⭐</span>
+      <span style="color: #00d9ff; font-size: 18px;">Thank you for visiting RAG-Anything!</span>
+      <span style="font-size: 24px;">⭐</span>
+    </div>
+    <div style="margin-top: 10px; color: #00d9ff; font-size: 16px;">Building the Future of Multimodal AI</div>
+  </div>
+</div>

rag_anything_smaranika/README_zh.md

@@ -0,0 +1,1258 @@
+<div align="center">
+
+<div style="margin: 20px 0;">
+  <img src="./assets/logo.png" width="120" height="120" alt="RAG-Anything Logo" style="border-radius: 20px; box-shadow: 0 8px 32px rgba(0, 217, 255, 0.3);">
+</div>
+
+# 🚀 RAG-Anything: All-in-One RAG System
+
+<div align="center">
+  <div style="width: 100%; height: 2px; margin: 20px 0; background: linear-gradient(90deg, transparent, #00d9ff, transparent);"></div>
+</div>
+
+<div align="center">
+  <div style="background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); border-radius: 15px; padding: 25px; text-align: center;">
+    <p>
+      <a href='https://github.com/HKUDS/RAG-Anything'><img src='https://img.shields.io/badge/🔥项目-主页-00d9ff?style=for-the-badge&logo=github&logoColor=white&labelColor=1a1a2e'></a>
+      <a href='https://arxiv.org/abs/2410.05779'><img src='https://img.shields.io/badge/📄arXiv-2410.05779-ff6b6b?style=for-the-badge&logo=arxiv&logoColor=white&labelColor=1a1a2e'></a>
+      <a href='https://github.com/HKUDS/LightRAG'><img src='https://img.shields.io/badge/⚡基于-LightRAG-4ecdc4?style=for-the-badge&logo=lightning&logoColor=white&labelColor=1a1a2e'></a>
+    </p>
+    <p>
+      <a href="https://github.com/HKUDS/RAG-Anything/stargazers"><img src='https://img.shields.io/github/stars/HKUDS/RAG-Anything?color=00d9ff&style=for-the-badge&logo=star&logoColor=white&labelColor=1a1a2e' /></a>
+      <img src="https://img.shields.io/badge/🐍Python-3.10-4ecdc4?style=for-the-badge&logo=python&logoColor=white&labelColor=1a1a2e">
+      <a href="https://pypi.org/project/raganything/"><img src="https://img.shields.io/pypi/v/raganything.svg?style=for-the-badge&logo=pypi&logoColor=white&labelColor=1a1a2e&color=ff6b6b"></a>
+    </p>
+    <p>
+      <a href="https://discord.gg/yF2MmDJyGJ"><img src="https://img.shields.io/badge/💬Discord-社区-7289da?style=for-the-badge&logo=discord&logoColor=white&labelColor=1a1a2e"></a>
+      <a href="https://github.com/HKUDS/RAG-Anything/issues/7"><img src="https://img.shields.io/badge/💬微信群-交流-07c160?style=for-the-badge&logo=wechat&logoColor=white&labelColor=1a1a2e"></a>
+    </p>
+    <p>
+      <a href="README_zh.md"><img src="https://img.shields.io/badge/🇨🇳中文版-1a1a2e?style=for-the-badge"></a>
+      <a href="README.md"><img src="https://img.shields.io/badge/🇺🇸English-1a1a2e?style=for-the-badge"></a>
+    </p>
+  </div>
+</div>
+
+</div>
+
+<div align="center" style="margin: 30px 0;">
+  <img src="https://user-images.githubusercontent.com/74038190/212284100-561aa473-3905-4a80-b561-0d28506553ee.gif" width="800">
+</div>
+
+<div align="center">
+  <a href="#-快速开始" style="text-decoration: none;">
+    <img src="https://img.shields.io/badge/快速开始-立即开始使用-00d9ff?style=for-the-badge&logo=rocket&logoColor=white&labelColor=1a1a2e">
+  </a>
+</div>
+
+---
+
+## 🎉 新闻
+- [X] [2025.08.12]🎯📢 🔍 RAGAnything 现在支持 **VLM增强查询** 模式！当文档包含图片时，系统可以自动将图片与文本上下文一起直接传递给VLM进行综合多模态分析。
+- [X] [2025.07.05]🎯📢 RAGAnything 新增[上下文配置模块](docs/context_aware_processing.md)，支持为多模态内容处理添加相关上下文信息。
+- [X] [2025.07.04]🎯📢 RAGAnything 现在支持多模态内容查询，实现了集成文本、图像、表格和公式处理的增强检索生成功能。
+- [X] [2025.07.03]🎯📢 RAGAnything 在GitHub上达到了1K星标🌟！感谢您的支持和贡献。
+
+---
+
+## 🌟 系统概述
+
+*下一代多模态智能*
+
+<div style="background: linear-gradient(135deg, #1a1a2e 0%, #16213e 50%, #0f3460 100%); border-radius: 15px; padding: 25px; margin: 20px 0; border: 2px solid #00d9ff; box-shadow: 0 0 30px rgba(0, 217, 255, 0.3);">
+
+**RAG-Anything**是一个综合性多模态文档处理RAG系统。该系统能够无缝处理和查询包含文本、图像、表格、公式等多模态内容的复杂文档，提供完整的检索增强(RAG)生成解决方案。
+
+<img src="assets/rag_anything_framework.png" alt="RAG-Anything" />
+
+</div>
+
+### 🎯 核心特性
+
+<div style="background: linear-gradient(135deg, #1a1a2e 0%, #16213e 100%); border-radius: 15px; padding: 25px; margin: 20px 0;">
+
+- **🔄 端到端多模态处理流水线** - 提供从文档解析到多模态查询响应的完整处理链路，确保系统的一体化运行
+- **📄 多格式文档支持** - 支持PDF、Office文档（DOC/DOCX/PPT/PPTX/XLS/XLSX）、图像等主流文档格式的统一处理和解析
+- **🧠 多模态内容分析引擎** - 针对图像、表格、公式和通用文本内容部署专门的处理器，确保各类内容的精准解析
+- **🔗 基于知识图谱索引** - 实现自动化实体提取和关系构建，建立跨模态的语义连接网络
+- **⚡ 灵活的处理架构** - 支持基于MinerU的智能解析模式和直接多模态内容插入模式，满足不同应用场景需求
+- **📋 直接内容列表插入** - 跳过文档解析，直接插入来自外部��的预解析内容列表，支持多种数据来源整合
+- **🎯 跨模态检索机制** - 实现跨文本和多模态内容的智能检索，提供精准的信息定位和匹配能力
+
+</div>
+
+---
+
+## 🏗️ 算法原理与架构
+
+<div style="background: linear-gradient(135deg, #0f0f23 0%, #1a1a2e 100%); border-radius: 15px; padding: 25px; margin: 20px 0; border-left: 5px solid #00d9ff;">
+
+### 核心算法
+
+**RAG-Anything** 采用灵活的分层架构设计，实现多阶段多模态处理流水线，将传统RAG系统扩展为支持异构内容类型的综合处理平台。
+
+</div>
+
+<div align="center">
+  <div style="width: 100%; max-width: 600px; margin: 20px auto; padding: 20px; background: linear-gradient(135deg, rgba(0, 217, 255, 0.1) 0%, rgba(0, 217, 255, 0.05) 100%); border-radius: 15px; border: 1px solid rgba(0, 217, 255, 0.2);">
+    <div style="display: flex; justify-content: space-around; align-items: center; flex-wrap: wrap; gap: 20px;">
+      <div style="text-align: center;">
+        <div style="font-size: 24px; margin-bottom: 10px;">📄</div>
+        <div style="font-size: 14px; color: #00d9ff;">文档解析</div>
+      </div>
+      <div style="font-size: 20px; color: #00d9ff;">→</div>
+      <div style="text-align: center;">
+        <div style="font-size: 24px; margin-bottom: 10px;">🧠</div>
+        <div style="font-size: 14px; color: #00d9ff;">内容分析</div>
+      </div>
+      <div style="font-size: 20px; color: #00d9ff;">→</div>
+      <div style="text-align: center;">
+        <div style="font-size: 24px; margin-bottom: 10px;">🔍</div>
+        <div style="font-size: 14px; color: #00d9ff;">知识图谱</div>
+      </div>
+      <div style="font-size: 20px; color: #00d9ff;">→</div>
+      <div style="text-align: center;">
+        <div style="font-size: 24px; margin-bottom: 10px;">🎯</div>
+        <div style="font-size: 14px; color: #00d9ff;">智能检索</div>
+      </div>
+    </div>
+  </div>
+</div>
+
+### 1. 文档解析阶段
+
+<div style="background: linear-gradient(90deg, #1a1a2e 0%, #16213e 100%); border-radius: 10px; padding: 20px; margin: 15px 0; border-left: 4px solid #4ecdc4;">
+
+该系统构建了高精度文档解析平台，通过结构化提取引擎实现多模态元素的完整识别与提取。系统采用自适应内容分解机制，智能分离文档中的文本、图像、表格、公式等异构内容，并保持其语义关联性。同时支持PDF、Office文档、图像等主流格式的统一处理，提供标准化的多模态内容输出。
+
+**核心组件：**
+
+- **⚙️ 结构化提取引擎**：集成 [MinerU](https://github.com/opendatalab/MinerU) 文档解析框架，实现精确的文档结构识别与内容提取，确保多模态元素的完整性和准确性。
+
+- **🧩 自适应内容分解机制**：建立智能内容分离系统，自动识别并提取文档中的文本块、图像、表格、公式等异构元素，保持元素间的语义关联关系。
+
+- **📁 多格式兼容处理**：部署专业化解析器矩阵，支持PDF、Office文档系列（DOC/DOCX/PPT/PPTX/XLS/XLSX）、图像等主流格式的统一处理与标准化输出。
+
+</div>
+
+### 2. 多模态内容理解与处理
+
+<div style="background: linear-gradient(90deg, #16213e 0%, #0f3460 100%); border-radius: 10px; padding: 20px; margin: 15px 0; border-left: 4px solid #ff6b6b;">
+
+该多模态内容处理系统通过自主分类路由机制实现异构内容的智能识别与优化分发。系统采用并发多流水线架构，确保文本和多模态内容的高效并行处理，在最大化吞吐量的同时保持内容完整性，并能完整提取和保持原始文档的层次结构与元素关联关系。
+
+**核心组件：**
+
+- **🎯 自主内容分类与路由**：自动识别、分类并将不同内容类型路由至优化的执行通道。
+
+- **⚡ 并发多流水线架构**：通过专用处理流水线实现文本和多模态内容的并发执行。这种方法在保持内容完整性的同时最大化吞吐效率。
+
+- **🏗️ 文档层次结构提取**：在内容转换过程中提取并保持原始文档的层次结构和元素间关系。
+
+</div>
+
+### 3. 多模态分析引擎
+
+<div style="background: linear-gradient(90deg, #0f3460 0%, #1a1a2e 100%); border-radius: 10px; padding: 20px; margin: 15px 0; border-left: 4px solid #00d9ff;">
+
+系统部署了面向异构数据模态的模态感知处理单元：
+
+**专用分析器：**
+
+- **🔍 视觉内容分析器**：
+  - 集成视觉模型进行图像分析和内容识别
+  - 基于视觉语义生成上下文感知的描述性标题
+  - 提取视觉元素间的空间关系和层次结构
+
+- **📊 结构化数据解释器**：
+  - 对表格和结构化数据格式进行系统性解释
+  - 实现数据趋势分析的统计模式识别算法
+  - 识别多个表格数据集间的语义关系和依赖性
+
+- **📐 数学表达式解析器**：
+  - 高精度解析复杂数学表达式和公式
+  - 提供原生LaTeX格式支持以实现��学术工作流的无缝集成
+  - 建立数学方程与领域特定知识库间的概念映射
+
+- **🔧 可扩展模态处理器**：
+  - 为自定义和新兴内容类型提供可配置的处理框架
+  - 通过插件架构实现新模态处理器的动态集成
+  - 支持专用场景下处理流水线的运行时配置
+
+</div>
+
+### 4. 多模态知识图谱索引
+
+<div style="background: linear-gradient(90deg, #1a1a2e 0%, #16213e 100%); border-radius: 10px; padding: 20px; margin: 15px 0; border-left: 4px solid #4ecdc4;">
+
+多模态知识图谱构建模块将文档内容转换为结构化语义表示。系统提取多模态实体，建立跨模态关系，并保持层次化组织结构。通过加权相关性评分实现优化的知识检索。
+
+**核心功能：**
+
+- **🔍 多模态实体提取**：将重要的多模态元素转换为结构化知识图谱实体。该过程包括语义标注和元数据保存。
+
+- **🔗 跨模态关系映射**：在文本实体和多模态组件之间建立语义连接和依赖关系。通过自动化关系推理算法实现这一功能。
+
+- **🏗️ 层次结构保持**：通过"归属于"关系链维护原始文档组织结构。这些关系链保持逻辑内容层次和章节依赖关系。
+
+- **⚖️ 加权关系评分**：为关系类型分配定量相关性分数。评分基于语义邻近性和文档结构内的上下文重要性。
+
+</div>
+
+### 5. 模态感知检索
+
+<div style="background: linear-gradient(90deg, #16213e 0%, #0f3460 100%); border-radius: 10px; padding: 20px; margin: 15px 0; border-left: 4px solid #ff6b6b;">
+
+混合检索系统结合向量相似性搜索与图遍历算法，实现全面的内容检索。系统实现模态感知排序机制，并维护检索元素间的关系一致性，确保上下文集成的信息传递。
+
+**检索机制：**
+
+- **🔀 向量-图谱融合**：集成向量相似性搜索与图遍历算法。该方法同时利用语义嵌入和结构关系实现全面的内容检索。
+
+- **📊 模态感知排序**：实现基于内容类型相关性的自适应评分机制。系统根据查询特定的模态偏好调整排序结果。
+
+- **🔗 关系一致性维护**：维护检索元素间的语义和结构关系。确保信息传递的连贯性和上下文完整性。
+
+</div>
+
+---
+
+## 🚀 快速开始
+
+*启动您的AI之旅*
+
+<div align="center">
+  <img src="https://user-images.githubusercontent.com/74038190/212284158-e840e285-664b-44d7-b79b-e264b5e54825.gif" width="400">
+</div>
+
+### 安装
+
+#### 选项1：从PyPI安装（推荐）
+
+```bash
+# 基础安装
+pip install raganything
+
+# 安装包含扩展格式支持的可选依赖：
+pip install 'raganything[all]'              # 所有可选功能
+pip install 'raganything[image]'            # 图像格式转换 (BMP, TIFF, GIF, WebP)
+pip install 'raganything[text]'             # 文本文件处理 (TXT, MD)
+pip install 'raganything[image,text]'       # 多个功能组合
+```
+
+#### 选项2：从源码安装
+
+```bash
+git clone https://github.com/HKUDS/RAG-Anything.git
+cd RAG-Anything
+pip install -e .
+
+# 安装可选依赖
+pip install -e '.[all]'
+```
+
+#### 可选依赖
+
+- **`[image]`** - 启用BMP、TIFF、GIF、WebP图像格式处理（需要Pillow）
+- **`[text]`** - 启用TXT和MD文件处理（需要ReportLab）
+- **`[all]`** - 包含所有Python可选依赖
+
+> **⚠️ Office文档处理配置要求：**
+> - Office文档 (.doc, .docx, .ppt, .pptx, .xls, .xlsx) 需要安装 **LibreOffice**
+> - 从[LibreOffice官网](https://www.libreoffice.org/download/download/)下载安装
+> - **Windows**：从官网下载安装包
+> - **macOS**：`brew install --cask libreoffice`
+> - **Ubuntu/Debian**：`sudo apt-get install libreoffice`
+> - **CentOS/RHEL**：`sudo yum install libreoffice`
+
+**检查MinerU安装：**
+
+```bash
+# 验证安装
+mineru --version
+
+# 检查是否正确配置
+python -c "from raganything import RAGAnything; rag = RAGAnything(); print('✅ MinerU安装正常' if rag.check_parser_installation() else '❌ MinerU安装有问题')"
+```
+
+模型在首次使用时自动下载。手动下载参考[MinerU模型源配置](https://github.com/opendatalab/MinerU/blob/master/README_zh-CN.md#22-%E6%A8%A1%E5%9E%8B%E6%BA%90%E9%85%8D%E7%BD%AE)：
+
+### 使用示例
+
+#### 1. 端到端文档处理
+
+```python
+import asyncio
+from raganything import RAGAnything, RAGAnythingConfig
+from lightrag.llm.openai import openai_complete_if_cache, openai_embed
+from lightrag.utils import EmbeddingFunc
+
+async def main():
+    # 设置 API 配置
+    api_key = "your-api-key"
+    base_url = "your-base-url"  # 可选
+
+    # 创建 RAGAnything 配置
+    config = RAGAnythingConfig(
+        working_dir="./rag_storage",
+        parser="mineru",  # 选择解析器：mineru 或 docling
+        parse_method="auto",  # 解析方法：auto, ocr 或 txt
+        enable_image_processing=True,
+        enable_table_processing=True,
+        enable_equation_processing=True,
+    )
+
+    # 定义 LLM 模型��数
+    def llm_model_func(prompt, system_prompt=None, history_messages=[], **kwargs):
+        return openai_complete_if_cache(
+            "gpt-4o-mini",
+            prompt,
+            system_prompt=system_prompt,
+            history_messages=history_messages,
+            api_key=api_key,
+            base_url=base_url,
+            **kwargs,
+        )
+
+    # 定义视觉模型函数用于图像处理
+    def vision_model_func(
+        prompt, system_prompt=None, history_messages=[], image_data=None, messages=None, **kwargs
+    ):
+        # 如果提供了messages格式（用于多模态VLM增强查询），直接使用
+        if messages:
+            return openai_complete_if_cache(
+                "gpt-4o",
+                "",
+                system_prompt=None,
+                history_messages=[],
+                messages=messages,
+                api_key=api_key,
+                base_url=base_url,
+                **kwargs,
+            )
+        # 传统单图片格式
+        elif image_data:
+            return openai_complete_if_cache(
+                "gpt-4o",
+                "",
+                system_prompt=None,
+                history_messages=[],
+                messages=[
+                    {"role": "system", "content": system_prompt}
+                    if system_prompt
+                    else None,
+                    {
+                        "role": "user",
+                        "content": [
+                            {"type": "text", "text": prompt},
+                            {
+                                "type": "image_url",
+                                "image_url": {
+                                    "url": f"data:image/jpeg;base64,{image_data}"
+                                },
+                            },
+                        ],
+                    }
+                    if image_data
+                    else {"role": "user", "content": prompt},
+                ],
+                api_key=api_key,
+                base_url=base_url,
+                **kwargs,
+            )
+        # 纯文本格式
+        else:
+            return llm_model_func(prompt, system_prompt, history_messages, **kwargs)
+
+    # 定义嵌入函数
+    embedding_func = EmbeddingFunc(
+        embedding_dim=3072,
+        max_token_size=8192,
+        func=lambda texts: openai_embed(
+            texts,
+            model="text-embedding-3-large",
+            api_key=api_key,
+            base_url=base_url,
+        ),
+    )
+
+    # 初始化 RAGAnything
+    rag = RAGAnything(
+        config=config,
+        llm_model_func=llm_model_func,
+        vision_model_func=vision_model_func,
+        embedding_func=embedding_func,
+    )
+
+    # 处理文档
+    await rag.process_document_complete(
+        file_path="path/to/your/document.pdf",
+        output_dir="./output",
+        parse_method="auto"
+    )
+
+    # 查询处理后的内容
+    # 纯文本查询 - 基本知识库搜索
+    text_result = await rag.aquery(
+        "文档的主要内容是什么？",
+        mode="hybrid"
+    )
+    print("文本查询结果:", text_result)
+
+    # 多模态查询 - 包含具体多模态内容的查询
+    multimodal_result = await rag.aquery_with_multimodal(
+        "分析这个性能数据并解释与现有文档内容的关系",
+        multimodal_content=[{
+            "type": "table",
+            "table_data": """系统,准确率,F1分数
+                            RAGAnything,95.2%,0.94
+                            基准方法,87.3%,0.85""",
+            "table_caption": "性能对比结果"
+        }],
+        mode="hybrid"
+    )
+    print("多模态查询结果:", multimodal_result)
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+#### 2. 直接多模态内容处理
+
+```python
+import asyncio
+from lightrag import LightRAG
+from lightrag.llm.openai import openai_complete_if_cache, openai_embed
+from lightrag.utils import EmbeddingFunc
+from raganything.modalprocessors import ImageModalProcessor, TableModalProcessor
+
+async def process_multimodal_content():
+    # 设置 API 配置
+    api_key = "your-api-key"
+    base_url = "your-base-url"  # 可选
+
+    # 初始化 LightRAG
+    rag = LightRAG(
+        working_dir="./rag_storage",
+        llm_model_func=lambda prompt, system_prompt=None, history_messages=[], **kwargs: openai_complete_if_cache(
+            "gpt-4o-mini",
+            prompt,
+            system_prompt=system_prompt,
+            history_messages=history_messages,
+            api_key=api_key,
+            base_url=base_url,
+            **kwargs,
+        ),
+        embedding_func=EmbeddingFunc(
+            embedding_dim=3072,
+            max_token_size=8192,
+            func=lambda texts: openai_embed(
+                texts,
+                model="text-embedding-3-large",
+                api_key=api_key,
+                base_url=base_url,
+            ),
+        )
+    )
+    await rag.initialize_storages()
+
+    # 处理图像
+    image_processor = ImageModalProcessor(
+        lightrag=rag,
+        modal_caption_func=lambda prompt, system_prompt=None, history_messages=[], image_data=None, **kwargs: openai_complete_if_cache(
+            "gpt-4o",
+            "",
+            system_prompt=None,
+            history_messages=[],
+            messages=[
+                {"role": "system", "content": system_prompt} if system_prompt else None,
+                {"role": "user", "content": [
+                    {"type": "text", "text": prompt},
+                    {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{image_data}"}}
+                ]} if image_data else {"role": "user", "content": prompt}
+            ],
+            api_key=api_key,
+            base_url=base_url,
+            **kwargs,
+        ) if image_data else openai_complete_if_cache(
+            "gpt-4o-mini",
+            prompt,
+            system_prompt=system_prompt,
+            history_messages=history_messages,
+            api_key=api_key,
+            base_url=base_url,
+            **kwargs,
+        )
+    )
+
+    image_content = {
+        "img_path": "path/to/image.jpg",
+        "image_caption": ["图1：实验结果"],
+        "image_footnote": ["数据收集于2024年"]
+    }
+
+    description, entity_info = await image_processor.process_multimodal_content(
+        modal_content=image_content,
+        content_type="image",
+        file_path="research_paper.pdf",
+        entity_name="实验结果图表"
+    )
+
+    # 处理表格
+    table_processor = TableModalProcessor(
+        lightrag=rag,
+        modal_caption_func=lambda prompt, system_prompt=None, history_messages=[], **kwargs: openai_complete_if_cache(
+            "gpt-4o-mini",
+            prompt,
+            system_prompt=system_prompt,
+            history_messages=history_messages,
+            api_key=api_key,
+            base_url=base_url,
+            **kwargs,
+        )
+    )
+
+    table_content = {
+        "table_body": """
+        | 方法 | 准确率 | F1分数 |
+        |------|--------|--------|
+        | RAGAnything | 95.2% | 0.94 |
+        | 基准方法 | 87.3% | 0.85 |
+        """,
+        "table_caption": ["性能对比"],
+        "table_footnote": ["测试数据集结果"]
+    }
+
+    description, entity_info = await table_processor.process_multimodal_content(
+        modal_content=table_content,
+        content_type="table",
+        file_path="research_paper.pdf",
+        entity_name="性能结果表格"
+    )
+
+if __name__ == "__main__":
+    asyncio.run(process_multimodal_content())
+```
+
+#### 3. 批量处理
+
+```python
+# 处理多个文档
+await rag.process_folder_complete(
+    folder_path="./documents",
+    output_dir="./output",
+    file_extensions=[".pdf", ".docx", ".pptx"],
+    recursive=True,
+    max_workers=4
+)
+```
+
+#### 4. 自定义模态处理器
+
+```python
+from raganything.modalprocessors import GenericModalProcessor
+
+class CustomModalProcessor(GenericModalProcessor):
+    async def process_multimodal_content(self, modal_content, content_type, file_path, entity_name):
+        # 你的自定义处理逻辑
+        enhanced_description = await self.analyze_custom_content(modal_content)
+        entity_info = self.create_custom_entity(enhanced_description, entity_name)
+        return await self._create_entity_and_chunk(enhanced_description, entity_info, file_path)
+```
+
+#### 5. 查询选项
+
+RAG-Anything 提供三种类型的查询方法：
+
+**纯文本查询** - 使用LightRAG直接进行知识库搜索：
+```python
+# 文本查询的不同模式
+text_result_hybrid = await rag.aquery("你的问题", mode="hybrid")
+text_result_local = await rag.aquery("你的问题", mode="local")
+text_result_global = await rag.aquery("你的问题", mode="global")
+text_result_naive = await rag.aquery("你的问题", mode="naive")
+
+# 同步版本
+sync_text_result = rag.query("你的问题", mode="hybrid")
+```
+
+**VLM增强查询** - 使用VLM自动分析检索上下文中的图像：
+```python
+# VLM增强查询（当提供vision_model_func时自动启用）
+vlm_result = await rag.aquery(
+    "分析文档中的图表和数据",
+    mode="hybrid"
+    # vlm_enhanced=True 当vision_model_func可用时自动设置
+)
+
+# 手动控制VLM增强
+vlm_enabled = await rag.aquery(
+    "这个文档中的图片显示了什么内容？",
+    mode="hybrid",
+    vlm_enhanced=True  # 强制启用VLM增强
+)
+
+vlm_disabled = await rag.aquery(
+    "这个文档中的图片显示了什么内容？",
+    mode="hybrid",
+    vlm_enhanced=False  # 强制禁用VLM增强
+)
+
+# 当文档包含图片时，VLM可以直接查看和分析图片
+# 系统将自动：
+# 1. 检索包含图片路径的相关上下文
+# 2. 加载图片并编码为base64格式
+# 3. 将文本上下文和图片一起发送给VLM进行综合分析
+```
+
+**多模态查询** - 包含特定多模态内容分析的增强查询：
+```python
+# 包含表格数据的查询
+table_result = await rag.aquery_with_multimodal(
+    "比较这些性能指标与文档内容",
+    multimodal_content=[{
+        "type": "table",
+        "table_data": """方法,准确率,速度
+                        LightRAG,95.2%,120ms
+                        传统方法,87.3%,180ms""",
+        "table_caption": "性能对比"
+    }],
+    mode="hybrid"
+)
+
+# 包含公式内容的查询
+equation_result = await rag.aquery_with_multimodal(
+    "解释这个公式及其与文档内容的相关性",
+    multimodal_content=[{
+        "type": "equation",
+        "latex": "P(d|q) = \\frac{P(q|d) \\cdot P(d)}{P(q)}",
+        "equation_caption": "文档相关性概率"
+    }],
+    mode="hybrid"
+)
+```
+
+#### 6. 加载已存在的LightRAG实例
+
+```python
+import asyncio
+from raganything import RAGAnything
+from lightrag import LightRAG
+from lightrag.llm.openai import openai_complete_if_cache, openai_embed
+from lightrag.utils import EmbeddingFunc
+import os
+
+async def load_existing_lightrag():
+    # 设置 API 配置
+    api_key = "your-api-key"
+    base_url = "your-base-url"  # 可选
+
+    # 首先，创建或加载已存在的 LightRAG 实例
+    lightrag_working_dir = "./existing_lightrag_storage"
+
+    # 检查是否存在之前的 LightRAG 实例
+    if os.path.exists(lightrag_working_dir) and os.listdir(lightrag_working_dir):
+        print("✅ 发现已存在的 LightRAG 实例，正在加载...")
+    else:
+        print("❌ 未找到已存在的 LightRAG 实例，将创建新实例")
+
+    # 使用您的配置创建/加载 LightRAG 实例
+    lightrag_instance = LightRAG(
+        working_dir=lightrag_working_dir,
+        llm_model_func=lambda prompt, system_prompt=None, history_messages=[], **kwargs: openai_complete_if_cache(
+            "gpt-4o-mini",
+            prompt,
+            system_prompt=system_prompt,
+            history_messages=history_messages,
+            api_key=api_key,
+            base_url=base_url,
+            **kwargs,
+        ),
+        embedding_func=EmbeddingFunc(
+            embedding_dim=3072,
+            max_token_size=8192,
+            func=lambda texts: openai_embed(
+                texts,
+                model="text-embedding-3-large",
+                api_key=api_key,
+                base_url=base_url,
+            ),
+        )
+    )
+
+    # 初始化存储（如果有现有数据，这将加载它们）
+    await lightrag_instance.initialize_storages()
+    await initialize_pipeline_status()
+
+    # 定义视觉模型函数用于图像处理
+    def vision_model_func(
+        prompt, system_prompt=None, history_messages=[], image_data=None, messages=None, **kwargs
+    ):
+        # 如果提供了messages格式（用于多模态VLM增强查询），直接使用
+        if messages:
+            return openai_complete_if_cache(
+                "gpt-4o",
+                "",
+                system_prompt=None,
+                history_messages=[],
+                messages=messages,
+                api_key=api_key,
+                base_url=base_url,
+                **kwargs,
+            )
+        # 传统单图片格式
+        elif image_data:
+            return openai_complete_if_cache(
+                "gpt-4o",
+                "",
+                system_prompt=None,
+                history_messages=[],
+                messages=[
+                    {"role": "system", "content": system_prompt}
+                    if system_prompt
+                    else None,
+                    {
+                        "role": "user",
+                        "content": [
+                            {"type": "text", "text": prompt},
+                            {
+                                "type": "image_url",
+                                "image_url": {
+                                    "url": f"data:image/jpeg;base64,{image_data}"
+                                },
+                            },
+                        ],
+                    }
+                    if image_data
+                    else {"role": "user", "content": prompt},
+                ],
+                api_key=api_key,
+                base_url=base_url,
+                **kwargs,
+            )
+        # 纯文本格式
+        else:
+            return lightrag_instance.llm_model_func(prompt, system_prompt, history_messages, **kwargs)
+
+    # 现在使用已存在的 LightRAG 实例初始化 RAGAnything
+    rag = RAGAnything(
+        lightrag=lightrag_instance,  # 传入已存在的 LightRAG 实例
+        vision_model_func=vision_model_func,
+        # 注意：working_dir、llm_model_func、embedding_func 等都从 lightrag_instance 继承
+    )
+
+    # 查询已存在的知识库
+    result = await rag.aquery(
+        "这个 LightRAG 实例中处理了哪些数据？",
+        mode="hybrid"
+    )
+    print("查询结果:", result)
+
+    # 向已存在的 LightRAG 实例添加新的多模态文档
+    await rag.process_document_complete(
+        file_path="path/to/new/multimodal_document.pdf",
+        output_dir="./output"
+    )
+
+if __name__ == "__main__":
+    asyncio.run(load_existing_lightrag())
+```
+
+#### 7. 直接插入内容列表
+
+当您已经有预解析的内容列表（例如，来自外部解析器或之前的处理结果）时，可以直接插入到 RAGAnything 中而无需文档解析：
+
+```python
+import asyncio
+from raganything import RAGAnything, RAGAnythingConfig
+from lightrag.llm.openai import openai_complete_if_cache, openai_embed
+from lightrag.utils import EmbeddingFunc
+
+async def insert_content_list_example():
+    # 设置 API 配置
+    api_key = "your-api-key"
+    base_url = "your-base-url"  # 可选
+
+    # 创建 RAGAnything 配置
+    config = RAGAnythingConfig(
+        working_dir="./rag_storage",
+        enable_image_processing=True,
+        enable_table_processing=True,
+        enable_equation_processing=True,
+    )
+
+    # 定义模型函数
+    def llm_model_func(prompt, system_prompt=None, history_messages=[], **kwargs):
+        return openai_complete_if_cache(
+            "gpt-4o-mini",
+            prompt,
+            system_prompt=system_prompt,
+            history_messages=history_messages,
+            api_key=api_key,
+            base_url=base_url,
+            **kwargs,
+        )
+
+    def vision_model_func(prompt, system_prompt=None, history_messages=[], image_data=None, messages=None, **kwargs):
+        # 如果提供了messages格式（用于多模态VLM增强查询），直接使用
+        if messages:
+            return openai_complete_if_cache(
+                "gpt-4o",
+                "",
+                system_prompt=None,
+                history_messages=[],
+                messages=messages,
+                api_key=api_key,
+                base_url=base_url,
+                **kwargs,
+            )
+        # 传统单图片格式
+        elif image_data:
+            return openai_complete_if_cache(
+                "gpt-4o",
+                "",
+                system_prompt=None,
+                history_messages=[],
+                messages=[
+                    {"role": "system", "content": system_prompt} if system_prompt else None,
+                    {
+                        "role": "user",
+                        "content": [
+                            {"type": "text", "text": prompt},
+                            {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{image_data}"}}
+                        ],
+                    } if image_data else {"role": "user", "content": prompt},
+                ],
+                api_key=api_key,
+                base_url=base_url,
+                **kwargs,
+            )
+        # 纯文本格式
+        else:
+            return llm_model_func(prompt, system_prompt, history_messages, **kwargs)
+
+    embedding_func = EmbeddingFunc(
+        embedding_dim=3072,
+        max_token_size=8192,
+        func=lambda texts: openai_embed(
+            texts,
+            model="text-embedding-3-large",
+            api_key=api_key,
+            base_url=base_url,
+        ),
+    )
+
+    # 初始化 RAGAnything
+    rag = RAGAnything(
+        config=config,
+        llm_model_func=llm_model_func,
+        vision_model_func=vision_model_func,
+        embedding_func=embedding_func,
+    )
+
+    # 示例：来自外部源的预解析内容列表
+    content_list = [
+        {
+            "type": "text",
+            "text": "这是我们研究论文的引言部分。",
+            "page_idx": 0  # 此内容出现的页码
+        },
+        {
+            "type": "image",
+            "img_path": "/absolute/path/to/figure1.jpg",  # 重要：使用绝对路径
+            "image_caption": ["图1：系统架构"],
+            "image_footnote": ["来源：作者原创设计"],
+            "page_idx": 1  # 此图像出现的页码
+        },
+        {
+            "type": "table",
+            "table_body": "| 方法 | 准确率 | F1分数 |\n|------|--------|--------|\n| 我们的方法 | 95.2% | 0.94 |\n| 基准方法 | 87.3% | 0.85 |",
+            "table_caption": ["表1：性能对比"],
+            "table_footnote": ["测试数据集结果"],
+            "page_idx": 2  # 此表格出现的页码
+        },
+        {
+            "type": "equation",
+            "latex": "P(d|q) = \\frac{P(q|d) \\cdot P(d)}{P(q)}",
+            "text": "文档相关性概率公式",
+            "page_idx": 3  # 此公式出现的页码
+        },
+        {
+            "type": "text",
+            "text": "总之，我们的方法在所有指标上都表现出优越的性能。",
+            "page_idx": 4  # 此内容出现的页码
+        }
+    ]
+
+    # 直接插入内容列表
+    await rag.insert_content_list(
+        content_list=content_list,
+        file_path="research_paper.pdf",  # 用于引用的参考文件名
+        split_by_character=None,         # 可选的文本分割
+        split_by_character_only=False,   # 可选的文本分割模式
+        doc_id=None,                     # 可选的自定义文档ID（如果未提供将自动生成）
+        display_stats=True               # 显示内容统计信息
+    )
+
+    # 查询��入的内容
+    result = await rag.aquery(
+        "研究中提到的主要发现和性能指标是什么？",
+        mode="hybrid"
+    )
+    print("查询结果:", result)
+
+    # 您也可以使用不同的文档ID插入多个内容列表
+    another_content_list = [
+        {
+            "type": "text",
+            "text": "这是来自另一个文档的内容。",
+            "page_idx": 0  # 此内容出现的页码
+        },
+        {
+            "type": "table",
+            "table_body": "| 特性 | 值 |\n|------|----|\n| 速度 | 快速 |\n| 准确性 | 高 |",
+            "table_caption": ["特性对比"],
+            "page_idx": 1  # 此表格出现的页码
+        }
+    ]
+
+    await rag.insert_content_list(
+        content_list=another_content_list,
+        file_path="another_document.pdf",
+        doc_id="custom-doc-id-123"  # 自定义文档ID
+    )
+
+if __name__ == "__main__":
+    asyncio.run(insert_content_list_example())
+```
+
+**内容列表格式：**
+
+`content_list` 应遵循标准格式，每个项目都是包含以下内容的字典：
+
+- **文本内容**: `{"type": "text", "text": "内容文本", "page_idx": 0}`
+- **图像内容**: `{"type": "image", "img_path": "/absolute/path/to/image.jpg", "image_caption": ["标题"], "image_footnote": ["注释"], "page_idx": 1}`
+- **表格内容**: `{"type": "table", "table_body": "markdown表格", "table_caption": ["标题"], "table_footnote": ["注释"], "page_idx": 2}`
+- **公式内容**: `{"type": "equation", "latex": "LaTeX公式", "text": "描述", "page_idx": 3}`
+- **通用内容**: `{"type": "custom_type", "content": "任何内容", "page_idx": 4}`
+
+**重要说明：**
+- **`img_path`**: 必须是图像文件的绝对路径（例如：`/home/user/images/chart.jpg` 或 `C:\Users\user\images\chart.jpg`）
+- **`page_idx`**: 表示内容在原始文档中出现的页码（从0开始的索引）
+- **内容顺序**: 项目按照在列表中出现的顺序进行处理
+
+此方法在以下情况下特别有用：
+- 您有来自外部解析器的内容（非MinerU/Docling）
+- 您想要处理程序化生成的内容
+- 您需要将来自多个源的内容插入到单个知识库中
+- 您有想要重用的缓存解析结果
+
+---
+
+## 🛠️ 示例
+
+*实际应用演示*
+
+<div align="center">
+  <img src="https://user-images.githubusercontent.com/74038190/212257455-13e3e01e-d6a6-45dc-bb92-3ab87b12dfc1.gif" width="300">
+</div>
+
+`examples/` 目录包含完整的使用示例：
+
+- **`raganything_example.py`**：基于MinerU的端到端文档处理
+- **`modalprocessors_example.py`**：直接多模态内容处理
+- **`office_document_test.py`**：Office文档解析测试（无需API密钥）
+- **`image_format_test.py`**：图像格式解析测试（无需API密钥）
+- **`text_format_test.py`**：文本格式解析测试（无需API密钥）
+
+**运行示例：**
+
+```bash
+# 端到端处理（包含解析器选择）
+python examples/raganything_example.py path/to/document.pdf --api-key YOUR_API_KEY --parser mineru
+
+# 直接模态处理
+python examples/modalprocessors_example.py --api-key YOUR_API_KEY
+
+# Office文档解析测试（仅MinerU功能）
+python examples/office_document_test.py --file path/to/document.docx
+
+# 图像格式解析测试（仅MinerU功能）
+python examples/image_format_test.py --file path/to/image.bmp
+
+# 文本格式解析测试（仅MinerU功能）
+python examples/text_format_test.py --file path/to/document.md
+
+# 检查LibreOffice安装
+python examples/office_document_test.py --check-libreoffice --file dummy
+
+# 检查PIL/Pillow安装
+python examples/image_format_test.py --check-pillow --file dummy
+
+# 检查ReportLab安装
+python examples/text_format_test.py --check-reportlab --file dummy
+```
+
+> **注意**：API密钥仅在完整RAG处理和LLM集成时需要。解析测试文件（`office_document_test.py`、`image_format_test.py` 和 `text_format_test.py`）仅测试MinerU功能，无需API密钥。
+
+---
+
+## 🔧 配置
+
+*系统优化参数*
+
+### 环境变量
+
+创建 `.env` 文件（参考 `.env.example`）：
+
+```bash
+OPENAI_API_KEY=your_openai_api_key
+OPENAI_BASE_URL=your_base_url  # 可选
+OUTPUT_DIR=./output             # 解析文档的默认输出目录
+PARSER=mineru                   # 解析器选择：mineru 或 docling
+PARSE_METHOD=auto              # 解析方法：auto, ocr 或 txt
+```
+
+**注意：** 为了向后兼容，旧的环境变量名称仍然有效：
+- `MINERU_PARSE_METHOD` 已弃用，请使用 `PARSE_METHOD`
+
+### 解析器配置
+
+RAGAnything 现在支持多种解析器，每种解析器都有其特定的优势：
+
+#### MinerU 解析器
+- 支持PDF、图像、Office文档等多种格式
+- 强大的OCR和表格提取能力
+- 支持GPU加速
+
+#### Docling 解析器
+- 专门优化Office文档和HTML文件的解析
+- 更好的文档结构保持
+- 原生支持多种Office格式
+
+### MinerU配置
+
+```bash
+# MinerU 2.0使用命令行参数而不是配置文件
+# 查看可用选项：
+mineru --help
+
+# 常用配置：
+mineru -p input.pdf -o output_dir -m auto    # 自动解析模式
+mineru -p input.pdf -o output_dir -m ocr     # OCR重点解析
+mineru -p input.pdf -o output_dir -b pipeline --device cuda  # GPU加速
+```
+
+你也可以通过RAGAnything参数配置解析：
+
+```python
+# 基础解析配置和解析器选择
+await rag.process_document_complete(
+    file_path="document.pdf",
+    output_dir="./output/",
+    parse_method="auto",          # 或 "ocr", "txt"
+    parser="mineru"               # 可选："mineru" 或 "docling"
+)
+
+# 高级解析配置（包含特殊参数）
+await rag.process_document_complete(
+    file_path="document.pdf",
+    output_dir="./output/",
+    parse_method="auto",          # 解析方法："auto", "ocr", "txt"
+    parser="mineru",              # 解析器选择："mineru" 或 "docling"
+
+    # MinerU特殊参数 - 支持的所有kwargs：
+    lang="ch",                   # 文档语言优化（如："ch", "en", "ja"）
+    device="cuda:0",             # 推理设备："cpu", "cuda", "cuda:0", "npu", "mps"
+    start_page=0,                # 起始页码（0为基准，适用于PDF）
+    end_page=10,                 # 结束页码（0为基准，适用于PDF）
+    formula=True,                # 启用公式解析
+    table=True,                  # 启用表格解析
+    backend="pipeline",          # 解析后端：pipeline|vlm-transformers|vlm-sglang-engine|vlm-sglang-client
+    source="huggingface",        # 模型源："huggingface", "modelscope", "local"
+    # vlm_url="http://127.0.0.1:3000" # 当backend=vlm-sglang-client时，需指定服务地址
+
+    # RAGAnything标准参数
+    display_stats=True,          # 显示内容统计信息
+    split_by_character=None,     # 可选的文本分割字符
+    doc_id=None                  # 可选的文档ID
+)
+```
+
+> **注意**：MinerU 2.0不再使用 `magic-pdf.json` 配置文件。所有设置现在通过命令行参数或函数参数传递。RAG-Anything现在支持多种文档解析器 - 你可以根据需要在MinerU和Docling之间选择。
+
+### 处理要求
+
+不同内容类型需要特定的可选依赖：
+
+- **Office文档** (.doc, .docx, .ppt, .pptx, .xls, .xlsx): 安装并配置 [LibreOffice](https://www.libreoffice.org/download/download/)
+- **扩展图像格式** (.bmp, .tiff, .gif, .webp): 使用 `pip install raganything[image]` 安装
+- **文本文件** (.txt, .md): 使用 `pip install raganything[text]` 安装
+
+> **📋 快速安装**: 使用 `pip install raganything[all]` 启用所有格式支持（仅Python依赖 - LibreOffice仍需单独安装）
+
+---
+
+## 🧪 支持的内容类型
+
+### 文档格式
+
+- **PDF** - 研究论文、报告、演示文稿
+- **Office文档** - DOC、DOCX、PPT、PPTX、XLS、XLSX
+- **图像** - JPG、PNG、BMP、TIFF、GIF、WebP
+- **文本文件** - TXT、MD
+
+### 多模态元素
+
+- **图像** - 照片、图表、示意图、截图
+- **表格** - 数据表、对比图、统计摘要
+- **公式** - LaTeX格式的数学公式
+- **通用内容** - 通过可扩展处理器支持的自定义内容类型
+
+*格式特定依赖的安装说明请参见[配置](#-配置)部分。*
+
+---
+
+## 📖 引用
+
+*学术参考*
+
+<div align="center">
+  <div style="width: 60px; height: 60px; margin: 20px auto; position: relative;">
+    <div style="width: 100%; height: 100%; border: 2px solid #00d9ff; border-radius: 50%; position: relative;">
+      <div style="position: absolute; top: 50%; left: 50%; transform: translate(-50%, -50%); font-size: 24px; color: #00d9ff;">📖</div>
+    </div>
+    <div style="position: absolute; bottom: -5px; left: 50%; transform: translateX(-50%); width: 20px; height: 20px; background: white; border-right: 2px solid #00d9ff; border-bottom: 2px solid #00d9ff; transform: rotate(45deg);"></div>
+  </div>
+</div>
+
+```bibtex
+@article{guo2024lightrag,
+  title={LightRAG: Simple and Fast Retrieval-Augmented Generation},
+  author={Zirui Guo and Lianghao Xia and Yanhua Yu and Tu Ao and Chao Huang},
+  year={2024},
+  eprint={2410.05779},
+  archivePrefix={arXiv},
+  primaryClass={cs.IR}
+}
+```
+
+---
+
+## 🔗 相关项目
+
+*生态系统与扩展*
+
+<div align="center">
+  <table>
+    <tr>
+      <td align="center">
+        <a href="https://github.com/HKUDS/LightRAG">
+          <div style="width: 100px; height: 100px; background: linear-gradient(135deg, rgba(0, 217, 255, 0.1) 0%, rgba(0, 217, 255, 0.05) 100%); border-radius: 15px; border: 1px solid rgba(0, 217, 255, 0.2); display: flex; align-items: center; justify-content: center; margin-bottom: 10px;">
+            <span style="font-size: 32px;">⚡</span>
+          </div>
+          <b>LightRAG</b><br>
+          <sub>简单快速的RAG系统</sub>
+        </a>
+      </td>
+      <td align="center">
+        <a href="https://github.com/HKUDS/VideoRAG">
+          <div style="width: 100px; height: 100px; background: linear-gradient(135deg, rgba(0, 217, 255, 0.1) 0%, rgba(0, 217, 255, 0.05) 100%); border-radius: 15px; border: 1px solid rgba(0, 217, 255, 0.2); display: flex; align-items: center; justify-content: center; margin-bottom: 10px;">
+            <span style="font-size: 32px;">🎥</span>
+          </div>
+          <b>VideoRAG</b><br>
+          <sub>超长上下文视频RAG系统</sub>
+        </a>
+      </td>
+      <td align="center">
+        <a href="https://github.com/HKUDS/MiniRAG">
+          <div style="width: 100px; height: 100px; background: linear-gradient(135deg, rgba(0, 217, 255, 0.1) 0%, rgba(0, 217, 255, 0.05) 100%); border-radius: 15px; border: 1px solid rgba(0, 217, 255, 0.2); display: flex; align-items: center; justify-content: center; margin-bottom: 10px;">
+            <span style="font-size: 32px;">✨</span>
+          </div>
+          <b>MiniRAG</b><br>
+          <sub>极简RAG系统</sub>
+        </a>
+      </td>
+    </tr>
+  </table>
+</div>
+
+---
+
+## ⭐ Star History
+
+*社区增长轨迹*
+
+<div align="center">
+  <a href="https://star-history.com/#HKUDS/RAG-Anything&Date">
+    <picture>
+      <source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=HKUDS/RAG-Anything&type=Date&theme=dark" />
+      <source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/svg?repos=HKUDS/RAG-Anything&type=Date" />
+      <img alt="Star History Chart" src="https://api.star-history.com/svg?repos=HKUDS/RAG-Anything&type=Date" style="border-radius: 15px; box-shadow: 0 0 30px rgba(0, 217, 255, 0.3);" />
+    </picture>
+  </a>
+</div>
+
+---
+
+## 🤝 贡献者
+
+*加入创新*
+
+<div align="center">
+  感谢所有贡献者！
+</div>
+
+<div align="center">
+  <a href="https://github.com/HKUDS/RAG-Anything/graphs/contributors">
+    <img src="https://contrib.rocks/image?repo=HKUDS/RAG-Anything" style="border-radius: 15px; box-shadow: 0 0 20px rgba(0, 217, 255, 0.3);" />
+  </a>
+</div>
+
+---
+
+<div align="center" style="background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); border-radius: 15px; padding: 30px; margin: 30px 0;">
+  <div>
+    <img src="https://user-images.githubusercontent.com/74038190/212284100-561aa473-3905-4a80-b561-0d28506553ee.gif" width="500">
+  </div>
+  <div style="margin-top: 20px;">
+    <a href="https://github.com/HKUDS/RAG-Anything" style="text-decoration: none;">
+      <img src="https://img.shields.io/badge/⭐%20在GitHub上为我们点星-1a1a2e?style=for-the-badge&logo=github&logoColor=white">
+    </a>
+    <a href="https://github.com/HKUDS/RAG-Anything/issues" style="text-decoration: none;">
+      <img src="https://img.shields.io/badge/🐛%20报告问题-ff6b6b?style=for-the-badge&logo=github&logoColor=white">
+    </a>
+    <a href="https://github.com/HKUDS/RAG-Anything/discussions" style="text-decoration: none;">
+      <img src="https://img.shields.io/badge/💬%20讨论交流-4ecdc4?style=for-the-badge&logo=github&logoColor=white">
+    </a>
+  </div>
+</div>
+
+<div align="center">
+  <div style="width: 100%; max-width: 600px; margin: 20px auto; padding: 20px; background: linear-gradient(135deg, rgba(0, 217, 255, 0.1) 0%, rgba(0, 217, 255, 0.05) 100%); border-radius: 15px; border: 1px solid rgba(0, 217, 255, 0.2);">
+    <div style="display: flex; justify-content: center; align-items: center; gap: 15px;">
+      <span style="font-size: 24px;">⭐</span>
+      <span style="color: #00d9ff; font-size: 18px;">感谢您访问RAG-Anything!</span>
+      <span style="font-size: 24px;">⭐</span>
+    </div>
+    <div style="margin-top: 10px; color: #00d9ff; font-size: 16px;">构建多模态AI的未来</div>
+  </div>
+</div>
+
+<div align="center">
+  <img src="https://readme-typing-svg.herokuapp.com?font=Orbitron&size=20&duration=3000&pause=1000&color=00D9FF&center=true&vCenter=true&width=600&lines=感谢您访问RAG-Anything!;构建多模态AI的未来;如果觉得有用请点星⭐!" alt="Closing Animation" />
+</div>
+
+<style>
+@keyframes pulse {
+  0% { transform: scale(1); }
+  50% { transform: scale(1.05); }
+  100% { transform: scale(1); }
+}
+
+@keyframes glow {
+  0% { box-shadow: 0 0 5px rgba(0, 217, 255, 0.5); }
+  50% { box-shadow: 0 0 20px rgba(0, 217, 255, 0.8); }
+  100% { box-shadow: 0 0 5px rgba(0, 217, 255, 0.5); }
+}
+</style>

rag_anything_smaranika/docs/batch_processing.md

@@ -0,0 +1,341 @@
+# Batch Processing
+
+This document describes the batch processing feature for RAG-Anything, which allows you to process multiple documents in parallel for improved throughput.
+
+## Overview
+
+The batch processing feature allows you to process multiple documents concurrently, significantly improving throughput for large document collections. It provides parallel processing, progress tracking, error handling, and flexible configuration options.
+
+## Key Features
+
+- **Parallel Processing**: Process multiple files concurrently using thread pools
+- **Progress Tracking**: Real-time progress bars with `tqdm`
+- **Error Handling**: Comprehensive error reporting and recovery
+- **Flexible Input**: Support for files, directories, and recursive search
+- **Configurable Workers**: Adjustable number of parallel workers
+- **Installation Check Bypass**: Optional skip for environments with package conflicts
+
+## Installation
+
+```bash
+# Basic installation
+pip install raganything[all]
+
+# Required for batch processing
+pip install tqdm
+```
+
+## Usage
+
+### Basic Batch Processing
+
+```python
+from raganything.batch_parser import BatchParser
+
+# Create batch parser
+batch_parser = BatchParser(
+    parser_type="mineru",  # or "docling"
+    max_workers=4,
+    show_progress=True,
+    timeout_per_file=300,
+    skip_installation_check=False  # Set to True if having parser installation issues
+)
+
+# Process multiple files
+result = batch_parser.process_batch(
+    file_paths=["doc1.pdf", "doc2.docx", "folder/"],
+    output_dir="./batch_output",
+    parse_method="auto",
+    recursive=True
+)
+
+# Check results
+print(result.summary())
+print(f"Success rate: {result.success_rate:.1f}%")
+print(f"Processing time: {result.processing_time:.2f} seconds")
+```
+
+### Asynchronous Batch Processing
+
+```python
+import asyncio
+from raganything.batch_parser import BatchParser
+
+async def async_batch_processing():
+    batch_parser = BatchParser(
+        parser_type="mineru",
+        max_workers=4,
+        show_progress=True
+    )
+
+    # Process files asynchronously
+    result = await batch_parser.process_batch_async(
+        file_paths=["doc1.pdf", "doc2.docx"],
+        output_dir="./output",
+        parse_method="auto"
+    )
+
+    return result
+
+# Run async processing
+result = asyncio.run(async_batch_processing())
+```
+
+### Integration with RAG-Anything
+
+```python
+from raganything import RAGAnything
+
+rag = RAGAnything()
+
+# Process documents with batch functionality
+result = rag.process_documents_batch(
+    file_paths=["doc1.pdf", "doc2.docx"],
+    output_dir="./output",
+    max_workers=4,
+    show_progress=True
+)
+
+print(f"Processed {len(result.successful_files)} files successfully")
+```
+
+### Process Documents with RAG Integration
+
+```python
+# Process documents in batch and then add them to RAG
+result = await rag.process_documents_with_rag_batch(
+    file_paths=["doc1.pdf", "doc2.docx"],
+    output_dir="./output",
+    max_workers=4,
+    show_progress=True
+)
+
+print(f"Processed {result['successful_rag_files']} files with RAG")
+print(f"Total processing time: {result['total_processing_time']:.2f} seconds")
+```
+
+### Command Line Interface
+
+```bash
+# Basic batch processing
+python -m raganything.batch_parser path/to/docs/ --output ./output --workers 4
+
+# With specific parser
+python -m raganything.batch_parser path/to/docs/ --parser mineru --method auto
+
+# Without progress bar
+python -m raganything.batch_parser path/to/docs/ --output ./output --no-progress
+
+# Help
+python -m raganything.batch_parser --help
+```
+
+## Configuration
+
+### Environment Variables
+
+```env
+# Batch processing configuration
+MAX_CONCURRENT_FILES=4
+SUPPORTED_FILE_EXTENSIONS=.pdf,.docx,.doc,.pptx,.ppt,.xlsx,.xls,.txt,.md
+RECURSIVE_FOLDER_PROCESSING=true
+PARSER_OUTPUT_DIR=./parsed_output
+```
+
+### BatchParser Parameters
+
+- **parser_type**: `"mineru"` or `"docling"` (default: `"mineru"`)
+- **max_workers**: Number of parallel workers (default: `4`)
+- **show_progress**: Show progress bar (default: `True`)
+- **timeout_per_file**: Timeout per file in seconds (default: `300`)
+- **skip_installation_check**: Skip parser installation check (default: `False`)
+
+## Supported File Types
+
+- **PDF files**: `.pdf`
+- **Office documents**: `.doc`, `.docx`, `.ppt`, `.pptx`, `.xls`, `.xlsx`
+- **Images**: `.png`, `.jpg`, `.jpeg`, `.bmp`, `.tiff`, `.tif`, `.gif`, `.webp`
+- **Text files**: `.txt`, `.md`
+
+## API Reference
+
+### BatchProcessingResult
+
+```python
+@dataclass
+class BatchProcessingResult:
+    successful_files: List[str]      # Successfully processed files
+    failed_files: List[str]          # Failed files
+    total_files: int                 # Total number of files
+    processing_time: float           # Total processing time in seconds
+    errors: Dict[str, str]           # Error messages for failed files
+    output_dir: str                  # Output directory used
+
+    def summary(self) -> str:        # Human-readable summary
+    def success_rate(self) -> float: # Success rate as percentage
+```
+
+### BatchParser Methods
+
+```python
+class BatchParser:
+    def __init__(self, parser_type: str = "mineru", max_workers: int = 4, ...):
+        """Initialize batch parser"""
+
+    def get_supported_extensions(self) -> List[str]:
+        """Get list of supported file extensions"""
+
+    def filter_supported_files(self, file_paths: List[str], recursive: bool = True) -> List[str]:
+        """Filter files to only supported types"""
+
+    def process_batch(self, file_paths: List[str], output_dir: str, ...) -> BatchProcessingResult:
+        """Process files in batch"""
+
+    async def process_batch_async(self, file_paths: List[str], output_dir: str, ...) -> BatchProcessingResult:
+        """Process files in batch asynchronously"""
+```
+
+## Performance Considerations
+
+### Memory Usage
+- Each worker uses additional memory
+- Recommended: 2-4 workers for most systems
+- Monitor memory usage with large files
+
+### CPU Usage
+- Parallel processing utilizes multiple cores
+- Optimal worker count depends on CPU cores and file sizes
+- I/O may become bottleneck with many small files
+
+### Recommended Settings
+- **Small files** (< 1MB): Higher worker count (6-8)
+- **Large files** (> 100MB): Lower worker count (2-3)
+- **Mixed sizes**: Start with 4 workers and adjust
+
+## Troubleshooting
+
+### Common Issues
+
+#### Memory Errors
+```python
+# Solution: Reduce max_workers
+batch_parser = BatchParser(max_workers=2)
+```
+
+#### Timeout Errors
+```python
+# Solution: Increase timeout_per_file
+batch_parser = BatchParser(timeout_per_file=600)  # 10 minutes
+```
+
+#### Parser Installation Issues
+```python
+# Solution: Skip installation check
+batch_parser = BatchParser(skip_installation_check=True)
+```
+
+#### File Not Found Errors
+- Check file paths and permissions
+- Ensure input files exist
+- Verify directory access rights
+
+### Debug Mode
+
+Enable debug logging for detailed information:
+
+```python
+import logging
+logging.basicConfig(level=logging.DEBUG)
+
+# Create batch parser with debug logging
+batch_parser = BatchParser(parser_type="mineru", max_workers=2)
+```
+
+### Error Handling
+
+The batch processor provides comprehensive error handling:
+
+```python
+result = batch_parser.process_batch(file_paths=["doc1.pdf", "doc2.docx"])
+
+# Check for errors
+if result.failed_files:
+    print("Failed files:")
+    for file_path in result.failed_files:
+        error_message = result.errors.get(file_path, "Unknown error")
+        print(f"  - {file_path}: {error_message}")
+
+# Process only successful files
+for file_path in result.successful_files:
+    print(f"Successfully processed: {file_path}")
+```
+
+## Examples
+
+### Process Entire Directory
+
+```python
+from pathlib import Path
+
+# Process all supported files in a directory
+batch_parser = BatchParser(max_workers=4)
+directory_path = Path("./documents")
+
+result = batch_parser.process_batch(
+    file_paths=[str(directory_path)],
+    output_dir="./processed",
+    recursive=True  # Include subdirectories
+)
+
+print(f"Processed {len(result.successful_files)} out of {result.total_files} files")
+```
+
+### Filter Files Before Processing
+
+```python
+# Get all files in directory
+all_files = ["doc1.pdf", "image.png", "spreadsheet.xlsx", "unsupported.xyz"]
+
+# Filter to supported files only
+supported_files = batch_parser.filter_supported_files(all_files)
+print(f"Will process {len(supported_files)} out of {len(all_files)} files")
+
+# Process only supported files
+result = batch_parser.process_batch(
+    file_paths=supported_files,
+    output_dir="./output"
+)
+```
+
+### Custom Error Handling
+
+```python
+def process_with_retry(file_paths, max_retries=3):
+    """Process files with retry logic"""
+
+    for attempt in range(max_retries):
+        result = batch_parser.process_batch(file_paths, "./output")
+
+        if not result.failed_files:
+            break  # All files processed successfully
+
+        print(f"Attempt {attempt + 1}: {len(result.failed_files)} files failed")
+        file_paths = result.failed_files  # Retry failed files
+
+    return result
+```
+
+## Best Practices
+
+1. **Start with default settings** and adjust based on performance
+2. **Monitor system resources** during batch processing
+3. **Use appropriate worker counts** for your hardware
+4. **Handle errors gracefully** with retry logic
+5. **Test with small batches** before processing large collections
+6. **Use skip_installation_check** if facing parser installation issues
+7. **Enable progress tracking** for long-running operations
+8. **Set appropriate timeouts** based on expected file processing times
+
+## Conclusion
+
+The batch processing feature significantly improves RAG-Anything's throughput for large document collections. It provides flexible configuration options, comprehensive error handling, and seamless integration with the existing RAG-Anything pipeline.

rag_anything_smaranika/docs/context_aware_processing.md

@@ -0,0 +1,375 @@
+# Context-Aware Multimodal Processing in RAGAnything
+
+This document describes the context-aware multimodal processing feature in RAGAnything, which provides surrounding content information to LLMs when analyzing images, tables, equations, and other multimodal content for enhanced accuracy and relevance.
+
+## Overview
+
+The context-aware feature enables RAGAnything to automatically extract and provide surrounding text content as context when processing multimodal content. This leads to more accurate and contextually relevant analysis by giving AI models additional information about where the content appears in the document structure.
+
+### Key Benefits
+
+- **Enhanced Accuracy**: Context helps AI understand the purpose and meaning of multimodal content
+- **Semantic Coherence**: Generated descriptions align with document context and terminology
+- **Automated Integration**: Context extraction is automatically enabled during document processing
+- **Flexible Configuration**: Multiple extraction modes and filtering options
+
+## Key Features
+
+### 1. Configuration Support
+- **Integrated Configuration**: Complete context options in `RAGAnythingConfig`
+- **Environment Variables**: Configure all context parameters via environment variables
+- **Dynamic Updates**: Runtime configuration updates supported
+- **Content Format Control**: Configurable content source format detection
+
+### 2. Automated Integration
+- **Auto-Initialization**: Modal processors automatically receive tokenizer and context configuration
+- **Content Source Setup**: Document processing automatically sets content sources for context extraction
+- **Position Information**: Automatic position info (page_idx, index) passed to processors
+- **Batch Processing**: Context-aware batch processing for efficient document handling
+
+### 3. Advanced Token Management
+- **Accurate Token Counting**: Uses LightRAG's tokenizer for precise token calculation
+- **Smart Boundary Preservation**: Truncates at sentence/paragraph boundaries
+- **Backward Compatibility**: Fallback to character truncation when tokenizer unavailable
+
+### 4. Universal Context Extraction
+- **Multiple Formats**: Support for MinerU, plain text, custom formats
+- **Flexible Modes**: Page-based and chunk-based context extraction
+- **Content Filtering**: Configurable content type filtering
+- **Header Support**: Optional inclusion of document headers and structure
+
+## Configuration
+
+### RAGAnythingConfig Parameters
+
+```python
+# Context Extraction Configuration
+context_window: int = 1                    # Context window size (pages/chunks)
+context_mode: str = "page"                 # Context mode ("page" or "chunk")
+max_context_tokens: int = 2000             # Maximum context tokens
+include_headers: bool = True               # Include document headers
+include_captions: bool = True              # Include image/table captions
+context_filter_content_types: List[str] = ["text"]  # Content types to include
+content_format: str = "minerU"             # Default content format for context extraction
+```
+
+### Environment Variables
+
+```bash
+# Context extraction settings
+CONTEXT_WINDOW=2
+CONTEXT_MODE=page
+MAX_CONTEXT_TOKENS=3000
+INCLUDE_HEADERS=true
+INCLUDE_CAPTIONS=true
+CONTEXT_FILTER_CONTENT_TYPES=text,image
+CONTENT_FORMAT=minerU
+```
+
+## Usage Guide
+
+### 1. Basic Configuration
+
+```python
+from raganything import RAGAnything, RAGAnythingConfig
+
+# Create configuration with context settings
+config = RAGAnythingConfig(
+    context_window=2,
+    context_mode="page",
+    max_context_tokens=3000,
+    include_headers=True,
+    include_captions=True,
+    context_filter_content_types=["text", "image"],
+    content_format="minerU"
+)
+
+# Create RAGAnything instance
+rag_anything = RAGAnything(
+    config=config,
+    llm_model_func=your_llm_function,
+    embedding_func=your_embedding_function
+)
+```
+
+### 2. Automatic Document Processing
+
+```python
+# Context is automatically enabled during document processing
+await rag_anything.process_document_complete("document.pdf")
+```
+
+### 3. Manual Content Source Configuration
+
+```python
+# Set content source for specific content lists
+rag_anything.set_content_source_for_context(content_list, "minerU")
+
+# Update context configuration at runtime
+rag_anything.update_context_config(
+    context_window=1,
+    max_context_tokens=1500,
+    include_captions=False
+)
+```
+
+### 4. Direct Modal Processor Usage
+
+```python
+from raganything.modalprocessors import (
+    ContextExtractor,
+    ContextConfig,
+    ImageModalProcessor
+)
+
+# Configure context extraction
+config = ContextConfig(
+    context_window=1,
+    context_mode="page",
+    max_context_tokens=2000,
+    include_headers=True,
+    include_captions=True,
+    filter_content_types=["text"]
+)
+
+# Initialize context extractor
+context_extractor = ContextExtractor(config)
+
+# Initialize modal processor with context support
+processor = ImageModalProcessor(lightrag, caption_func, context_extractor)
+
+# Set content source
+processor.set_content_source(content_list, "minerU")
+
+# Process with context
+item_info = {
+    "page_idx": 2,
+    "index": 5,
+    "type": "image"
+}
+
+result = await processor.process_multimodal_content(
+    modal_content=image_data,
+    content_type="image",
+    file_path="document.pdf",
+    entity_name="Architecture Diagram",
+    item_info=item_info
+)
+```
+
+## Context Modes
+
+### Page-Based Context (`context_mode="page"`)
+- Extracts context based on page boundaries
+- Uses `page_idx` field from content items
+- Suitable for document-structured content
+- Example: Include text from 2 pages before and after current image
+
+### Chunk-Based Context (`context_mode="chunk"`)
+- Extracts context based on content item positions
+- Uses sequential position in content list
+- Suitable for fine-grained control
+- Example: Include 5 content items before and after current table
+
+## Processing Workflow
+
+### 1. Document Parsing
+```
+Document Input → MinerU Parsing → content_list Generation
+```
+
+### 2. Context Setup
+```
+content_list → Set as Context Source → All Modal Processors Gain Context Capability
+```
+
+### 3. Multimodal Processing
+```
+Multimodal Content → Extract Surrounding Context → Enhanced LLM Analysis → More Accurate Results
+```
+
+## Content Source Formats
+
+### MinerU Format
+```json
+[
+    {
+        "type": "text",
+        "text": "Document content here...",
+        "text_level": 1,
+        "page_idx": 0
+    },
+    {
+        "type": "image",
+        "img_path": "images/figure1.jpg",
+        "image_caption": ["Figure 1: Architecture"],
+        "image_footnote": [],
+        "page_idx": 1
+    }
+]
+```
+
+### Custom Text Chunks
+```python
+text_chunks = [
+    "First chunk of text content...",
+    "Second chunk of text content...",
+    "Third chunk of text content..."
+]
+```
+
+### Plain Text
+```python
+full_document = "Complete document text with all content..."
+```
+
+## Configuration Examples
+
+### High-Precision Context
+For focused analysis with minimal context:
+```python
+config = RAGAnythingConfig(
+    context_window=1,
+    context_mode="page",
+    max_context_tokens=1000,
+    include_headers=True,
+    include_captions=False,
+    context_filter_content_types=["text"]
+)
+```
+
+### Comprehensive Context
+For broad analysis with rich context:
+```python
+config = RAGAnythingConfig(
+    context_window=2,
+    context_mode="page",
+    max_context_tokens=3000,
+    include_headers=True,
+    include_captions=True,
+    context_filter_content_types=["text", "image", "table"]
+)
+```
+
+### Chunk-Based Analysis
+For fine-grained sequential context:
+```python
+config = RAGAnythingConfig(
+    context_window=5,
+    context_mode="chunk",
+    max_context_tokens=2000,
+    include_headers=False,
+    include_captions=False,
+    context_filter_content_types=["text"]
+)
+```
+
+## Performance Optimization
+
+### 1. Accurate Token Control
+- Uses real tokenizer for precise token counting
+- Avoids exceeding LLM token limits
+- Provides consistent performance
+
+### 2. Smart Truncation
+- Truncates at sentence boundaries
+- Maintains semantic integrity
+- Adds truncation indicators
+
+### 3. Caching Optimization
+- Context extraction results can be reused
+- Reduces redundant computation overhead
+
+## Advanced Features
+
+### Context Truncation
+The system automatically truncates context to fit within token limits:
+- Uses actual tokenizer for accurate token counting
+- Attempts to end at sentence boundaries (periods)
+- Falls back to line boundaries if needed
+- Adds "..." indicator for truncated content
+
+### Header Formatting
+When `include_headers=True`, headers are formatted with markdown-style prefixes:
+```
+# Level 1 Header
+## Level 2 Header
+### Level 3 Header
+```
+
+### Caption Integration
+When `include_captions=True`, image and table captions are included as:
+```
+[Image: Figure 1 caption text]
+[Table: Table 1 caption text]
+```
+
+## Integration with RAGAnything
+
+The context-aware feature is seamlessly integrated into RAGAnything's workflow:
+
+1. **Automatic Setup**: Context extractors are automatically created and configured
+2. **Content Source Management**: Document processing automatically sets content sources
+3. **Processor Integration**: All modal processors receive context capabilities
+4. **Configuration Consistency**: Single configuration system for all context settings
+
+## Error Handling
+
+The system includes robust error handling:
+- Gracefully handles missing or invalid content sources
+- Returns empty context for unsupported formats
+- Logs warnings for configuration issues
+- Continues processing even if context extraction fails
+
+## Compatibility
+
+- **Backward Compatible**: Existing code works without modification
+- **Optional Feature**: Context can be selectively enabled/disabled
+- **Flexible Configuration**: Supports multiple configuration combinations
+
+## Best Practices
+
+1. **Token Limits**: Ensure `max_context_tokens` doesn't exceed LLM context limits
+2. **Performance Impact**: Larger context windows increase processing time
+3. **Content Quality**: Context quality directly affects analysis accuracy
+4. **Window Size**: Match window size to content structure (documents vs articles)
+5. **Content Filtering**: Use `context_filter_content_types` to reduce noise
+
+## Troubleshooting
+
+### Common Issues
+
+**Context Not Extracted**
+- Check if `set_content_source_for_context()` was called
+- Verify `item_info` contains required fields (`page_idx`, `index`)
+- Confirm content source format is correct
+
+**Context Too Long/Short**
+- Adjust `max_context_tokens` setting
+- Modify `context_window` size
+- Check `context_filter_content_types` configuration
+
+**Irrelevant Context**
+- Refine `context_filter_content_types` to exclude noise
+- Reduce `context_window` size
+- Set `include_captions=False` if captions are not helpful
+
+**Configuration Issues**
+- Verify environment variables are set correctly
+- Check RAGAnythingConfig parameter names
+- Ensure content_format matches your data source
+
+## Examples
+
+Check out these example files for complete usage demonstrations:
+
+- **Configuration Examples**: See how to set up different context configurations
+- **Integration Examples**: Learn how to integrate context-aware processing into your workflow
+- **Custom Processors**: Examples of creating custom modal processors with context support
+
+## API Reference
+
+For detailed API documentation, see the docstrings in:
+- `raganything/modalprocessors.py` - Context extraction and modal processors
+- `raganything/config.py` - Configuration options
+- `raganything/raganything.py` - Main RAGAnything class integration

rag_anything_smaranika/docs/enhanced_markdown.md

@@ -0,0 +1,552 @@
+# Enhanced Markdown Conversion
+
+This document describes the enhanced markdown conversion feature for RAG-Anything, which provides high-quality PDF generation from markdown files with multiple backend options and advanced styling.
+
+## Overview
+
+The enhanced markdown conversion feature provides professional-quality PDF generation from markdown files. It supports multiple conversion backends, advanced styling options, syntax highlighting, and seamless integration with RAG-Anything's document processing pipeline.
+
+## Key Features
+
+- **Multiple Backends**: WeasyPrint, Pandoc, and automatic backend selection
+- **Advanced Styling**: Custom CSS, syntax highlighting, and professional layouts
+- **Image Support**: Embedded images with proper scaling and positioning
+- **Table Support**: Formatted tables with borders and professional styling
+- **Code Highlighting**: Syntax highlighting for code blocks using Pygments
+- **Custom Templates**: Support for custom CSS and document templates
+- **Table of Contents**: Automatic TOC generation with navigation links
+- **Professional Typography**: High-quality fonts and spacing
+
+## Installation
+
+### Required Dependencies
+
+```bash
+# Basic installation
+pip install raganything[all]
+
+# Required for enhanced markdown conversion
+pip install markdown weasyprint pygments
+```
+
+### Optional Dependencies
+
+```bash
+# For Pandoc backend (system installation required)
+# Ubuntu/Debian:
+sudo apt-get install pandoc wkhtmltopdf
+
+# macOS:
+brew install pandoc wkhtmltopdf
+
+# Or using conda:
+conda install -c conda-forge pandoc wkhtmltopdf
+```
+
+### Backend-Specific Installation
+
+#### WeasyPrint (Recommended)
+```bash
+# Install WeasyPrint with system dependencies
+pip install weasyprint
+
+# Ubuntu/Debian system dependencies:
+sudo apt-get install -y build-essential python3-dev python3-pip \
+    python3-setuptools python3-wheel python3-cffi libcairo2 \
+    libpango-1.0-0 libpangocairo-1.0-0 libgdk-pixbuf2.0-0 \
+    libffi-dev shared-mime-info
+```
+
+#### Pandoc
+- Download from: https://pandoc.org/installing.html
+- Requires system-wide installation
+- Used for complex document structures and LaTeX-quality output
+
+## Usage
+
+### Basic Conversion
+
+```python
+from raganything.enhanced_markdown import EnhancedMarkdownConverter, MarkdownConfig
+
+# Create converter with default settings
+converter = EnhancedMarkdownConverter()
+
+# Convert markdown file to PDF
+success = converter.convert_file_to_pdf(
+    input_path="document.md",
+    output_path="document.pdf",
+    method="auto"  # Automatically select best available backend
+)
+
+if success:
+    print("✅ Conversion successful!")
+else:
+    print("❌ Conversion failed")
+```
+
+### Advanced Configuration
+
+```python
+# Create custom configuration
+config = MarkdownConfig(
+    page_size="A4",           # A4, Letter, Legal, etc.
+    margin="1in",             # CSS-style margins
+    font_size="12pt",         # Base font size
+    line_height="1.5",        # Line spacing
+    include_toc=True,         # Generate table of contents
+    syntax_highlighting=True, # Enable code syntax highlighting
+
+    # Custom CSS styling
+    custom_css="""
+    body {
+        font-family: 'Georgia', serif;
+        color: #333;
+    }
+    h1 {
+        color: #2c3e50;
+        border-bottom: 2px solid #3498db;
+        padding-bottom: 0.3em;
+    }
+    code {
+        background-color: #f8f9fa;
+        padding: 2px 4px;
+        border-radius: 3px;
+    }
+    pre {
+        background-color: #f8f9fa;
+        border-left: 4px solid #3498db;
+        padding: 15px;
+        border-radius: 5px;
+    }
+    table {
+        border-collapse: collapse;
+        width: 100%;
+        margin: 1em 0;
+    }
+    th, td {
+        border: 1px solid #ddd;
+        padding: 8px 12px;
+        text-align: left;
+    }
+    th {
+        background-color: #f2f2f2;
+        font-weight: bold;
+    }
+    """
+)
+
+converter = EnhancedMarkdownConverter(config)
+```
+
+### Backend Selection
+
+```python
+# Check available backends
+converter = EnhancedMarkdownConverter()
+backend_info = converter.get_backend_info()
+
+print("Available backends:")
+for backend, available in backend_info["available_backends"].items():
+    status = "✅" if available else "❌"
+    print(f"  {status} {backend}")
+
+print(f"Recommended backend: {backend_info['recommended_backend']}")
+
+# Use specific backend
+converter.convert_file_to_pdf(
+    input_path="document.md",
+    output_path="document.pdf",
+    method="weasyprint"  # or "pandoc", "pandoc_system", "auto"
+)
+```
+
+### Content Conversion
+
+```python
+# Convert markdown content directly (not from file)
+markdown_content = """
+# Sample Document
+
+## Introduction
+This is a **bold** statement with *italic* text.
+
+## Code Example
+```python
+def hello_world():
+    print("Hello, World!")
+    return "Success"
+```
+
+## Table
+| Feature | Status | Notes |
+|---------|--------|-------|
+| PDF Generation | ✅ | Working |
+| Syntax Highlighting | ✅ | Pygments |
+| Custom CSS | ✅ | Full support |
+"""
+
+success = converter.convert_markdown_to_pdf(
+    markdown_content=markdown_content,
+    output_path="sample.pdf",
+    method="auto"
+)
+```
+
+### Command Line Interface
+
+```bash
+# Basic conversion
+python -m raganything.enhanced_markdown document.md --output document.pdf
+
+# With specific backend
+python -m raganything.enhanced_markdown document.md --method weasyprint
+
+# With custom CSS file
+python -m raganything.enhanced_markdown document.md --css custom_style.css
+
+# Show backend information
+python -m raganything.enhanced_markdown --info
+
+# Help
+python -m raganything.enhanced_markdown --help
+```
+
+## Backend Comparison
+
+| Backend | Pros | Cons | Best For | Quality |
+|---------|------|------|----------|---------|
+| **WeasyPrint** | • Excellent CSS support<br>• Fast rendering<br>• Great web-style layouts<br>• Python-based | • Limited LaTeX features<br>• Requires system deps | • Web-style documents<br>• Custom styling<br>• Fast conversion | ⭐⭐⭐⭐ |
+| **Pandoc** | • Extensive features<br>• LaTeX-quality output<br>• Academic formatting<br>• Many input/output formats | • Slower conversion<br>• System installation<br>• Complex setup | • Academic papers<br>• Complex documents<br>• Publication quality | ⭐⭐⭐⭐⭐ |
+| **Auto** | • Automatic selection<br>• Fallback support<br>• User-friendly | • May not use optimal backend | • General use<br>• Quick setup<br>• Development | ⭐⭐⭐⭐ |
+
+## Configuration Options
+
+### MarkdownConfig Parameters
+
+```python
+@dataclass
+class MarkdownConfig:
+    # Page layout
+    page_size: str = "A4"              # A4, Letter, Legal, A3, etc.
+    margin: str = "1in"                # CSS margin format
+    font_size: str = "12pt"            # Base font size
+    line_height: str = "1.5"           # Line spacing multiplier
+
+    # Content options
+    include_toc: bool = True           # Generate table of contents
+    syntax_highlighting: bool = True   # Enable code highlighting
+    image_max_width: str = "100%"      # Maximum image width
+    table_style: str = "..."           # Default table CSS
+
+    # Styling
+    css_file: Optional[str] = None     # External CSS file path
+    custom_css: Optional[str] = None   # Inline CSS content
+    template_file: Optional[str] = None # Custom HTML template
+
+    # Output options
+    output_format: str = "pdf"         # Currently only PDF supported
+    output_dir: Optional[str] = None   # Output directory
+
+    # Metadata
+    metadata: Optional[Dict[str, str]] = None  # Document metadata
+```
+
+### Supported Markdown Features
+
+#### Basic Formatting
+- **Headers**: `# ## ### #### ##### ######`
+- **Emphasis**: `*italic*`, `**bold**`, `***bold italic***`
+- **Links**: `[text](url)`, `[text][ref]`
+- **Images**: `![alt](url)`, `![alt][ref]`
+- **Lists**: Ordered and unordered, nested
+- **Blockquotes**: `> quote`
+- **Line breaks**: Double space or `\n\n`
+
+#### Advanced Features
+- **Tables**: GitHub-style tables with alignment
+- **Code blocks**: Fenced code blocks with language specification
+- **Inline code**: `backtick code`
+- **Horizontal rules**: `---` or `***`
+- **Footnotes**: `[^1]` references
+- **Definition lists**: Term and definition pairs
+- **Attributes**: `{#id .class key=value}`
+
+#### Code Highlighting
+
+```markdown
+```python
+def example_function():
+    """This will be syntax highlighted"""
+    return "Hello, World!"
+```
+
+```javascript
+function exampleFunction() {
+    // This will also be highlighted
+    return "Hello, World!";
+}
+```
+```
+
+## Integration with RAG-Anything
+
+The enhanced markdown conversion integrates seamlessly with RAG-Anything:
+
+```python
+from raganything import RAGAnything
+
+# Initialize RAG-Anything
+rag = RAGAnything()
+
+# Process markdown files - enhanced conversion is used automatically
+await rag.process_document_complete("document.md")
+
+# Batch processing with enhanced markdown conversion
+result = rag.process_documents_batch(
+    file_paths=["doc1.md", "doc2.md", "doc3.md"],
+    output_dir="./output"
+)
+
+# The .md files will be converted to PDF using enhanced conversion
+# before being processed by the RAG system
+```
+
+## Performance Considerations
+
+### Conversion Speed
+- **WeasyPrint**: ~1-3 seconds for typical documents
+- **Pandoc**: ~3-10 seconds for typical documents
+- **Large documents**: Time scales roughly linearly with content
+
+### Memory Usage
+- **WeasyPrint**: ~50-100MB per conversion
+- **Pandoc**: ~100-200MB per conversion
+- **Images**: Large images increase memory usage significantly
+
+### Optimization Tips
+1. **Resize large images** before embedding
+2. **Use compressed images** (JPEG for photos, PNG for graphics)
+3. **Limit concurrent conversions** to avoid memory issues
+4. **Cache converted content** when processing multiple times
+
+## Examples
+
+### Sample Markdown Document
+
+```markdown
+# Technical Documentation
+
+## Table of Contents
+[TOC]
+
+## Overview
+This document provides comprehensive technical specifications.
+
+## Architecture
+
+### System Components
+1. **Parser Engine**: Handles document processing
+2. **Storage Layer**: Manages data persistence
+3. **Query Interface**: Provides search capabilities
+
+### Code Implementation
+```python
+from raganything import RAGAnything
+
+# Initialize system
+rag = RAGAnything(config={
+    "working_dir": "./storage",
+    "enable_image_processing": True
+})
+
+# Process document
+await rag.process_document_complete("document.pdf")
+```
+
+### Performance Metrics
+
+| Component | Throughput | Latency | Memory |
+|-----------|------------|---------|--------|
+| Parser | 100 docs/hour | 36s avg | 2.5 GB |
+| Storage | 1000 ops/sec | 1ms avg | 512 MB |
+| Query | 50 queries/sec | 20ms avg | 1 GB |
+
+## Integration Notes
+
+> **Important**: Always validate input before processing.
+
+## Conclusion
+The enhanced system provides excellent performance for document processing workflows.
+```
+
+### Generated PDF Features
+
+The enhanced markdown converter produces PDFs with:
+
+- **Professional typography** with proper font selection and spacing
+- **Syntax-highlighted code blocks** using Pygments
+- **Formatted tables** with borders and alternating row colors
+- **Clickable table of contents** with navigation links
+- **Responsive images** that scale appropriately
+- **Custom styling** through CSS
+- **Proper page breaks** and margins
+- **Document metadata** and properties
+
+## Troubleshooting
+
+### Common Issues
+
+#### WeasyPrint Installation Problems
+```bash
+# Ubuntu/Debian: Install system dependencies
+sudo apt-get update
+sudo apt-get install -y build-essential python3-dev libcairo2 \
+    libpango-1.0-0 libpangocairo-1.0-0 libgdk-pixbuf2.0-0 \
+    libffi-dev shared-mime-info
+
+# Then reinstall WeasyPrint
+pip install --force-reinstall weasyprint
+```
+
+#### Pandoc Not Found
+```bash
+# Check if Pandoc is installed
+pandoc --version
+
+# Install Pandoc (Ubuntu/Debian)
+sudo apt-get install pandoc wkhtmltopdf
+
+# Or download from: https://pandoc.org/installing.html
+```
+
+#### CSS Issues
+- Check CSS syntax in custom_css
+- Verify CSS file paths exist
+- Test CSS with simple HTML first
+- Use browser developer tools to debug styling
+
+#### Image Problems
+- Ensure images are accessible (correct paths)
+- Check image file formats (PNG, JPEG, GIF supported)
+- Verify image file permissions
+- Consider image size and format optimization
+
+#### Font Issues
+```python
+# Use web-safe fonts
+config = MarkdownConfig(
+    custom_css="""
+    body {
+        font-family: 'Arial', 'Helvetica', sans-serif;
+    }
+    """
+)
+```
+
+### Debug Mode
+
+Enable detailed logging for troubleshooting:
+
+```python
+import logging
+
+# Enable debug logging
+logging.basicConfig(
+    level=logging.DEBUG,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+)
+
+# Create converter with debug logging
+converter = EnhancedMarkdownConverter()
+result = converter.convert_file_to_pdf("test.md", "test.pdf")
+```
+
+### Error Handling
+
+```python
+def robust_conversion(input_path, output_path):
+    """Convert with fallback backends"""
+    converter = EnhancedMarkdownConverter()
+
+    # Try backends in order of preference
+    backends = ["weasyprint", "pandoc", "auto"]
+
+    for backend in backends:
+        try:
+            success = converter.convert_file_to_pdf(
+                input_path=input_path,
+                output_path=output_path,
+                method=backend
+            )
+            if success:
+                print(f"✅ Conversion successful with {backend}")
+                return True
+        except Exception as e:
+            print(f"❌ {backend} failed: {str(e)}")
+            continue
+
+    print("❌ All backends failed")
+    return False
+```
+
+## API Reference
+
+### EnhancedMarkdownConverter
+
+```python
+class EnhancedMarkdownConverter:
+    def __init__(self, config: Optional[MarkdownConfig] = None):
+        """Initialize converter with optional configuration"""
+
+    def convert_file_to_pdf(self, input_path: str, output_path: str, method: str = "auto") -> bool:
+        """Convert markdown file to PDF"""
+
+    def convert_markdown_to_pdf(self, markdown_content: str, output_path: str, method: str = "auto") -> bool:
+        """Convert markdown content to PDF"""
+
+    def get_backend_info(self) -> Dict[str, Any]:
+        """Get information about available backends"""
+
+    def convert_with_weasyprint(self, markdown_content: str, output_path: str) -> bool:
+        """Convert using WeasyPrint backend"""
+
+    def convert_with_pandoc(self, markdown_content: str, output_path: str) -> bool:
+        """Convert using Pandoc backend"""
+```
+
+## Best Practices
+
+1. **Choose the right backend** for your use case:
+   - **WeasyPrint** for web-style documents and custom CSS
+   - **Pandoc** for academic papers and complex formatting
+   - **Auto** for general use and development
+
+2. **Optimize images** before embedding:
+   - Use appropriate formats (JPEG for photos, PNG for graphics)
+   - Compress images to reduce file size
+   - Set reasonable maximum widths
+
+3. **Design responsive layouts**:
+   - Use relative units (%, em) instead of absolute (px)
+   - Test with different page sizes
+   - Consider print-specific CSS
+
+4. **Test your styling**:
+   - Start with default styling and incrementally customize
+   - Test with sample content before production use
+   - Validate CSS syntax
+
+5. **Handle errors gracefully**:
+   - Implement fallback backends
+   - Provide meaningful error messages
+   - Log conversion attempts for debugging
+
+6. **Performance optimization**:
+   - Cache converted content when possible
+   - Process large batches with appropriate worker counts
+   - Monitor memory usage with large documents
+
+## Conclusion
+
+The enhanced markdown conversion feature provides professional-quality PDF generation with flexible styling options and multiple backend support. It seamlessly integrates with RAG-Anything's document processing pipeline while offering standalone functionality for markdown-to-PDF conversion needs.

rag_anything_smaranika/env.example

@@ -0,0 +1,192 @@
+### This is sample file of .env
+
+
+### Server Configuration
+HOST=0.0.0.0
+PORT=9621
+WEBUI_TITLE='My Graph KB'
+WEBUI_DESCRIPTION="Simple and Fast Graph Based RAG System"
+OLLAMA_EMULATING_MODEL_TAG=latest
+# WORKERS=2
+# CORS_ORIGINS=http://localhost:3000,http://localhost:8080
+
+### Login Configuration
+# AUTH_ACCOUNTS='admin:admin123,user1:pass456'
+# TOKEN_SECRET=Your-Key-For-LightRAG-API-Server
+# TOKEN_EXPIRE_HOURS=48
+# GUEST_TOKEN_EXPIRE_HOURS=24
+# JWT_ALGORITHM=HS256
+
+### API-Key to access LightRAG Server API
+# LIGHTRAG_API_KEY=your-secure-api-key-here
+# WHITELIST_PATHS=/health,/api/*
+
+### Optional SSL Configuration
+# SSL=true
+# SSL_CERTFILE=/path/to/cert.pem
+# SSL_KEYFILE=/path/to/key.pem
+
+### Directory Configuration (defaults to current working directory)
+### Should not be set if deploy by docker (Set by Dockerfile instead of .env)
+### Default value is ./inputs and ./rag_storage
+# INPUT_DIR=<absolute_path_for_doc_input_dir>
+
+### RAGAnything Configuration (Multimodal Document Processing)
+### ---
+### Parser Configuration
+# PARSE_METHOD=auto
+# OUTPUT_DIR=./output
+# PARSER=mineru
+# DISPLAY_CONTENT_STATS=true
+
+### Multimodal Processing Configuration
+# ENABLE_IMAGE_PROCESSING=true
+# ENABLE_TABLE_PROCESSING=true
+# ENABLE_EQUATION_PROCESSING=true
+
+### Batch Processing Configuration
+# MAX_CONCURRENT_FILES=1
+# SUPPORTED_FILE_EXTENSIONS=.pdf,.jpg,.jpeg,.png,.bmp,.tiff,.tif,.gif,.webp,.doc,.docx,.ppt,.pptx,.xls,.xlsx,.txt,.md
+# RECURSIVE_FOLDER_PROCESSING=true
+
+### Context Extraction Configuration
+# CONTEXT_WINDOW=1
+# CONTEXT_MODE=page
+# MAX_CONTEXT_TOKENS=2000
+# INCLUDE_HEADERS=true
+# INCLUDE_CAPTIONS=true
+# CONTEXT_FILTER_CONTENT_TYPES=text
+# CONTENT_FORMAT=minerU
+
+### Max nodes return from grap retrieval
+# MAX_GRAPH_NODES=1000
+
+### Logging level
+# LOG_LEVEL=INFO
+# VERBOSE=False
+# LOG_MAX_BYTES=10485760
+# LOG_BACKUP_COUNT=5
+### Logfile location (defaults to current working directory)
+# LOG_DIR=/path/to/log/directory
+
+### Settings for RAG query
+# HISTORY_TURNS=3
+# COSINE_THRESHOLD=0.2
+# TOP_K=60
+# MAX_TOKEN_TEXT_CHUNK=4000
+# MAX_TOKEN_RELATION_DESC=4000
+# MAX_TOKEN_ENTITY_DESC=4000
+
+### Entity and ralation summarization configuration
+### Language: English, Chinese, French, German ...
+SUMMARY_LANGUAGE=English
+### Number of duplicated entities/edges to trigger LLM re-summary on merge ( at least 3 is recommented)
+# FORCE_LLM_SUMMARY_ON_MERGE=6
+### Max tokens for entity/relations description after merge
+# MAX_TOKEN_SUMMARY=500
+
+### Number of parallel processing documents(Less than MAX_ASYNC/2 is recommended)
+# MAX_PARALLEL_INSERT=2
+### Chunk size for document splitting, 500~1500 is recommended
+# CHUNK_SIZE=1200
+# CHUNK_OVERLAP_SIZE=100
+
+### LLM Configuration
+ENABLE_LLM_CACHE=true
+ENABLE_LLM_CACHE_FOR_EXTRACT=true
+### Time out in seconds for LLM, None for infinite timeout
+TIMEOUT=240
+### Some models like o1-mini require temperature to be set to 1
+TEMPERATURE=0
+### Max concurrency requests of LLM
+MAX_ASYNC=4
+### MAX_TOKENS: max tokens send to LLM for entity relation summaries (less than context size of the model)
+### MAX_TOKENS: set as num_ctx option for Ollama by API Server
+MAX_TOKENS=32768
+### LLM Binding type: openai, ollama, lollms, azure_openai, lmstudio
+LLM_BINDING=openai
+LLM_MODEL=gpt-4o
+LLM_BINDING_HOST=https://api.openai.com/v1
+LLM_BINDING_API_KEY=your_api_key
+### Optional for Azure
+# AZURE_OPENAI_API_VERSION=2024-08-01-preview
+# AZURE_OPENAI_DEPLOYMENT=gpt-4o
+
+### Embedding Configuration
+### Embedding Binding type: openai, ollama, lollms, azure_openai, lmstudio
+EMBEDDING_BINDING=ollama
+EMBEDDING_MODEL=bge-m3:latest
+EMBEDDING_DIM=1024
+EMBEDDING_BINDING_API_KEY=your_api_key
+# If the embedding service is deployed within the same Docker stack, use host.docker.internal instead of localhost
+EMBEDDING_BINDING_HOST=http://localhost:11434
+### Num of chunks send to Embedding in single request
+# EMBEDDING_BATCH_NUM=32
+### Max concurrency requests for Embedding
+# EMBEDDING_FUNC_MAX_ASYNC=16
+### Maximum tokens sent to Embedding for each chunk (no longer in use?)
+# MAX_EMBED_TOKENS=8192
+### Optional for Azure
+# AZURE_EMBEDDING_DEPLOYMENT=text-embedding-3-large
+# AZURE_EMBEDDING_API_VERSION=2023-05-15
+
+### Data storage selection
+# LIGHTRAG_KV_STORAGE=PGKVStorage
+# LIGHTRAG_VECTOR_STORAGE=PGVectorStorage
+# LIGHTRAG_DOC_STATUS_STORAGE=PGDocStatusStorage
+# LIGHTRAG_GRAPH_STORAGE=Neo4JStorage
+
+### TiDB Configuration (Deprecated)
+# TIDB_HOST=localhost
+# TIDB_PORT=4000
+# TIDB_USER=your_username
+# TIDB_PASSWORD='your_password'
+# TIDB_DATABASE=your_database
+### separating all data from difference Lightrag instances(deprecating)
+# TIDB_WORKSPACE=default
+
+### PostgreSQL Configuration
+POSTGRES_HOST=localhost
+POSTGRES_PORT=5432
+POSTGRES_USER=your_username
+POSTGRES_PASSWORD='your_password'
+POSTGRES_DATABASE=your_database
+POSTGRES_MAX_CONNECTIONS=12
+### separating all data from difference Lightrag instances(deprecating)
+# POSTGRES_WORKSPACE=default
+
+### Neo4j Configuration
+NEO4J_URI=neo4j+s://xxxxxxxx.databases.neo4j.io
+NEO4J_USERNAME=neo4j
+NEO4J_PASSWORD='your_password'
+
+### Independent AGM Configuration(not for AMG embedded in PostreSQL)
+# AGE_POSTGRES_DB=
+# AGE_POSTGRES_USER=
+# AGE_POSTGRES_PASSWORD=
+# AGE_POSTGRES_HOST=
+# AGE_POSTGRES_PORT=8529
+
+# AGE Graph Name(apply to PostgreSQL and independent AGM)
+### AGE_GRAPH_NAME is precated
+# AGE_GRAPH_NAME=lightrag
+
+### MongoDB Configuration
+MONGO_URI=mongodb://root:root@localhost:27017/
+MONGO_DATABASE=LightRAG
+### separating all data from difference Lightrag instances(deprecating)
+# MONGODB_GRAPH=false
+
+### Milvus Configuration
+MILVUS_URI=http://localhost:19530
+MILVUS_DB_NAME=lightrag
+# MILVUS_USER=root
+# MILVUS_PASSWORD=your_password
+# MILVUS_TOKEN=your_token
+
+### Qdrant
+QDRANT_URL=http://localhost:16333
+# QDRANT_API_KEY=your-api-key
+
+### Redis
+REDIS_URI=redis://localhost:6379

rag_anything_smaranika/examples/batch_processing_example.py

@@ -0,0 +1,561 @@
+#!/usr/bin/env python
+"""
+Batch Processing Example for RAG-Anything
+
+This example demonstrates how to use the batch processing capabilities
+to process multiple documents in parallel for improved throughput.
+
+Features demonstrated:
+- Basic batch processing with BatchParser
+- Asynchronous batch processing
+- Integration with RAG-Anything
+- Error handling and progress tracking
+- File filtering and directory processing
+"""
+
+import asyncio
+import logging
+from pathlib import Path
+import tempfile
+import time
+
+# Add project root directory to Python path
+import sys
+
+sys.path.append(str(Path(__file__).parent.parent))
+
+from raganything import RAGAnything, RAGAnythingConfig
+from raganything.batch_parser import BatchParser
+
+
+def create_sample_documents():
+    """Create sample documents for batch processing testing"""
+    temp_dir = Path(tempfile.mkdtemp())
+    sample_files = []
+
+    # Create various document types
+    documents = {
+        "document1.txt": "This is a simple text document for testing batch processing.",
+        "document2.txt": "Another text document with different content.",
+        "document3.md": """# Markdown Document
+
+## Introduction
+This is a markdown document for testing.
+
+### Features
+- Markdown formatting
+- Code blocks
+- Lists
+
+```python
+def example():
+    return "Hello from markdown"
+```
+""",
+        "report.txt": """Business Report
+
+Executive Summary:
+This report demonstrates batch processing capabilities.
+
+Key Findings:
+1. Parallel processing improves throughput
+2. Progress tracking enhances user experience
+3. Error handling ensures reliability
+
+Conclusion:
+Batch processing is essential for large-scale document processing.
+""",
+        "notes.md": """# Meeting Notes
+
+## Date: 2024-01-15
+
+### Attendees
+- Alice Johnson
+- Bob Smith
+- Carol Williams
+
+### Discussion Topics
+1. **Batch Processing Implementation**
+   - Parallel document processing
+   - Progress tracking
+   - Error handling strategies
+
+2. **Performance Metrics**
+   - Target: 100 documents/hour
+   - Memory usage: < 4GB
+   - Success rate: > 95%
+
+### Action Items
+- [ ] Implement batch processing
+- [ ] Add progress bars
+- [ ] Test with large document sets
+- [ ] Optimize memory usage
+
+### Next Steps
+Continue development and testing of batch processing features.
+""",
+    }
+
+    # Create files
+    for filename, content in documents.items():
+        file_path = temp_dir / filename
+        with open(file_path, "w", encoding="utf-8") as f:
+            f.write(content)
+        sample_files.append(str(file_path))
+
+    return sample_files, temp_dir
+
+
+def demonstrate_basic_batch_processing():
+    """Demonstrate basic batch processing functionality"""
+    print("\n" + "=" * 60)
+    print("BASIC BATCH PROCESSING DEMONSTRATION")
+    print("=" * 60)
+
+    # Create sample documents
+    sample_files, temp_dir = create_sample_documents()
+
+    try:
+        print(f"Created {len(sample_files)} sample documents in: {temp_dir}")
+        for file_path in sample_files:
+            print(f"  - {Path(file_path).name}")
+
+        # Create batch parser
+        batch_parser = BatchParser(
+            parser_type="mineru",
+            max_workers=3,
+            show_progress=True,
+            timeout_per_file=60,
+            skip_installation_check=True,  # Skip installation check for demo
+        )
+
+        print("\nBatch parser configured:")
+        print("  - Parser type: mineru")
+        print("  - Max workers: 3")
+        print("  - Progress tracking: enabled")
+        print("  - Timeout per file: 60 seconds")
+
+        # Check supported extensions
+        supported_extensions = batch_parser.get_supported_extensions()
+        print(f"  - Supported extensions: {supported_extensions}")
+
+        # Filter files to supported types
+        supported_files = batch_parser.filter_supported_files(sample_files)
+        print("\nFile filtering results:")
+        print(f"  - Total files: {len(sample_files)}")
+        print(f"  - Supported files: {len(supported_files)}")
+
+        # Process batch
+        output_dir = temp_dir / "batch_output"
+        print("\nStarting batch processing...")
+        print(f"Output directory: {output_dir}")
+
+        start_time = time.time()
+        result = batch_parser.process_batch(
+            file_paths=supported_files,
+            output_dir=str(output_dir),
+            parse_method="auto",
+            recursive=False,
+        )
+        processing_time = time.time() - start_time
+
+        # Display results
+        print("\n" + "-" * 40)
+        print("BATCH PROCESSING RESULTS")
+        print("-" * 40)
+        print(result.summary())
+        print(f"Total processing time: {processing_time:.2f} seconds")
+        print(f"Success rate: {result.success_rate:.1f}%")
+
+        if result.successful_files:
+            print("\nSuccessfully processed files:")
+            for file_path in result.successful_files:
+                print(f"  ✅ {Path(file_path).name}")
+
+        if result.failed_files:
+            print("\nFailed files:")
+            for file_path in result.failed_files:
+                error = result.errors.get(file_path, "Unknown error")
+                print(f"  ❌ {Path(file_path).name}: {error}")
+
+        return result
+
+    except Exception as e:
+        print(f"❌ Batch processing demonstration failed: {str(e)}")
+        return None
+
+
+async def demonstrate_async_batch_processing():
+    """Demonstrate asynchronous batch processing"""
+    print("\n" + "=" * 60)
+    print("ASYNCHRONOUS BATCH PROCESSING DEMONSTRATION")
+    print("=" * 60)
+
+    # Create sample documents
+    sample_files, temp_dir = create_sample_documents()
+
+    try:
+        print(f"Processing {len(sample_files)} documents asynchronously...")
+
+        # Create batch parser
+        batch_parser = BatchParser(
+            parser_type="mineru",
+            max_workers=2,
+            show_progress=True,
+            skip_installation_check=True,
+        )
+
+        # Process batch asynchronously
+        output_dir = temp_dir / "async_output"
+
+        start_time = time.time()
+        result = await batch_parser.process_batch_async(
+            file_paths=sample_files,
+            output_dir=str(output_dir),
+            parse_method="auto",
+            recursive=False,
+        )
+        processing_time = time.time() - start_time
+
+        # Display results
+        print("\n" + "-" * 40)
+        print("ASYNC BATCH PROCESSING RESULTS")
+        print("-" * 40)
+        print(result.summary())
+        print(f"Async processing time: {processing_time:.2f} seconds")
+        print(f"Success rate: {result.success_rate:.1f}%")
+
+        return result
+
+    except Exception as e:
+        print(f"❌ Async batch processing demonstration failed: {str(e)}")
+        return None
+
+
+async def demonstrate_rag_integration():
+    """Demonstrate batch processing integration with RAG-Anything"""
+    print("\n" + "=" * 60)
+    print("RAG-ANYTHING BATCH INTEGRATION DEMONSTRATION")
+    print("=" * 60)
+
+    # Create sample documents
+    sample_files, temp_dir = create_sample_documents()
+
+    try:
+        # Initialize RAG-Anything with temporary storage
+        config = RAGAnythingConfig(
+            working_dir=str(temp_dir / "rag_storage"),
+            enable_image_processing=True,
+            enable_table_processing=True,
+            enable_equation_processing=True,
+            max_concurrent_files=2,
+        )
+
+        rag = RAGAnything(config=config)
+
+        print("RAG-Anything initialized with batch processing capabilities")
+
+        # Show available batch methods
+        batch_methods = [method for method in dir(rag) if "batch" in method.lower()]
+        print(f"Available batch methods: {batch_methods}")
+
+        # Demonstrate batch processing with RAG integration
+        print(f"\nProcessing {len(sample_files)} documents with RAG integration...")
+
+        # Use the RAG-integrated batch processing
+        try:
+            # Process documents in batch
+            result = rag.process_documents_batch(
+                file_paths=sample_files,
+                output_dir=str(temp_dir / "rag_batch_output"),
+                max_workers=2,
+                show_progress=True,
+            )
+
+            print("\n" + "-" * 40)
+            print("RAG BATCH PROCESSING RESULTS")
+            print("-" * 40)
+            print(result.summary())
+            print(f"Success rate: {result.success_rate:.1f}%")
+
+            # Demonstrate batch processing with full RAG integration
+            print("\nProcessing documents with full RAG integration...")
+
+            rag_result = await rag.process_documents_with_rag_batch(
+                file_paths=sample_files[:2],  # Process subset for demo
+                output_dir=str(temp_dir / "rag_full_output"),
+                max_workers=1,
+                show_progress=True,
+            )
+
+            print("\n" + "-" * 40)
+            print("FULL RAG INTEGRATION RESULTS")
+            print("-" * 40)
+            print(f"Parse result: {rag_result['parse_result'].summary()}")
+            print(
+                f"RAG processing time: {rag_result['total_processing_time']:.2f} seconds"
+            )
+            print(
+                f"Successfully processed with RAG: {rag_result['successful_rag_files']}"
+            )
+            print(f"Failed RAG processing: {rag_result['failed_rag_files']}")
+
+            return rag_result
+
+        except Exception as e:
+            print(f"⚠️ RAG integration demo completed with limitations: {str(e)}")
+            print(
+                "Note: This is expected in environments without full API configuration"
+            )
+            return None
+
+    except Exception as e:
+        print(f"❌ RAG integration demonstration failed: {str(e)}")
+        return None
+
+
+def demonstrate_directory_processing():
+    """Demonstrate processing entire directories"""
+    print("\n" + "=" * 60)
+    print("DIRECTORY PROCESSING DEMONSTRATION")
+    print("=" * 60)
+
+    # Create a directory structure with nested files
+    temp_dir = Path(tempfile.mkdtemp())
+
+    # Create main directory files
+    main_files = {
+        "overview.txt": "Main directory overview document",
+        "readme.md": "# Project README\n\nThis is the main project documentation.",
+    }
+
+    # Create subdirectory
+    sub_dir = temp_dir / "subdirectory"
+    sub_dir.mkdir()
+
+    sub_files = {
+        "details.txt": "Detailed information in subdirectory",
+        "notes.md": "# Notes\n\nAdditional notes and information.",
+    }
+
+    # Write all files
+    all_files = []
+    for filename, content in main_files.items():
+        file_path = temp_dir / filename
+        with open(file_path, "w", encoding="utf-8") as f:
+            f.write(content)
+        all_files.append(str(file_path))
+
+    for filename, content in sub_files.items():
+        file_path = sub_dir / filename
+        with open(file_path, "w", encoding="utf-8") as f:
+            f.write(content)
+        all_files.append(str(file_path))
+
+    try:
+        print("Created directory structure:")
+        print(f"  Main directory: {temp_dir}")
+        print(f"  Files in main: {list(main_files.keys())}")
+        print(f"  Subdirectory: {sub_dir}")
+        print(f"  Files in sub: {list(sub_files.keys())}")
+
+        # Create batch parser
+        batch_parser = BatchParser(
+            parser_type="mineru",
+            max_workers=2,
+            show_progress=True,
+            skip_installation_check=True,
+        )
+
+        # Process entire directory recursively
+        print("\nProcessing entire directory recursively...")
+
+        result = batch_parser.process_batch(
+            file_paths=[str(temp_dir)],  # Pass directory path
+            output_dir=str(temp_dir / "directory_output"),
+            parse_method="auto",
+            recursive=True,  # Include subdirectories
+        )
+
+        print("\n" + "-" * 40)
+        print("DIRECTORY PROCESSING RESULTS")
+        print("-" * 40)
+        print(result.summary())
+        print(f"Total files found and processed: {result.total_files}")
+        print(f"Success rate: {result.success_rate:.1f}%")
+
+        if result.successful_files:
+            print("\nSuccessfully processed:")
+            for file_path in result.successful_files:
+                relative_path = Path(file_path).relative_to(temp_dir)
+                print(f"  ✅ {relative_path}")
+
+        return result
+
+    except Exception as e:
+        print(f"❌ Directory processing demonstration failed: {str(e)}")
+        return None
+
+
+def demonstrate_error_handling():
+    """Demonstrate error handling and recovery"""
+    print("\n" + "=" * 60)
+    print("ERROR HANDLING DEMONSTRATION")
+    print("=" * 60)
+
+    temp_dir = Path(tempfile.mkdtemp())
+
+    # Create files with various issues
+    files_with_issues = {
+        "valid_file.txt": "This is a valid file that should process successfully.",
+        "empty_file.txt": "",  # Empty file
+        "large_file.txt": "x" * 1000000,  # Large file (1MB of 'x')
+    }
+
+    created_files = []
+    for filename, content in files_with_issues.items():
+        file_path = temp_dir / filename
+        with open(file_path, "w", encoding="utf-8") as f:
+            f.write(content)
+        created_files.append(str(file_path))
+
+    # Add a non-existent file to the list
+    created_files.append(str(temp_dir / "non_existent_file.txt"))
+
+    try:
+        print(f"Testing error handling with {len(created_files)} files:")
+        for file_path in created_files:
+            name = Path(file_path).name
+            exists = Path(file_path).exists()
+            size = Path(file_path).stat().st_size if exists else 0
+            print(f"  - {name}: {'exists' if exists else 'missing'}, {size} bytes")
+
+        # Create batch parser with short timeout for demonstration
+        batch_parser = BatchParser(
+            parser_type="mineru",
+            max_workers=2,
+            show_progress=True,
+            timeout_per_file=30,  # Short timeout for demo
+            skip_installation_check=True,
+        )
+
+        # Process files and handle errors
+        result = batch_parser.process_batch(
+            file_paths=created_files,
+            output_dir=str(temp_dir / "error_test_output"),
+            parse_method="auto",
+        )
+
+        print("\n" + "-" * 40)
+        print("ERROR HANDLING RESULTS")
+        print("-" * 40)
+        print(result.summary())
+
+        if result.successful_files:
+            print("\nSuccessful files:")
+            for file_path in result.successful_files:
+                print(f"  ✅ {Path(file_path).name}")
+
+        if result.failed_files:
+            print("\nFailed files with error details:")
+            for file_path in result.failed_files:
+                error = result.errors.get(file_path, "Unknown error")
+                print(f"  ❌ {Path(file_path).name}: {error}")
+
+        # Demonstrate retry logic
+        if result.failed_files:
+            print(
+                f"\nDemonstrating retry logic for {len(result.failed_files)} failed files..."
+            )
+
+            # Retry only the failed files
+            retry_result = batch_parser.process_batch(
+                file_paths=result.failed_files,
+                output_dir=str(temp_dir / "retry_output"),
+                parse_method="auto",
+            )
+
+            print(f"Retry results: {retry_result.summary()}")
+
+        return result
+
+    except Exception as e:
+        print(f"❌ Error handling demonstration failed: {str(e)}")
+        return None
+
+
+async def main():
+    """Main demonstration function"""
+    # Configure logging
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+    )
+
+    print("RAG-Anything Batch Processing Demonstration")
+    print("=" * 70)
+    print("This example demonstrates various batch processing capabilities:")
+    print("  - Basic batch processing with progress tracking")
+    print("  - Asynchronous processing for improved performance")
+    print("  - Integration with RAG-Anything pipeline")
+    print("  - Directory processing with recursive file discovery")
+    print("  - Comprehensive error handling and recovery")
+
+    results = {}
+
+    # Run demonstrations
+    print("\n🚀 Starting demonstrations...")
+
+    # Basic batch processing
+    results["basic"] = demonstrate_basic_batch_processing()
+
+    # Asynchronous processing
+    results["async"] = await demonstrate_async_batch_processing()
+
+    # RAG integration
+    results["rag"] = await demonstrate_rag_integration()
+
+    # Directory processing
+    results["directory"] = demonstrate_directory_processing()
+
+    # Error handling
+    results["error_handling"] = demonstrate_error_handling()
+
+    # Summary
+    print("\n" + "=" * 70)
+    print("DEMONSTRATION SUMMARY")
+    print("=" * 70)
+
+    for demo_name, result in results.items():
+        if result:
+            if hasattr(result, "success_rate"):
+                print(
+                    f"✅ {demo_name.upper()}: {result.success_rate:.1f}% success rate"
+                )
+            else:
+                print(f"✅ {demo_name.upper()}: Completed successfully")
+        else:
+            print(f"❌ {demo_name.upper()}: Failed or had limitations")
+
+    print("\n📊 Key Features Demonstrated:")
+    print("  - Parallel document processing with configurable worker counts")
+    print("  - Real-time progress tracking with tqdm progress bars")
+    print("  - Comprehensive error handling and reporting")
+    print("  - File filtering based on supported document types")
+    print("  - Directory processing with recursive file discovery")
+    print("  - Asynchronous processing for improved performance")
+    print("  - Integration with RAG-Anything document pipeline")
+    print("  - Retry logic for failed documents")
+    print("  - Detailed processing statistics and timing")
+
+    print("\n💡 Best Practices Highlighted:")
+    print("  - Use appropriate worker counts for your system")
+    print("  - Enable progress tracking for long-running operations")
+    print("  - Handle errors gracefully with retry mechanisms")
+    print("  - Filter files to supported types before processing")
+    print("  - Set reasonable timeouts for document processing")
+    print("  - Use skip_installation_check for environments with conflicts")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

rag_anything_smaranika/examples/batch_processing_optimized_example.py

@@ -0,0 +1,216 @@
+"""
+Example: Optimized Batch Processing for RAGAnything
+
+This example demonstrates the new optimized batch processing capabilities
+that provide 2-3x faster processing for large document collections.
+
+Features demonstrated:
+- Concurrent document parsing with prefetching
+- Pipeline architecture (parse + process simultaneously)
+- Progress tracking with ETA estimation
+- Adaptive rate limiting
+- Performance statistics
+"""
+
+import asyncio
+import time
+from pathlib import Path
+from raganything import RAGAnything
+
+async def progress_callback(progress_data):
+    """
+    Callback function to handle progress updates
+
+    Args:
+        progress_data: Dict containing:
+            - processed: Number of processed documents
+            - total: Total number of documents
+            - failed: Number of failed documents
+            - percentage: Completion percentage
+            - eta_seconds: Estimated time remaining
+            - rate_docs_per_sec: Processing rate
+    """
+    print(f"\rProgress: {progress_data['processed']}/{progress_data['total']} "
+          f"({progress_data['percentage']:.1f}%) | "
+          f"Rate: {progress_data['rate_docs_per_sec']:.2f} docs/s | "
+          f"ETA: {progress_data['eta_seconds']:.1f}s", end='', flush=True)
+
+
+async def main():
+    # Initialize RAGAnything
+    rag = RAGAnything(
+        working_dir="./rag_storage",
+        rag_dir="./rag_index",
+        parser="mineru",  # or "docling"
+    )
+
+    # Example 1: Process a list of documents with optimization
+    print("=" * 60)
+    print("Example 1: Optimized Batch Processing")
+    print("=" * 60)
+
+    documents = [
+        "./data/report1.pdf",
+        "./data/report2.pdf",
+        "./data/research_paper.pdf",
+        "./data/technical_spec.docx",
+    ]
+
+    start_time = time.time()
+
+    result = await rag.process_documents_batch_optimized(
+        file_paths=documents,
+        max_concurrent_parsers=4,        # Parse up to 4 documents at once
+        max_concurrent_processors=10,     # Process up to 10 chunks concurrently
+        enable_progress_tracking=True,
+        progress_callback=progress_callback,
+    )
+
+    print()  # New line after progress bar
+
+    elapsed_time = time.time() - start_time
+
+    # Display results
+    print(f"\n📊 Processing Results:")
+    print(f"  ✅ Successful: {len(result['successful_files'])} documents")
+    print(f"  ❌ Failed: {len(result['failed_files'])} documents")
+    print(f"  ⏱️  Total time: {elapsed_time:.2f}s")
+
+    # Display detailed statistics
+    stats = result['statistics']
+    print(f"\n📈 Performance Statistics:")
+    print(f"  Processing rate: {stats['processing_rate_docs_per_sec']:.2f} docs/sec")
+    print(f"  Parsing time: {stats['parsing_time']:.2f}s")
+    print(f"  Text processing: {stats['text_processing_time']:.2f}s")
+    print(f"  Multimodal processing: {stats['multimodal_processing_time']:.2f}s")
+    print(f"  Cache hit rate: {stats['cache_hit_rate']:.1f}%")
+
+    # Show per-document results
+    if result['successful_files']:
+        print(f"\n✅ Successfully processed files:")
+        for file_info in result['successful_files'][:5]:  # Show first 5
+            print(f"  - {Path(file_info['file_path']).name} "
+                  f"(processing: {file_info['processing_time']:.2f}s, "
+                  f"parsing: {file_info['parse_time']:.2f}s)")
+
+    if result['failed_files']:
+        print(f"\n❌ Failed files:")
+        for file_info in result['failed_files']:
+            print(f"  - {Path(file_info['file_path']).name}: {file_info['error']}")
+
+    # Example 2: Process an entire folder with optimization
+    print("\n" + "=" * 60)
+    print("Example 2: Optimized Folder Processing")
+    print("=" * 60)
+
+    folder_result = await rag.process_folder_optimized(
+        folder_path="./data/documents",
+        file_extensions=['.pdf', '.docx', '.pptx'],
+        recursive=True,
+        max_concurrent_parsers=6,
+        max_concurrent_processors=12,
+        progress_callback=progress_callback,
+    )
+
+    print()  # New line after progress bar
+
+    print(f"\n📁 Folder Processing Complete:")
+    print(f"  Successful: {len(folder_result['successful_files'])} files")
+    print(f"  Failed: {len(folder_result['failed_files'])} files")
+    print(f"  Rate: {folder_result['statistics']['processing_rate_docs_per_sec']:.2f} docs/sec")
+
+    # Example 3: Compare standard vs optimized processing
+    print("\n" + "=" * 60)
+    print("Example 3: Performance Comparison")
+    print("=" * 60)
+
+    test_docs = ["./data/test1.pdf", "./data/test2.pdf", "./data/test3.pdf"]
+
+    # Standard processing
+    print("\n🐢 Standard batch processing...")
+    standard_start = time.time()
+    await rag.process_folder_complete(
+        folder_path="./data/test",
+        max_workers=4,
+        display_stats=False
+    )
+    standard_time = time.time() - standard_start
+
+    # Optimized processing (on different set to avoid cache)
+    print("🚀 Optimized batch processing...")
+    optimized_start = time.time()
+    await rag.process_documents_batch_optimized(
+        file_paths=test_docs,
+        max_concurrent_parsers=4,
+        max_concurrent_processors=10,
+        enable_progress_tracking=False,
+    )
+    optimized_time = time.time() - optimized_start
+
+    print(f"\n⚡ Performance Improvement:")
+    print(f"  Standard: {standard_time:.2f}s")
+    print(f"  Optimized: {optimized_time:.2f}s")
+    if standard_time > 0:
+        speedup = (standard_time / optimized_time)
+        print(f"  Speedup: {speedup:.2f}x faster")
+
+    # Example 4: Custom progress tracking
+    print("\n" + "=" * 60)
+    print("Example 4: Custom Progress Tracking")
+    print("=" * 60)
+
+    class CustomProgressTracker:
+        def __init__(self):
+            self.start_time = time.time()
+            self.logs = []
+
+        def __call__(self, progress):
+            """Progress callback"""
+            elapsed = time.time() - self.start_time
+            log_entry = {
+                "timestamp": elapsed,
+                "processed": progress['processed'],
+                "total": progress['total'],
+                "percentage": progress['percentage'],
+                "rate": progress['rate_docs_per_sec'],
+            }
+            self.logs.append(log_entry)
+
+            # Print formatted progress
+            bar_length = 40
+            filled_length = int(bar_length * progress['percentage'] / 100)
+            bar = '█' * filled_length + '-' * (bar_length - filled_length)
+
+            print(f"\r|{bar}| {progress['percentage']:.1f}% "
+                  f"[{progress['processed']}/{progress['total']}] "
+                  f"ETA: {progress['eta_seconds']:.0f}s", end='', flush=True)
+
+        def save_log(self, filename="processing_log.txt"):
+            """Save progress log to file"""
+            with open(filename, 'w') as f:
+                f.write("Batch Processing Log\n")
+                f.write("=" * 50 + "\n")
+                for entry in self.logs:
+                    f.write(f"Time: {entry['timestamp']:.2f}s | "
+                           f"Progress: {entry['processed']}/{entry['total']} "
+                           f"({entry['percentage']:.1f}%) | "
+                           f"Rate: {entry['rate']:.2f} docs/s\n")
+
+    tracker = CustomProgressTracker()
+
+    await rag.process_documents_batch_optimized(
+        file_paths=documents,
+        progress_callback=tracker,
+    )
+
+    print()  # New line
+    tracker.save_log("./batch_processing_log.txt")
+    print("📝 Progress log saved to batch_processing_log.txt")
+
+    print("\n" + "=" * 60)
+    print("All examples completed!")
+    print("=" * 60)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

rag_anything_smaranika/examples/enhanced_markdown_example.py

@@ -0,0 +1,1055 @@
+#!/usr/bin/env python
+"""
+Enhanced Markdown Conversion Example for RAG-Anything
+
+This example demonstrates the enhanced markdown to PDF conversion capabilities
+with multiple backends, advanced styling, and professional formatting.
+
+Features demonstrated:
+- Basic markdown to PDF conversion
+- Multiple conversion backends (WeasyPrint, Pandoc)
+- Custom CSS styling and configuration
+- Backend detection and selection
+- Error handling and fallback mechanisms
+- Command-line interface usage
+"""
+
+import logging
+from pathlib import Path
+import tempfile
+
+# Add project root directory to Python path
+import sys
+
+sys.path.append(str(Path(__file__).parent.parent))
+
+from raganything.enhanced_markdown import EnhancedMarkdownConverter, MarkdownConfig
+
+
+def create_sample_markdown_content():
+    """Create comprehensive sample markdown content for testing"""
+
+    # Basic sample
+    basic_content = """# Basic Markdown Sample
+
+## Introduction
+This is a simple markdown document demonstrating basic formatting.
+
+### Text Formatting
+- **Bold text** and *italic text*
+- `Inline code` examples
+- [Links to external sites](https://github.com)
+
+### Lists
+1. First ordered item
+2. Second ordered item
+3. Third ordered item
+
+- Unordered item
+- Another unordered item
+  - Nested item
+  - Another nested item
+
+### Blockquotes
+> This is a blockquote with important information.
+> It can span multiple lines.
+
+### Code Block
+```python
+def hello_world():
+    print("Hello, World!")
+    return "Success"
+```
+"""
+
+    # Technical documentation sample
+    technical_content = """# Technical Documentation
+
+## Table of Contents
+- [Overview](#overview)
+- [Architecture](#architecture)
+- [Implementation](#implementation)
+- [Performance](#performance)
+
+## Overview
+This document provides comprehensive technical specifications for the enhanced markdown conversion system.
+
+## Architecture
+
+### Core Components
+1. **Markdown Parser**: Processes markdown syntax
+2. **CSS Engine**: Applies styling and layout
+3. **PDF Generator**: Creates final PDF output
+4. **Backend Manager**: Handles multiple conversion engines
+
+### Data Flow
+```mermaid
+graph LR
+    A[Markdown Input] --> B[Parser]
+    B --> C[CSS Processor]
+    C --> D[PDF Generator]
+    D --> E[PDF Output]
+```
+
+## Implementation
+
+### Python Code Example
+```python
+from raganything.enhanced_markdown import EnhancedMarkdownConverter, MarkdownConfig
+
+# Configure converter
+config = MarkdownConfig(
+    page_size="A4",
+    margin="1in",
+    include_toc=True,
+    syntax_highlighting=True
+)
+
+# Create converter
+converter = EnhancedMarkdownConverter(config)
+
+# Convert to PDF
+success = converter.convert_file_to_pdf(
+    input_path="document.md",
+    output_path="output.pdf",
+    method="weasyprint"
+)
+```
+
+### Configuration Options
+```yaml
+converter:
+  page_size: A4
+  margin: 1in
+  font_size: 12pt
+  include_toc: true
+  syntax_highlighting: true
+  backend: weasyprint
+```
+
+## Performance
+
+### Benchmark Results
+| Backend | Speed | Quality | Features |
+|---------|-------|---------|----------|
+| WeasyPrint | ⭐⭐⭐⭐ | ⭐⭐⭐⭐ | ⭐⭐⭐⭐ |
+| Pandoc | ⭐⭐⭐ | ⭐⭐⭐⭐⭐ | ⭐⭐⭐⭐⭐ |
+
+### Processing Times
+- **Small documents** (< 10 pages): 1-3 seconds
+- **Medium documents** (10-50 pages): 3-10 seconds
+- **Large documents** (> 50 pages): 10-30 seconds
+
+## Advanced Features
+
+### Custom CSS Styling
+The system supports advanced CSS customization:
+
+```css
+body {
+    font-family: 'Georgia', serif;
+    line-height: 1.6;
+    color: #333;
+}
+
+h1 {
+    color: #2c3e50;
+    border-bottom: 2px solid #3498db;
+    padding-bottom: 0.3em;
+}
+
+code {
+    background-color: #f8f9fa;
+    padding: 2px 4px;
+    border-radius: 3px;
+    font-family: 'Courier New', monospace;
+}
+
+pre {
+    background-color: #f8f9fa;
+    border-left: 4px solid #3498db;
+    padding: 15px;
+    border-radius: 5px;
+    overflow-x: auto;
+}
+
+table {
+    border-collapse: collapse;
+    width: 100%;
+    margin: 1em 0;
+}
+
+th, td {
+    border: 1px solid #ddd;
+    padding: 8px 12px;
+    text-align: left;
+}
+
+th {
+    background-color: #f2f2f2;
+    font-weight: bold;
+}
+```
+
+### Image Support
+![Sample Image](https://via.placeholder.com/400x200/3498db/ffffff?text=Sample+Image)
+
+Images are automatically scaled and positioned appropriately in the PDF output.
+
+## Conclusion
+The enhanced markdown conversion system provides professional-quality PDF generation with extensive customization options and multiple backend support.
+
+---
+
+*Generated on: 2024-01-15*
+*Version: 1.0.0*
+"""
+
+    # Academic paper sample
+    academic_content = """# Research Paper: Advanced Document Processing
+
+**Authors:** Alice Johnson¹, Bob Smith², Carol Williams¹
+**Affiliations:**
+¹ University of Technology
+² Research Institute
+
+## Abstract
+
+This paper presents a comprehensive analysis of advanced document processing techniques using enhanced markdown conversion. Our research demonstrates significant improvements in processing speed and output quality through optimized backend selection and custom styling approaches.
+
+**Keywords:** document processing, markdown conversion, PDF generation, performance optimization
+
+## 1. Introduction
+
+Document processing has become increasingly important in modern information systems. The ability to convert markdown documents to high-quality PDF outputs with professional formatting is crucial for academic, technical, and business applications.
+
+### 1.1 Research Objectives
+
+1. Evaluate different markdown conversion backends
+2. Analyze performance characteristics of each approach
+3. Develop optimization strategies for large-scale processing
+4. Design flexible configuration systems for diverse use cases
+
+### 1.2 Contributions
+
+This work makes the following contributions:
+- Comprehensive comparison of markdown conversion backends
+- Performance optimization techniques for large documents
+- Flexible configuration framework for customization
+- Integration patterns for document processing pipelines
+
+## 2. Methodology
+
+### 2.1 Experimental Setup
+
+We conducted experiments using the following configuration:
+
+```python
+# Experimental configuration
+config = MarkdownConfig(
+    page_size="A4",
+    margin="1in",
+    font_size="11pt",
+    line_height="1.4",
+    include_toc=True,
+    syntax_highlighting=True
+)
+```
+
+### 2.2 Test Documents
+
+| Category | Count | Avg Size | Complexity |
+|----------|-------|----------|------------|
+| Simple | 100 | 2 pages | Low |
+| Medium | 50 | 10 pages | Medium |
+| Complex | 25 | 25 pages | High |
+
+### 2.3 Metrics
+
+We evaluated performance using the following metrics:
+- **Conversion Speed**: Time to generate PDF (seconds)
+- **Memory Usage**: Peak memory consumption (MB)
+- **Output Quality**: Visual assessment score (1-10)
+- **Feature Support**: Number of supported markdown features
+
+## 3. Results
+
+### 3.1 Performance Comparison
+
+The following table summarizes our performance results:
+
+| Backend | Speed (s) | Memory (MB) | Quality | Features |
+|---------|-----------|-------------|---------|----------|
+| WeasyPrint | 2.3 ± 0.5 | 85 ± 15 | 8.5 | 85% |
+| Pandoc | 4.7 ± 1.2 | 120 ± 25 | 9.2 | 95% |
+
+### 3.2 Quality Analysis
+
+#### 3.2.1 Typography
+WeasyPrint excels in web-style typography with excellent CSS support, while Pandoc provides superior academic formatting with LaTeX-quality output.
+
+#### 3.2.2 Code Highlighting
+Both backends support syntax highlighting through Pygments:
+
+```python
+def analyze_performance(backend, documents):
+    '''Analyze conversion performance for given backend'''
+    results = []
+
+    for doc in documents:
+        start_time = time.time()
+        success = backend.convert(doc)
+        end_time = time.time()
+
+        results.append({
+            'document': doc,
+            'time': end_time - start_time,
+            'success': success
+        })
+
+    return results
+```
+
+### 3.3 Scalability
+
+Our scalability analysis shows:
+- Linear scaling with document size for both backends
+- Memory usage proportional to content complexity
+- Optimal batch sizes of 10-20 documents for parallel processing
+
+## 4. Discussion
+
+### 4.1 Backend Selection Guidelines
+
+Choose **WeasyPrint** for:
+- Web-style documents with custom CSS
+- Fast conversion requirements
+- Simple to medium complexity documents
+
+Choose **Pandoc** for:
+- Academic papers and publications
+- Complex document structures
+- Maximum feature support requirements
+
+### 4.2 Optimization Strategies
+
+1. **Image Optimization**: Compress images before embedding
+2. **CSS Minimization**: Use efficient CSS selectors
+3. **Content Chunking**: Process large documents in sections
+4. **Caching**: Cache converted content for repeated use
+
+## 5. Conclusion
+
+This research demonstrates that enhanced markdown conversion provides significant benefits for document processing workflows. The choice between WeasyPrint and Pandoc depends on specific requirements for speed, quality, and features.
+
+### 5.1 Future Work
+
+- Integration with cloud processing services
+- Real-time collaborative editing support
+- Advanced template systems
+- Performance optimization for very large documents
+
+## References
+
+1. Johnson, A. et al. (2024). "Advanced Document Processing Techniques." *Journal of Information Systems*, 15(3), 45-62.
+2. Smith, B. (2023). "PDF Generation Optimization." *Technical Computing Review*, 8(2), 12-28.
+3. Williams, C. (2024). "Markdown Processing Frameworks." *Software Engineering Quarterly*, 22(1), 78-95.
+
+---
+
+**Manuscript received:** January 10, 2024
+**Accepted for publication:** January 15, 2024
+**Published online:** January 20, 2024
+"""
+
+    return {
+        "basic": basic_content,
+        "technical": technical_content,
+        "academic": academic_content,
+    }
+
+
+def demonstrate_basic_conversion():
+    """Demonstrate basic markdown to PDF conversion"""
+    print("\n" + "=" * 60)
+    print("BASIC MARKDOWN CONVERSION DEMONSTRATION")
+    print("=" * 60)
+
+    try:
+        # Create converter with default settings
+        converter = EnhancedMarkdownConverter()
+
+        # Show backend information
+        backend_info = converter.get_backend_info()
+        print("Available conversion backends:")
+        for backend, available in backend_info["available_backends"].items():
+            status = "✅" if available else "❌"
+            print(f"  {status} {backend}")
+        print(f"Recommended backend: {backend_info['recommended_backend']}")
+
+        # Get sample content
+        samples = create_sample_markdown_content()
+        temp_dir = Path(tempfile.mkdtemp())
+
+        # Convert basic sample
+        basic_md_path = temp_dir / "basic_sample.md"
+        with open(basic_md_path, "w", encoding="utf-8") as f:
+            f.write(samples["basic"])
+
+        print(f"\nConverting basic sample: {basic_md_path}")
+
+        success = converter.convert_file_to_pdf(
+            input_path=str(basic_md_path),
+            output_path=str(temp_dir / "basic_sample.pdf"),
+            method="auto",  # Let the system choose the best backend
+        )
+
+        if success:
+            print("✅ Basic conversion successful!")
+            print(f"   Output: {temp_dir / 'basic_sample.pdf'}")
+        else:
+            print("❌ Basic conversion failed")
+
+        return success, temp_dir
+
+    except Exception as e:
+        print(f"❌ Basic conversion demonstration failed: {str(e)}")
+        return False, None
+
+
+def demonstrate_backend_comparison():
+    """Demonstrate different conversion backends"""
+    print("\n" + "=" * 60)
+    print("BACKEND COMPARISON DEMONSTRATION")
+    print("=" * 60)
+
+    try:
+        samples = create_sample_markdown_content()
+        temp_dir = Path(tempfile.mkdtemp())
+
+        # Create technical document
+        tech_md_path = temp_dir / "technical.md"
+        with open(tech_md_path, "w", encoding="utf-8") as f:
+            f.write(samples["technical"])
+
+        print("Testing different backends with technical document...")
+
+        # Test different backends
+        backends = ["auto", "weasyprint", "pandoc"]
+        results = {}
+
+        for backend in backends:
+            try:
+                print(f"\nTesting {backend} backend...")
+
+                converter = EnhancedMarkdownConverter()
+                output_path = temp_dir / f"technical_{backend}.pdf"
+
+                import time
+
+                start_time = time.time()
+
+                success = converter.convert_file_to_pdf(
+                    input_path=str(tech_md_path),
+                    output_path=str(output_path),
+                    method=backend,
+                )
+
+                end_time = time.time()
+                conversion_time = end_time - start_time
+
+                if success:
+                    file_size = (
+                        output_path.stat().st_size if output_path.exists() else 0
+                    )
+                    print(
+                        f"  ✅ {backend}: Success in {conversion_time:.2f}s, {file_size} bytes"
+                    )
+                    results[backend] = {
+                        "success": True,
+                        "time": conversion_time,
+                        "size": file_size,
+                        "output": str(output_path),
+                    }
+                else:
+                    print(f"  ❌ {backend}: Failed")
+                    results[backend] = {"success": False, "time": conversion_time}
+
+            except Exception as e:
+                print(f"  ❌ {backend}: Error - {str(e)}")
+                results[backend] = {"success": False, "error": str(e)}
+
+        # Summary
+        print("\n" + "-" * 40)
+        print("BACKEND COMPARISON SUMMARY")
+        print("-" * 40)
+        successful_backends = [b for b, r in results.items() if r.get("success", False)]
+        print(f"Successful backends: {successful_backends}")
+
+        if successful_backends:
+            fastest = min(successful_backends, key=lambda b: results[b]["time"])
+            print(f"Fastest backend: {fastest} ({results[fastest]['time']:.2f}s)")
+
+        return results, temp_dir
+
+    except Exception as e:
+        print(f"❌ Backend comparison demonstration failed: {str(e)}")
+        return None, None
+
+
+def demonstrate_custom_styling():
+    """Demonstrate custom CSS styling and configuration"""
+    print("\n" + "=" * 60)
+    print("CUSTOM STYLING DEMONSTRATION")
+    print("=" * 60)
+
+    try:
+        samples = create_sample_markdown_content()
+        temp_dir = Path(tempfile.mkdtemp())
+
+        # Create custom CSS
+        custom_css = """
+        body {
+            font-family: 'Times New Roman', serif;
+            font-size: 11pt;
+            line-height: 1.4;
+            color: #2c3e50;
+            max-width: 800px;
+            margin: 0 auto;
+            padding: 20px;
+        }
+
+        h1 {
+            color: #c0392b;
+            font-size: 2.2em;
+            border-bottom: 3px solid #e74c3c;
+            padding-bottom: 0.5em;
+            margin-top: 2em;
+        }
+
+        h2 {
+            color: #8e44ad;
+            font-size: 1.6em;
+            border-bottom: 2px solid #9b59b6;
+            padding-bottom: 0.3em;
+            margin-top: 1.5em;
+        }
+
+        h3 {
+            color: #2980b9;
+            font-size: 1.3em;
+            margin-top: 1.2em;
+        }
+
+        code {
+            background-color: #ecf0f1;
+            color: #e74c3c;
+            padding: 3px 6px;
+            border-radius: 4px;
+            font-family: 'Courier New', monospace;
+            font-size: 0.9em;
+        }
+
+        pre {
+            background-color: #2c3e50;
+            color: #ecf0f1;
+            padding: 20px;
+            border-radius: 8px;
+            border-left: 5px solid #3498db;
+            overflow-x: auto;
+            font-size: 0.9em;
+        }
+
+        pre code {
+            background-color: transparent;
+            color: inherit;
+            padding: 0;
+        }
+
+        blockquote {
+            background-color: #f8f9fa;
+            border-left: 5px solid #3498db;
+            margin: 1em 0;
+            padding: 15px 20px;
+            font-style: italic;
+            color: #555;
+        }
+
+        table {
+            border-collapse: collapse;
+            width: 100%;
+            margin: 1.5em 0;
+            background-color: white;
+            border-radius: 8px;
+            overflow: hidden;
+            box-shadow: 0 2px 4px rgba(0,0,0,0.1);
+        }
+
+        th {
+            background-color: #3498db;
+            color: white;
+            padding: 12px 15px;
+            text-align: left;
+            font-weight: bold;
+        }
+
+        td {
+            padding: 10px 15px;
+            border-bottom: 1px solid #ecf0f1;
+        }
+
+        tr:nth-child(even) {
+            background-color: #f8f9fa;
+        }
+
+        tr:hover {
+            background-color: #e8f4fd;
+        }
+
+        ul, ol {
+            margin-bottom: 1em;
+            padding-left: 2em;
+        }
+
+        li {
+            margin-bottom: 0.5em;
+            line-height: 1.6;
+        }
+
+        a {
+            color: #3498db;
+            text-decoration: none;
+            border-bottom: 1px dotted #3498db;
+        }
+
+        a:hover {
+            color: #2980b9;
+            border-bottom: 1px solid #2980b9;
+        }
+
+        .toc {
+            background-color: #f8f9fa;
+            border: 2px solid #e9ecef;
+            border-radius: 8px;
+            padding: 20px;
+            margin: 2em 0;
+        }
+
+        .toc h2 {
+            color: #2c3e50;
+            margin-top: 0;
+            border-bottom: none;
+        }
+
+        .toc ul {
+            list-style-type: none;
+            padding-left: 0;
+        }
+
+        .toc li {
+            margin-bottom: 0.8em;
+        }
+
+        .toc a {
+            color: #2c3e50;
+            font-weight: 500;
+            border-bottom: none;
+        }
+        """
+
+        # Create custom configuration
+        config = MarkdownConfig(
+            page_size="A4",
+            margin="0.8in",
+            font_size="11pt",
+            line_height="1.4",
+            include_toc=True,
+            syntax_highlighting=True,
+            custom_css=custom_css,
+        )
+
+        converter = EnhancedMarkdownConverter(config)
+
+        # Convert academic sample with custom styling
+        academic_md_path = temp_dir / "academic_styled.md"
+        with open(academic_md_path, "w", encoding="utf-8") as f:
+            f.write(samples["academic"])
+
+        print("Converting academic paper with custom styling...")
+        print("Custom styling features:")
+        print("  - Custom color scheme (reds, purples, blues)")
+        print("  - Times New Roman serif font")
+        print("  - Enhanced table styling with hover effects")
+        print("  - Styled code blocks with dark theme")
+        print("  - Custom blockquote styling")
+        print("  - Professional header styling")
+
+        success = converter.convert_file_to_pdf(
+            input_path=str(academic_md_path),
+            output_path=str(temp_dir / "academic_styled.pdf"),
+            method="weasyprint",  # WeasyPrint is best for custom CSS
+        )
+
+        if success:
+            print("✅ Custom styling conversion successful!")
+            print(f"   Output: {temp_dir / 'academic_styled.pdf'}")
+
+            # Also create a default version for comparison
+            default_converter = EnhancedMarkdownConverter()
+            default_success = default_converter.convert_file_to_pdf(
+                input_path=str(academic_md_path),
+                output_path=str(temp_dir / "academic_default.pdf"),
+                method="weasyprint",
+            )
+
+            if default_success:
+                print(f"   Comparison (default): {temp_dir / 'academic_default.pdf'}")
+        else:
+            print("❌ Custom styling conversion failed")
+
+        return success, temp_dir
+
+    except Exception as e:
+        print(f"❌ Custom styling demonstration failed: {str(e)}")
+        return False, None
+
+
+def demonstrate_content_conversion():
+    """Demonstrate converting markdown content directly (not from file)"""
+    print("\n" + "=" * 60)
+    print("CONTENT CONVERSION DEMONSTRATION")
+    print("=" * 60)
+
+    try:
+        # Create markdown content programmatically
+        dynamic_content = f"""# Dynamic Content Example
+
+## Generated Information
+This document was generated programmatically on {Path(__file__).name}.
+
+## System Information
+- **Python Path**: {sys.executable}
+- **Script Location**: {Path(__file__).absolute()}
+- **Working Directory**: {Path.cwd()}
+
+## Dynamic Table
+| Property | Value |
+|----------|-------|
+| Script Name | {Path(__file__).name} |
+| Python Version | {sys.version.split()[0]} |
+| Platform | {sys.platform} |
+
+## Code Example
+```python
+# This content was generated dynamically
+import sys
+from pathlib import Path
+
+def generate_report():
+    return f"Report generated from {{Path(__file__).name}}"
+
+print(generate_report())
+```
+
+## Features Demonstrated
+This example shows how to:
+1. Generate markdown content programmatically
+2. Convert content directly without saving to file first
+3. Include dynamic information in documents
+4. Use different conversion methods
+
+> **Note**: This content was created in memory and converted directly to PDF
+> without intermediate file storage.
+
+## Conclusion
+Direct content conversion is useful for:
+- Dynamic report generation
+- Programmatic document creation
+- API-based document services
+- Real-time content processing
+"""
+
+        temp_dir = Path(tempfile.mkdtemp())
+        converter = EnhancedMarkdownConverter()
+
+        print("Converting dynamically generated markdown content...")
+        print("Content includes:")
+        print("  - System information")
+        print("  - Dynamic tables with current values")
+        print("  - Generated timestamps")
+        print("  - Programmatic examples")
+
+        # Convert content directly to PDF
+        output_path = temp_dir / "dynamic_content.pdf"
+
+        success = converter.convert_markdown_to_pdf(
+            markdown_content=dynamic_content,
+            output_path=str(output_path),
+            method="auto",
+        )
+
+        if success:
+            print("✅ Content conversion successful!")
+            print(f"   Output: {output_path}")
+
+            # Show file size
+            file_size = output_path.stat().st_size
+            print(f"   Generated PDF size: {file_size} bytes")
+        else:
+            print("❌ Content conversion failed")
+
+        return success, temp_dir
+
+    except Exception as e:
+        print(f"❌ Content conversion demonstration failed: {str(e)}")
+        return False, None
+
+
+def demonstrate_error_handling():
+    """Demonstrate error handling and fallback mechanisms"""
+    print("\n" + "=" * 60)
+    print("ERROR HANDLING DEMONSTRATION")
+    print("=" * 60)
+
+    try:
+        temp_dir = Path(tempfile.mkdtemp())
+
+        # Test cases with various issues
+        test_cases = {
+            "invalid_markdown": """# Invalid Markdown
+
+This markdown has some {{invalid}} syntax and [broken links](http://nonexistent.invalid).
+
+```unknown_language
+This code block uses an unknown language
+```
+
+![Missing Image](nonexistent_image.png)
+""",
+            "complex_content": """# Complex Content Test
+
+## Mathematical Expressions
+This tests content that might be challenging for some backends:
+
+$$ E = mc^2 $$
+
+$$\\sum_{i=1}^{n} x_i = \\frac{n(n+1)}{2}$$
+
+## Complex Tables
+| A | B | C | D | E | F | G |
+|---|---|---|---|---|---|---|
+| Very long content that might wrap | Short | Medium length content | X | Y | Z | End |
+| Another row with different lengths | A | B | C | D | E | F |
+
+## Special Characters
+Unicode: α, β, γ, δ, ε, ζ, η, θ, ι, κ, λ, μ, ν, ξ, ο, π, ρ, σ, τ, υ, φ, χ, ψ, ω
+Symbols: ♠ ♣ ♥ ♦ ☀ ☁ ☂ ☃ ☄ ★ ☆ ☉ ☊ ☋ ☌ ☍ ☎ ☏
+Arrows: ← ↑ → ↓ ↔ ↕ ↖ ↗ ↘ ↙
+""",
+            "empty_content": "",
+            "minimal_content": "# Just a title",
+        }
+
+        print("Testing error handling with various content types...")
+
+        results = {}
+
+        for test_name, content in test_cases.items():
+            print(f"\nTesting: {test_name}")
+
+            try:
+                # Try multiple backends for each test case
+                for backend in ["auto", "weasyprint", "pandoc"]:
+                    try:
+                        converter = EnhancedMarkdownConverter()
+                        output_path = temp_dir / f"{test_name}_{backend}.pdf"
+
+                        success = converter.convert_markdown_to_pdf(
+                            markdown_content=content,
+                            output_path=str(output_path),
+                            method=backend,
+                        )
+
+                        if success:
+                            file_size = (
+                                output_path.stat().st_size
+                                if output_path.exists()
+                                else 0
+                            )
+                            print(f"  ✅ {backend}: Success ({file_size} bytes)")
+                            results[f"{test_name}_{backend}"] = {
+                                "success": True,
+                                "size": file_size,
+                            }
+                        else:
+                            print(f"  ❌ {backend}: Failed")
+                            results[f"{test_name}_{backend}"] = {"success": False}
+
+                    except Exception as e:
+                        print(f"  ❌ {backend}: Error - {str(e)[:60]}...")
+                        results[f"{test_name}_{backend}"] = {
+                            "success": False,
+                            "error": str(e),
+                        }
+
+            except Exception as e:
+                print(f"  ❌ Test case failed: {str(e)}")
+
+        # Demonstrate robust conversion with fallbacks
+        print("\nDemonstrating robust conversion with fallback logic...")
+
+        def robust_convert(content, output_path):
+            """Convert with multiple backend fallbacks"""
+            backends = ["weasyprint", "pandoc", "auto"]
+
+            for backend in backends:
+                try:
+                    converter = EnhancedMarkdownConverter()
+                    success = converter.convert_markdown_to_pdf(
+                        markdown_content=content,
+                        output_path=output_path,
+                        method=backend,
+                    )
+                    if success:
+                        return backend, True
+                except Exception:
+                    continue
+
+            return None, False
+
+        # Test robust conversion
+        test_content = test_cases["complex_content"]
+        robust_output = temp_dir / "robust_conversion.pdf"
+
+        successful_backend, success = robust_convert(test_content, str(robust_output))
+
+        if success:
+            print(f"✅ Robust conversion successful using {successful_backend}")
+            print(f"   Output: {robust_output}")
+        else:
+            print("❌ All backends failed for robust conversion")
+
+        # Summary
+        print("\n" + "-" * 40)
+        print("ERROR HANDLING SUMMARY")
+        print("-" * 40)
+        successful_conversions = sum(
+            1 for r in results.values() if r.get("success", False)
+        )
+        total_attempts = len(results)
+        success_rate = (
+            (successful_conversions / total_attempts * 100) if total_attempts > 0 else 0
+        )
+
+        print(f"Total conversion attempts: {total_attempts}")
+        print(f"Successful conversions: {successful_conversions}")
+        print(f"Success rate: {success_rate:.1f}%")
+
+        return results, temp_dir
+
+    except Exception as e:
+        print(f"❌ Error handling demonstration failed: {str(e)}")
+        return None, None
+
+
+def main():
+    """Main demonstration function"""
+    # Configure logging
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+    )
+
+    print("RAG-Anything Enhanced Markdown Conversion Demonstration")
+    print("=" * 70)
+    print(
+        "This example demonstrates various enhanced markdown conversion capabilities:"
+    )
+    print("  - Basic markdown to PDF conversion")
+    print("  - Multiple backend comparison (WeasyPrint vs Pandoc)")
+    print("  - Custom CSS styling and professional formatting")
+    print("  - Direct content conversion without file I/O")
+    print("  - Comprehensive error handling and fallback mechanisms")
+
+    results = {}
+
+    # Run demonstrations
+    print("\n🚀 Starting demonstrations...")
+
+    # Basic conversion
+    success, temp_dir = demonstrate_basic_conversion()
+    results["basic"] = success
+
+    # Backend comparison
+    backend_results, _ = demonstrate_backend_comparison()
+    results["backends"] = backend_results
+
+    # Custom styling
+    styling_success, _ = demonstrate_custom_styling()
+    results["styling"] = styling_success
+
+    # Content conversion
+    content_success, _ = demonstrate_content_conversion()
+    results["content"] = content_success
+
+    # Error handling
+    error_results, _ = demonstrate_error_handling()
+    results["error_handling"] = error_results
+
+    # Summary
+    print("\n" + "=" * 70)
+    print("DEMONSTRATION SUMMARY")
+    print("=" * 70)
+
+    print("✅ Features Successfully Demonstrated:")
+    if results["basic"]:
+        print("  - Basic markdown to PDF conversion")
+    if results["backends"]:
+        successful_backends = [
+            b for b, r in results["backends"].items() if r.get("success", False)
+        ]
+        print(f"  - Multiple backends: {successful_backends}")
+    if results["styling"]:
+        print("  - Custom CSS styling and professional formatting")
+    if results["content"]:
+        print("  - Direct content conversion without file I/O")
+    if results["error_handling"]:
+        success_rate = (
+            sum(
+                1 for r in results["error_handling"].values() if r.get("success", False)
+            )
+            / len(results["error_handling"])
+            * 100
+        )
+        print(f"  - Error handling with {success_rate:.1f}% overall success rate")
+
+    print("\n📊 Key Capabilities Highlighted:")
+    print("  - Professional PDF generation with high-quality typography")
+    print("  - Multiple conversion backends with automatic selection")
+    print("  - Extensive CSS customization for branded documents")
+    print("  - Syntax highlighting for code blocks using Pygments")
+    print("  - Table formatting with professional styling")
+    print("  - Image embedding with proper scaling")
+    print("  - Table of contents generation with navigation")
+    print("  - Comprehensive error handling and fallback mechanisms")
+
+    print("\n💡 Best Practices Demonstrated:")
+    print("  - Choose WeasyPrint for web-style documents and custom CSS")
+    print("  - Choose Pandoc for academic papers and complex formatting")
+    print("  - Use 'auto' method for general-purpose conversion")
+    print("  - Implement fallback logic for robust conversion")
+    print("  - Optimize images before embedding in documents")
+    print("  - Test custom CSS with simple content first")
+    print("  - Handle errors gracefully with multiple backend attempts")
+    print("  - Use appropriate page sizes and margins for target use case")
+
+    print("\n🎯 Integration Patterns:")
+    print("  - Standalone conversion for document generation")
+    print("  - Integration with RAG-Anything document pipeline")
+    print("  - API-based document services")
+    print("  - Batch processing for multiple documents")
+    print("  - Dynamic content generation from templates")
+
+
+if __name__ == "__main__":
+    main()

rag_anything_smaranika/examples/image_format_test.py

@@ -0,0 +1,234 @@
+#!/usr/bin/env python3
+"""
+Image Format Parsing Test Script for RAG-Anything
+
+This script demonstrates how to parse various image formats
+using MinerU, including JPG, PNG, BMP, TIFF, GIF, and WebP files.
+
+Requirements:
+- PIL/Pillow library for format conversion
+- RAG-Anything package
+
+Usage:
+    python image_format_test.py --file path/to/image.bmp
+"""
+
+import argparse
+import asyncio
+import sys
+from pathlib import Path
+from raganything import RAGAnything
+
+
+def check_pillow_installation():
+    """Check if PIL/Pillow is installed and available"""
+    try:
+        from PIL import Image
+
+        print(
+            f"✅ PIL/Pillow found: PIL version {Image.__version__ if hasattr(Image, '__version__') else 'Unknown'}"
+        )
+        return True
+    except ImportError:
+        print("❌ PIL/Pillow not found. Please install Pillow:")
+        print("  pip install Pillow")
+        return False
+
+
+def get_image_info(image_path: Path):
+    """Get detailed image information"""
+    try:
+        from PIL import Image
+
+        with Image.open(image_path) as img:
+            return {
+                "format": img.format,
+                "mode": img.mode,
+                "size": img.size,
+                "has_transparency": img.mode in ("RGBA", "LA")
+                or "transparency" in img.info,
+            }
+    except Exception as e:
+        return {"error": str(e)}
+
+
+async def test_image_format_parsing(file_path: str):
+    """Test image format parsing with MinerU"""
+
+    print(f"🧪 Testing image format parsing: {file_path}")
+
+    # Check if file exists and is a supported image format
+    file_path = Path(file_path)
+    if not file_path.exists():
+        print(f"❌ File does not exist: {file_path}")
+        return False
+
+    supported_extensions = {
+        ".jpg",
+        ".jpeg",
+        ".png",
+        ".bmp",
+        ".tiff",
+        ".tif",
+        ".gif",
+        ".webp",
+    }
+    if file_path.suffix.lower() not in supported_extensions:
+        print(f"❌ Unsupported file format: {file_path.suffix}")
+        print(f"   Supported formats: {', '.join(supported_extensions)}")
+        return False
+
+    print(f"📸 File format: {file_path.suffix.upper()}")
+    print(f"📏 File size: {file_path.stat().st_size / 1024:.1f} KB")
+
+    # Get detailed image information
+    img_info = get_image_info(file_path)
+    if "error" not in img_info:
+        print("🖼️  Image info:")
+        print(f"   • Format: {img_info['format']}")
+        print(f"   • Mode: {img_info['mode']}")
+        print(f"   • Size: {img_info['size'][0]}x{img_info['size'][1]}")
+        print(f"   • Has transparency: {img_info['has_transparency']}")
+
+    # Check format compatibility with MinerU
+    mineru_native_formats = {".jpg", ".jpeg", ".png"}
+    needs_conversion = file_path.suffix.lower() not in mineru_native_formats
+
+    if needs_conversion:
+        print(
+            f"ℹ️  Format {file_path.suffix.upper()} will be converted to PNG for MinerU compatibility"
+        )
+    else:
+        print(f"✅ Format {file_path.suffix.upper()} is natively supported by MinerU")
+
+    # Initialize RAGAnything (only for parsing functionality)
+    rag = RAGAnything()
+
+    try:
+        # Test image parsing with MinerU
+        print("\n🔄 Testing image parsing with MinerU...")
+        content_list, md_content = await rag.parse_document(
+            file_path=str(file_path),
+            output_dir="./test_output",
+            parse_method="ocr",  # Images use OCR method
+            display_stats=True,
+        )
+
+        print("✅ Parsing successful!")
+        print(f"   📊 Content blocks: {len(content_list)}")
+        print(f"   📝 Markdown length: {len(md_content)} characters")
+
+        # Analyze content types
+        content_types = {}
+        for item in content_list:
+            if isinstance(item, dict):
+                content_type = item.get("type", "unknown")
+                content_types[content_type] = content_types.get(content_type, 0) + 1
+
+        if content_types:
+            print("   📋 Content distribution:")
+            for content_type, count in sorted(content_types.items()):
+                print(f"      • {content_type}: {count}")
+
+        # Display extracted text (if any)
+        if md_content.strip():
+            print("\n📄 Extracted text preview (first 500 characters):")
+            preview = md_content.strip()[:500]
+            print(f"   {preview}{'...' if len(md_content) > 500 else ''}")
+        else:
+            print("\n📄 No text extracted from the image")
+
+        # Display image processing results
+        image_items = [
+            item
+            for item in content_list
+            if isinstance(item, dict) and item.get("type") == "image"
+        ]
+        if image_items:
+            print(f"\n🖼️  Found {len(image_items)} processed image(s):")
+            for i, item in enumerate(image_items, 1):
+                print(f"   {i}. Image path: {item.get('img_path', 'N/A')}")
+                caption = item.get("image_caption", item.get("img_caption", []))
+                if caption:
+                    print(f"      Caption: {caption[0] if caption else 'N/A'}")
+
+        # Display text blocks (OCR results)
+        text_items = [
+            item
+            for item in content_list
+            if isinstance(item, dict) and item.get("type") == "text"
+        ]
+        if text_items:
+            print("\n📝 OCR text blocks found:")
+            for i, item in enumerate(text_items, 1):
+                text_content = item.get("text", "")
+                if text_content.strip():
+                    preview = text_content.strip()[:200]
+                    print(
+                        f"   {i}. {preview}{'...' if len(text_content) > 200 else ''}"
+                    )
+
+        # Check for any tables detected in the image
+        table_items = [
+            item
+            for item in content_list
+            if isinstance(item, dict) and item.get("type") == "table"
+        ]
+        if table_items:
+            print(f"\n📊 Found {len(table_items)} table(s) in image:")
+            for i, item in enumerate(table_items, 1):
+                print(f"   {i}. Table detected with content")
+
+        print("\n🎉 Image format parsing test completed successfully!")
+        print("📁 Output files saved to: ./test_output")
+        return True
+
+    except Exception as e:
+        print(f"\n❌ Image format parsing failed: {str(e)}")
+        import traceback
+
+        print(f"   Full error: {traceback.format_exc()}")
+        return False
+
+
+def main():
+    """Main function"""
+    parser = argparse.ArgumentParser(
+        description="Test image format parsing with MinerU"
+    )
+    parser.add_argument("--file", help="Path to the image file to test")
+    parser.add_argument(
+        "--check-pillow", action="store_true", help="Only check PIL/Pillow installation"
+    )
+
+    args = parser.parse_args()
+
+    # Check PIL/Pillow installation
+    print("🔧 Checking PIL/Pillow installation...")
+    if not check_pillow_installation():
+        return 1
+
+    if args.check_pillow:
+        print("✅ PIL/Pillow installation check passed!")
+        return 0
+
+    # If not just checking dependencies, file argument is required
+    if not args.file:
+        print("❌ Error: --file argument is required when not using --check-pillow")
+        parser.print_help()
+        return 1
+
+    # Run the parsing test
+    try:
+        success = asyncio.run(test_image_format_parsing(args.file))
+        return 0 if success else 1
+    except KeyboardInterrupt:
+        print("\n⏹️ Test interrupted by user")
+        return 1
+    except Exception as e:
+        print(f"\n❌ Unexpected error: {str(e)}")
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

rag_anything_smaranika/examples/insert_content_list_example.py

@@ -0,0 +1,419 @@
+#!/usr/bin/env python
+"""
+Example script demonstrating direct content list insertion with RAGAnything
+
+This example shows how to:
+1. Create a simple content list with different content types
+2. Insert content list directly without document parsing using insert_content_list() method
+3. Perform pure text queries using aquery() method
+4. Perform multimodal queries with specific multimodal content using aquery_with_multimodal() method
+5. Handle different types of multimodal content in the inserted knowledge base
+"""
+
+import os
+import argparse
+import asyncio
+import logging
+import logging.config
+from pathlib import Path
+
+# Add project root directory to Python path
+import sys
+
+sys.path.append(str(Path(__file__).parent.parent))
+
+from lightrag.llm.openai import openai_complete_if_cache, openai_embed
+from lightrag.utils import EmbeddingFunc, logger, set_verbose_debug
+from raganything import RAGAnything, RAGAnythingConfig
+
+from dotenv import load_dotenv
+
+load_dotenv(dotenv_path=".env", override=False)
+
+
+def configure_logging():
+    """Configure logging for the application"""
+    # Get log directory path from environment variable or use current directory
+    log_dir = os.getenv("LOG_DIR", os.getcwd())
+    log_file_path = os.path.abspath(
+        os.path.join(log_dir, "insert_content_list_example.log")
+    )
+
+    print(f"\nInsert Content List example log file: {log_file_path}\n")
+    os.makedirs(os.path.dirname(log_dir), exist_ok=True)
+
+    # Get log file max size and backup count from environment variables
+    log_max_bytes = int(os.getenv("LOG_MAX_BYTES", 10485760))  # Default 10MB
+    log_backup_count = int(os.getenv("LOG_BACKUP_COUNT", 5))  # Default 5 backups
+
+    logging.config.dictConfig(
+        {
+            "version": 1,
+            "disable_existing_loggers": False,
+            "formatters": {
+                "default": {
+                    "format": "%(levelname)s: %(message)s",
+                },
+                "detailed": {
+                    "format": "%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+                },
+            },
+            "handlers": {
+                "console": {
+                    "formatter": "default",
+                    "class": "logging.StreamHandler",
+                    "stream": "ext://sys.stderr",
+                },
+                "file": {
+                    "formatter": "detailed",
+                    "class": "logging.handlers.RotatingFileHandler",
+                    "filename": log_file_path,
+                    "maxBytes": log_max_bytes,
+                    "backupCount": log_backup_count,
+                    "encoding": "utf-8",
+                },
+            },
+            "loggers": {
+                "lightrag": {
+                    "handlers": ["console", "file"],
+                    "level": "INFO",
+                    "propagate": False,
+                },
+            },
+        }
+    )
+
+    # Set the logger level to INFO
+    logger.setLevel(logging.INFO)
+    # Enable verbose debug if needed
+    set_verbose_debug(os.getenv("VERBOSE", "false").lower() == "true")
+
+
+def create_sample_content_list():
+    """
+    Create a simple content list for testing insert_content_list functionality
+
+    Returns:
+        List[Dict]: Sample content list with various content types
+
+    Note:
+        - img_path should be absolute path to the image file
+        - page_idx represents the page number where the content appears (0-based)
+    """
+    content_list = [
+        # Introduction text
+        {
+            "type": "text",
+            "text": "Welcome to the RAGAnything System Documentation. This guide covers the advanced multimodal document processing capabilities and features of our comprehensive RAG system.",
+            "page_idx": 0,  # Page number where this content appears
+        },
+        # System architecture image
+        {
+            "type": "image",
+            "img_path": "/absolute/path/to/system_architecture.jpg",  # IMPORTANT: Use absolute path to image file
+            "image_caption": ["Figure 1: RAGAnything System Architecture"],
+            "image_footnote": [
+                "The architecture shows the complete pipeline from document parsing to multimodal query processing"
+            ],
+            "page_idx": 1,  # Page number where this image appears
+        },
+        # Performance comparison table
+        {
+            "type": "table",
+            "table_body": """| System | Accuracy | Processing Speed | Memory Usage |
+                            |--------|----------|------------------|--------------|
+                            | RAGAnything | 95.2% | 120ms | 2.1GB |
+                            | Traditional RAG | 87.3% | 180ms | 3.2GB |
+                            | Baseline System | 82.1% | 220ms | 4.1GB |
+                            | Simple Retrieval | 76.5% | 95ms | 1.8GB |""",
+            "table_caption": [
+                "Table 1: Performance Comparison of Different RAG Systems"
+            ],
+            "table_footnote": [
+                "All tests conducted on the same hardware with identical test datasets"
+            ],
+            "page_idx": 2,  # Page number where this table appears
+        },
+        # Mathematical formula
+        {
+            "type": "equation",
+            "latex": "Relevance(d, q) = \\sum_{i=1}^{n} w_i \\cdot sim(t_i^d, t_i^q) \\cdot \\alpha_i",
+            "text": "Document relevance scoring formula where w_i are term weights, sim() is similarity function, and α_i are modality importance factors",
+            "page_idx": 3,  # Page number where this equation appears
+        },
+        # Feature description
+        {
+            "type": "text",
+            "text": "The system supports multiple content modalities including text, images, tables, and mathematical equations. Each modality is processed using specialized processors optimized for that content type.",
+            "page_idx": 4,  # Page number where this content appears
+        },
+        # Technical specifications table
+        {
+            "type": "table",
+            "table_body": """| Feature | Specification |
+                            |---------|---------------|
+                            | Supported Formats | PDF, DOCX, PPTX, XLSX, Images |
+                            | Max Document Size | 100MB |
+                            | Concurrent Processing | Up to 8 documents |
+                            | Query Response Time | <200ms average |
+                            | Knowledge Graph Nodes | Up to 1M entities |""",
+            "table_caption": ["Table 2: Technical Specifications"],
+            "table_footnote": [
+                "Specifications may vary based on hardware configuration"
+            ],
+            "page_idx": 5,  # Page number where this table appears
+        },
+        # Conclusion
+        {
+            "type": "text",
+            "text": "RAGAnything represents a significant advancement in multimodal document processing, providing comprehensive solutions for complex knowledge extraction and retrieval tasks.",
+            "page_idx": 6,  # Page number where this content appears
+        },
+    ]
+
+    return content_list
+
+
+async def demo_insert_content_list(
+    api_key: str,
+    base_url: str = None,
+    working_dir: str = None,
+):
+    """
+    Demonstrate content list insertion and querying with RAGAnything
+
+    Args:
+        api_key: OpenAI API key
+        base_url: Optional base URL for API
+        working_dir: Working directory for RAG storage
+    """
+    try:
+        # Create RAGAnything configuration
+        config = RAGAnythingConfig(
+            working_dir=working_dir or "./rag_storage",
+            enable_image_processing=True,
+            enable_table_processing=True,
+            enable_equation_processing=True,
+            display_content_stats=True,  # Show content statistics
+        )
+
+        # Define LLM model function
+        def llm_model_func(prompt, system_prompt=None, history_messages=[], **kwargs):
+            return openai_complete_if_cache(
+                "gpt-4o-mini",
+                prompt,
+                system_prompt=system_prompt,
+                history_messages=history_messages,
+                api_key=api_key,
+                base_url=base_url,
+                **kwargs,
+            )
+
+        # Define vision model function for image processing
+        def vision_model_func(
+            prompt, system_prompt=None, history_messages=[], image_data=None, **kwargs
+        ):
+            if image_data:
+                return openai_complete_if_cache(
+                    "gpt-4o",
+                    "",
+                    system_prompt=None,
+                    history_messages=[],
+                    messages=[
+                        {"role": "system", "content": system_prompt}
+                        if system_prompt
+                        else None,
+                        {
+                            "role": "user",
+                            "content": [
+                                {"type": "text", "text": prompt},
+                                {
+                                    "type": "image_url",
+                                    "image_url": {
+                                        "url": f"data:image/jpeg;base64,{image_data}"
+                                    },
+                                },
+                            ],
+                        }
+                        if image_data
+                        else {"role": "user", "content": prompt},
+                    ],
+                    api_key=api_key,
+                    base_url=base_url,
+                    **kwargs,
+                )
+            else:
+                return llm_model_func(prompt, system_prompt, history_messages, **kwargs)
+
+        # Define embedding function
+        embedding_func = EmbeddingFunc(
+            embedding_dim=3072,
+            max_token_size=8192,
+            func=lambda texts: openai_embed(
+                texts,
+                model="text-embedding-3-large",
+                api_key=api_key,
+                base_url=base_url,
+            ),
+        )
+
+        # Initialize RAGAnything
+        rag = RAGAnything(
+            config=config,
+            llm_model_func=llm_model_func,
+            vision_model_func=vision_model_func,
+            embedding_func=embedding_func,
+        )
+
+        # Create sample content list
+        logger.info("Creating sample content list...")
+        content_list = create_sample_content_list()
+        logger.info(f"Created content list with {len(content_list)} items")
+
+        # Insert content list directly
+        logger.info("\nInserting content list into RAGAnything...")
+        await rag.insert_content_list(
+            content_list=content_list,
+            file_path="raganything_documentation.pdf",  # Reference file name for citation
+            split_by_character=None,  # Optional text splitting
+            split_by_character_only=False,  # Optional text splitting mode
+            doc_id="demo-doc-001",  # Custom document ID
+            display_stats=True,  # Show content statistics
+        )
+        logger.info("Content list insertion completed!")
+
+        # Example queries - demonstrating different query approaches
+        logger.info("\nQuerying inserted content:")
+
+        # 1. Pure text queries using aquery()
+        text_queries = [
+            "What is RAGAnything and what are its main features?",
+            "How does RAGAnything compare to traditional RAG systems?",
+            "What are the technical specifications of the system?",
+        ]
+
+        for query in text_queries:
+            logger.info(f"\n[Text Query]: {query}")
+            result = await rag.aquery(query, mode="hybrid")
+            logger.info(f"Answer: {result}")
+
+        # 2. Multimodal query with specific multimodal content using aquery_with_multimodal()
+        logger.info(
+            "\n[Multimodal Query]: Analyzing new performance data against existing benchmarks"
+        )
+        multimodal_result = await rag.aquery_with_multimodal(
+            "Compare this new performance data with the existing benchmark results in the documentation",
+            multimodal_content=[
+                {
+                    "type": "table",
+                    "table_data": """Method,Accuracy,Speed,Memory
+                                New_Approach,97.1%,110ms,1.9GB
+                                Enhanced_RAG,91.4%,140ms,2.5GB""",
+                    "table_caption": "Latest experimental results",
+                }
+            ],
+            mode="hybrid",
+        )
+        logger.info(f"Answer: {multimodal_result}")
+
+        # 3. Another multimodal query with equation content
+        logger.info("\n[Multimodal Query]: Mathematical formula analysis")
+        equation_result = await rag.aquery_with_multimodal(
+            "How does this similarity formula relate to the relevance scoring mentioned in the documentation?",
+            multimodal_content=[
+                {
+                    "type": "equation",
+                    "latex": "sim(a, b) = \\frac{a \\cdot b}{||a|| \\times ||b||} + \\beta \\cdot context\\_weight",
+                    "equation_caption": "Enhanced cosine similarity with context weighting",
+                }
+            ],
+            mode="hybrid",
+        )
+        logger.info(f"Answer: {equation_result}")
+
+        # 4. Insert another content list with different document ID
+        logger.info("\nInserting additional content list...")
+        additional_content = [
+            {
+                "type": "text",
+                "text": "This is additional documentation about advanced features and configuration options.",
+                "page_idx": 0,  # Page number where this content appears
+            },
+            {
+                "type": "table",
+                "table_body": """| Configuration | Default Value | Range |
+                                    |---------------|---------------|-------|
+                                    | Chunk Size | 512 tokens | 128-2048 |
+                                    | Context Window | 4096 tokens | 1024-8192 |
+                                    | Batch Size | 32 | 1-128 |""",
+                "table_caption": ["Advanced Configuration Parameters"],
+                "page_idx": 1,  # Page number where this table appears
+            },
+        ]
+
+        await rag.insert_content_list(
+            content_list=additional_content,
+            file_path="advanced_configuration.pdf",
+            doc_id="demo-doc-002",  # Different document ID
+        )
+
+        # Query combined knowledge base
+        logger.info("\n[Combined Query]: What configuration options are available?")
+        combined_result = await rag.aquery(
+            "What configuration options are available and what are their default values?",
+            mode="hybrid",
+        )
+        logger.info(f"Answer: {combined_result}")
+
+    except Exception as e:
+        logger.error(f"Error in content list insertion demo: {str(e)}")
+        import traceback
+
+        logger.error(traceback.format_exc())
+
+
+def main():
+    """Main function to run the example"""
+    parser = argparse.ArgumentParser(description="Insert Content List Example")
+    parser.add_argument(
+        "--working_dir", "-w", default="./rag_storage", help="Working directory path"
+    )
+    parser.add_argument(
+        "--api-key",
+        default=os.getenv("LLM_BINDING_API_KEY"),
+        help="OpenAI API key (defaults to LLM_BINDING_API_KEY env var)",
+    )
+    parser.add_argument(
+        "--base-url",
+        default=os.getenv("LLM_BINDING_HOST"),
+        help="Optional base URL for API",
+    )
+
+    args = parser.parse_args()
+
+    # Check if API key is provided
+    if not args.api_key:
+        logger.error("Error: OpenAI API key is required")
+        logger.error("Set api key environment variable or use --api-key option")
+        return
+
+    # Run the demo
+    asyncio.run(
+        demo_insert_content_list(
+            args.api_key,
+            args.base_url,
+            args.working_dir,
+        )
+    )
+
+
+if __name__ == "__main__":
+    # Configure logging first
+    configure_logging()
+
+    print("RAGAnything Insert Content List Example")
+    print("=" * 45)
+    print("Demonstrating direct content list insertion without document parsing")
+    print("=" * 45)
+
+    main()