Clean commit without images

.dockerignore

@@ -0,0 +1,70 @@
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+# Byte-compiled / optimized / DLL files
+# C extensions
+# Distribution / packaging
+# Django stuff:
+# Installer logs
+# Project specific ignores
+# PyBuilder
+# PyInstaller
+# Sphinx documentation
+# Translations
+# Unit test / coverage reports
+*$py.class
+*,cover
+*.csv
+*.egg
+*.egg-info/
+*.log
+*.manifest
+*.mo
+*.pkl
+*.pot
+*.py[cod]
+*.pyc
+*.so
+*.spec
+.Python
+.cache
+.coverage
+.coverage.*
+.dockerignore
+.eggs/
+.env
+.git
+.gitignore
+.idea/
+.installed.cfg
+.python-version
+.tox/
+.venv
+.vscode/
+__pycache__/
+artifact/
+artifacts/
+build/
+coverage.xml
+develop-eggs/
+dist/
+docs/_build/
+downloads/
+eggs/
+env/
+housepricepredictor.egg-info/
+htmlcov/
+lib/
+lib64/
+logs/
+mlruns/
+myenv/
+nosetests.xml
+notebook/
+parts/
+pip-delete-this-directory.txt
+pip-log.txt
+sdist/
+target/
+uv.lock
+var/
+venv/

.env.example

@@ -0,0 +1,15 @@
+GROQ_API_KEY=
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=
+AWS_DEFAULT_REGION=
+HUGGINGFACEHUB_ACCESS_TOKEN=
+PORT=
+
+
+
+
+TAVILY_API_KEY=
+Gemini_API_Key=
+HF_TOKEN=
+
+APP_API_KEY=

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,208 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+#poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+#pdm.lock
+#pdm.toml
+.pdm-python
+.pdm-build/
+
+# pixi
+#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+#pixi.lock
+#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+#   in the .venv directory. It is recommended not to include this directory in version control.
+.pixi
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.envrc
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Abstra
+# Abstra is an AI-powered process automation framework.
+# Ignore directories containing user credentials, local state, and settings.
+# Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#  Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
+#  that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#  and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#  you could uncomment the following to ignore the entire vscode folder
+# .vscode/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Cursor
+#  Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
+#  exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
+#  refer to https://docs.cursor.com/context/ignore-files
+.cursorignore
+.cursorindexingignore
+
+# Marimo
+marimo/_static/
+marimo/_lsp/
+__marimo__/
+images/

.python-version

@@ -0,0 +1 @@
+3.12

Dockerfile

@@ -0,0 +1,17 @@
+FROM python:3.12-slim
+
+WORKDIR /app
+
+# Copy dependency files first (better caching)
+COPY requirements.txt pyproject.toml ./
+
+# Install dependencies
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy rest of the application
+COPY . .
+
+EXPOSE 7860
+
+# Run FastAPI app
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Vansh Sharma
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,134 @@
+
+
+---
+title: Multi-Rag AI
+emoji: 🐠
+colorFrom: blue
+colorTo: green
+sdk: docker
+app_file: Dockerfile
+app_port: 7860
+pinned: false
+---
+
+
+<div align="center">
+  <h1>🤖 AIAgents Platform</h1>
+  <p><strong>Intelligent AI Agents Powered by LangGraph, LangChain, and FastAPI</strong></p>
+</div>
+
+<br />
+
+Welcome to **AIAgents**, a full-stack, state-of-the-art framework for building and deploying extremely scalable, multi-agent AI ecosystems! Featuring powerful autonomous agents for complex Web Research, Blog Generation, Document RAG functionality, and interactive multi-turn chatting!
+
+---
+
+## 🚀 Features
+
+- **✍️ Bloggig (Blog Agent)**: Powerful autonomous agent that researches, writes, and generates high-quality blog posts complete with AI-generated visuals.
+- **🌐 Web Research Agent**: Automatically browse, scrape, and synthesize live internet data straight from any URL (including YouTube videos!) directly within the web interface.
+- **📚 Multi-turn RAG Chat**: Chat with arbitrary text or PDF documents using deep LangGraph memory, powerful sentence transformers for vector retrieval, and advanced orchestration logic.
+- **🎨 Stunning UI**: Beautiful, fully-responsive, custom Dark Mode interface crafted natively with Jinja2 Templating, vanilla HTML/CSS/JS, and glassmorphism UI elements.
+- **⚡ Supercharged Backend**: High-performance asynchronous API crafted using FastAPI.
+- **🛠️ Extensible AI Architecture**: Built on top of the robust **LangChain** and **LangGraph** Python ecosystem to allow autonomous scaling of multi-agent workflows.
+
+## 🛠️ Tech Stack
+
+- **Backend**: Python 3.12+, FastAPI, Uvicorn
+- **AI Frameworks**: LangChain, LangGraph, Sentence-Transformers, HuggingFace
+- **LLMs**: AWS Bedrock (Claude 3.5 Sonnet, Claude 3 Haiku, Llama 3), OpenAI (GPT-4o)
+- **Vector Database**: FAISS (Facebook AI Similarity Search)
+- **Frontend**: Jinja2 Templates, Vanilla JS, CSS3, DOM manipulation
+- **Development Tooling**: `uv` (Fast Python Package Manager)
+
+---
+
+## ⚙️ Quickstart
+
+### Prerequisites
+
+- Ensure you have **Python >= 3.12** installed on your system.
+- Make sure you are using [uv](https://github.com/astral-sh/uv) to manage project dependencies!
+
+### 1. Installation
+
+1. **Clone the repository**:
+```bash
+git clone https://github.com/VashuTheGreat/AiAgents.git
+cd AiAgents
+```
+
+2. **Set up the virtual environment & install dependencies** using `uv`:
+```bash
+uv sync
+```
+
+### 2. Environment Variables
+
+Create a `.env` file in the root of the project and place your necessary API keys inside.
+
+```env
+# General
+APP_API_KEY="your_custom_auth_key"
+
+# AWS Bedrock (For Blog Agent)
+AWS_ACCESS_KEY_ID="your_key"
+AWS_SECRET_ACCESS_KEY="your_secret"
+AWS_REGION_NAME="us-east-1"
+
+# OpenAI
+OPENAI_API_KEY="sk-..."
+```
+
+### 3. Run the Server
+
+Simply launch the FastAPI application:
+```bash
+uv run .\main.py
+```
+This will start the development server. Navigate to `http://127.0.0.1:8000/` to see the AIAgents Hub!
+
+---
+
+## 🎨 Walkthrough of the Application
+
+### 🏠 Home Page (`/`)
+An elegant gateway into the available AI agent interfaces.
+
+### ✍️ Blog Agent (`/blog`)
+The flagship feature. Enter a topic, and Bloggig will autonomously research the subject, plan its structure, write the content in Markdown, and generate relevant images. It features a real-time "pipeline console" to track the agent's progress.
+
+### 🌐 Web Summarizer (`/web`)
+Paste any URL or YouTube Link to extract and summarize content using our custom LangGraph architecture.
+
+### 💬 Chat MultiGraph (`/chat`)
+Engage with your locally uploaded documents via RAG (Retrieval-Augmented Generation) with intelligent memory buffers.
+
+---
+
+## 📂 Project Structure
+
+```bash
+AiAgents/
+├─ api/
+│  ├─ Blog/           # Bloggig-specific routers and models
+│  ├─ MultiRag/       # Document RAG routers
+│  └─ Web/            # Web Summarizer routers
+├─ src/
+│  ├─ Blog/           # Bloggig Agent logic (Graph, Nodes, Prompts)
+│  ├─ MultiRag/       # RAG Agent logic (Retrievers, Vectorstores, etc.)
+│  └─ Web/            # Web Agent logic (Loaders, Graph)
+├─ images/            # Generated blog visualizations
+├─ results/           # Saved blog markdown outputs
+├─ static/            # CSS, JS, and local frontend assets
+├─ templates/         # Jinja2 HTML templates
+├─ data/              # Raw document storage for RAG
+├─ db/                # Local FAISS vector database storage
+└─ pyproject.toml     # Project dependencies (uv)
+```
+
+---
+
+<div align="center">
+  <p>Crafted with ❤️ for professional creators.</p>
+</div>

api/Blog/models/models.py

@@ -0,0 +1,3 @@
+from pydantic import BaseModel
+class BlogDeleteRequest(BaseModel):
+    data: dict

api/Blog/routes/blog_router.py

@@ -0,0 +1,123 @@
+import re
+import os
+import sys
+import logging
+import zipfile
+import io
+from fastapi import  HTTPException, WebSocket, WebSocketDisconnect
+from fastapi.responses import  StreamingResponse
+from fastapi.staticfiles import StaticFiles
+from api.Blog.models.models import BlogDeleteRequest
+import fastapi
+import uvicorn as uv
+
+PROJECT_ROOT = os.getcwd()
+if PROJECT_ROOT not in sys.path:
+    sys.path.append(PROJECT_ROOT)
+
+from logger import *
+from src.Blog.graph.Compile_graph import run
+from src.Blog.utils.blog_utils import delete_blog_content
+
+router=fastapi.APIRouter()
+os.makedirs("images", exist_ok=True)
+os.makedirs("results", exist_ok=True)
+
+# Mount static files
+router.mount("/images", StaticFiles(directory="images"), name="images")
+router.mount("/results", StaticFiles(directory="results"), name="results")
+
+
+
+
+
+@router.get("/blogs")
+async def list_blogs():
+    results_dir = "results"
+    if not os.path.exists(results_dir):
+        return []
+    blogs = [f[:-3] for f in os.listdir(results_dir) if f.endswith(".md") and f != "README.md"]
+    return blogs
+
+@router.get("/blog/{title}")
+async def get_blog(title: str):
+    file_path = os.path.join("results", f"{title}.md")
+    if not os.path.exists(file_path):
+        raise HTTPException(status_code=404, detail="Blog not found")
+    with open(file_path, "r", encoding="utf-8") as f:
+        content = f.read()
+    return {"title": title, "content": content}
+
+from fastapi.encoders import jsonable_encoder
+
+@router.websocket("/ws/generate_blog")
+async def generate_blog_ws(websocket: WebSocket):
+    await websocket.accept()
+    try:
+        data = await websocket.receive_json()
+        topic = data.get("topic")
+        if not topic:
+            await websocket.send_json({"error": "Topic is required"})
+            await websocket.close()
+            return
+
+        logging.info(f"WebSocket: Starting blog generation for topic: {topic}")
+        
+        async for step in run(topic):
+            serializable_step = jsonable_encoder(step)
+            await websocket.send_json(serializable_step)
+            
+        await websocket.send_json({"status": "completed"})
+    except WebSocketDisconnect:
+        logging.info("WebSocket disconnected")
+    except Exception as e:
+        logging.error(f"WebSocket error: {str(e)}")
+        await websocket.send_json({"error": str(e)})
+    finally:
+        try:
+            await websocket.close()
+        except:
+            pass
+
+@router.delete("/delete_blog")
+async def delete_blog(request: BlogDeleteRequest):
+    success = delete_blog_content(request.data)
+    if success:
+        return {"message": "Blog and associated images deleted successfully"}
+    else:
+        raise HTTPException(status_code=404, detail="Blog not found or could not be deleted")
+
+@router.get("/download_blog/{title}")
+async def download_blog(title: str):
+    md_path = os.path.join("results", f"{title}.md")
+    if not os.path.exists(md_path):
+        raise HTTPException(status_code=404, detail="Blog not found")
+
+    with open(md_path, "r", encoding="utf-8") as f:
+        content = f.read()
+
+    # Find images
+    image_pattern = r"!\[.*?\]\(\.\./images/(.*?)\)"
+    image_filenames = re.findall(image_pattern, content)
+
+    # Create zip in memory
+    zip_buffer = io.BytesIO()
+    with zipfile.ZipFile(zip_buffer, "a", zipfile.ZIP_DEFLATED, False) as zip_file:
+        # Add markdown file
+        zip_file.writestr(f"{title}.md", content)
+        
+        # Add images
+        for img_name in image_filenames:
+            img_path = os.path.join("images", img_name)
+            if os.path.exists(img_path):
+                zip_file.write(img_path, os.path.join("images", img_name))
+
+    zip_buffer.seek(0)
+    return StreamingResponse(
+        zip_buffer,
+        media_type="application/x-zip-compressed",
+        headers={"Content-Disposition": f"attachment; filename={title}.zip"}
+    )
+
+if __name__ == "__main__":
+    uv.run("app:app", host="0.0.0.0", port=8000, reload=False)

api/Blog/routes/page_route_blog.py

@@ -0,0 +1,16 @@
+import os
+import fastapi
+from fastapi.templating import Jinja2Templates
+from fastapi import Request
+
+router = fastapi.APIRouter()
+templates = Jinja2Templates(directory="templates")
+
+_APP_USER_ID = os.getenv("APP_API_KEY", "")
+
+@router.get("/blog")
+async def blog_root(request: Request):
+    return templates.TemplateResponse(
+        name="blog.html",
+        context={"request": request, "app_user_id": _APP_USER_ID}
+    )

api/MultiRag/routes/chat_route.py

@@ -0,0 +1,44 @@
+from fastapi import APIRouter, Request, Query
+import logging
+import logging
+from src.MultiRag.graph.builder import graph
+
+router = APIRouter()
+
+
+async def run_agent(user_id, userQuery: str):
+    logging.info("Starting AIAgents application...")
+    # Sample initial state for testing
+    config = {"configurable": {"thread_id": user_id}}
+    initial_state = {
+        "userQuery": userQuery,
+        "db_path": f"db/{user_id}",
+        "docs_path": f"data/{user_id}",
+        "k": 3
+    }
+    try:
+        response = await graph.ainvoke(initial_state, config=config)
+        logging.debug(f"Graph response: {response}")
+        logging.info("Graph invocation successful.")
+        res = response.get("llm_response", "No response found.")
+        return res
+    except Exception as e:
+        logging.error(f"Application failed: {e}")
+        import traceback
+        logging.error(traceback.format_exc())
+        return "Chat failed due to internal error"
+    finally:
+        logging.info("AIAgents application finished.")
+
+
+@router.post("/chat")
+async def chat(req: Request, message: str = Query(...)):
+    try:
+        user_id = req.headers.get("user_id")
+        if not user_id:
+            return {"data": "User ID missing in headers"}
+        res = await run_agent(user_id, message)
+        return {"data": res}
+    except Exception as e:
+        logging.error(f"Chat endpoint error: {e}")
+        return {"data": "Chat failed"}

api/MultiRag/routes/pages_route.py

@@ -0,0 +1,31 @@
+import os
+import fastapi
+from fastapi import Request
+from fastapi.templating import Jinja2Templates
+
+router = fastapi.APIRouter()
+templates = Jinja2Templates(directory="templates")
+
+_APP_USER_ID = os.getenv("APP_API_KEY", "")
+
+@router.get("/")
+async def read_root(request: Request):
+    return templates.TemplateResponse(
+        name="home.html",
+        context={"request": request, "app_user_id": _APP_USER_ID}
+    )
+
+
+@router.get("/chat")
+async def chat_model(request: Request):
+    return templates.TemplateResponse(
+        name="chat.html",
+        context={"request": request, "app_user_id": _APP_USER_ID}
+    )
+
+@router.get("/web")
+async def web_page(request: Request):
+    return templates.TemplateResponse(
+        name="web.html",
+        context={"request": request, "app_user_id": _APP_USER_ID}
+    )

api/MultiRag/routes/uploader_route.py

@@ -0,0 +1,73 @@
+import fastapi
+from fastapi import UploadFile, Request, BackgroundTasks
+import os
+import shutil
+import asyncio
+import logging
+from src.MultiRag.constants import CONTENT_PERSISTENT_TIME, DATA_FOLDER_PATH, DB_FOLDER_PATH
+from src.MultiRag.graph.builder import deleteThread
+from utils.asyncHandler import asyncHandler
+from src.MultiRag.nodes.retreiver_check_node import clear_cached_retriever
+router = fastapi.APIRouter()
+
+
+@asyncHandler
+async def delete_folder_after_time(user_id):
+
+    await asyncio.sleep(CONTENT_PERSISTENT_TIME)
+
+    folder_path = f"{DATA_FOLDER_PATH}/{user_id}"
+    db_path = f"{DB_FOLDER_PATH}/{user_id}"
+
+    # Step 1: null refs, gc.collect(), clear_system_cache() — in that order
+    clear_cached_retriever(db_path)
+    await deleteThread(user_id)
+
+    # Step 2: give Windows 3s to fully release OS-level file locks after GC
+    await asyncio.sleep(3)
+
+    if os.path.exists(folder_path):
+        shutil.rmtree(folder_path)
+        logging.info(f"Folder deleted: {folder_path}")
+
+    if os.path.exists(db_path):
+        for attempt in range(6):
+            try:
+                shutil.rmtree(db_path)
+                logging.info(f"DB deleted: {db_path}")
+                return
+            except PermissionError as e:
+                logging.warning(f"DB delete attempt {attempt+1} failed: {e}")
+                await asyncio.sleep(3)
+
+        logging.error(f"Failed to delete DB after all retries: {db_path}")
+
+
+
+    
+
+
+@router.post("/post_content")
+async def post_content(
+    req: Request,
+    file: UploadFile,
+    background_tasks: BackgroundTasks
+):
+    try:
+        user_id = req.headers.get("user_id")
+
+        folder = f"{DATA_FOLDER_PATH}/{user_id}"
+        os.makedirs(folder, exist_ok=True)
+
+        file_path = f"{folder}/{file.filename}"
+
+        with open(file_path, "wb") as f:
+            f.write(await file.read())
+
+        # start background delete timer
+        background_tasks.add_task(delete_folder_after_time, user_id)
+
+        return {"message": "File uploaded successfully"}
+
+    except Exception as e:
+        return {"message": "File upload failed"}

api/Web/routes/page_route_web.py

@@ -0,0 +1,16 @@
+import os
+import fastapi
+from fastapi import Request
+from fastapi.templating import Jinja2Templates
+
+router = fastapi.APIRouter()
+templates = Jinja2Templates(directory="templates")
+
+_APP_USER_ID = os.getenv("APP_API_KEY", "")
+
+@router.get("/web")
+async def web_page(request: Request):
+    return templates.TemplateResponse(
+        name="web.html",
+        context={"request": request, "app_user_id": _APP_USER_ID}
+    )

api/Web/routes/web_talk_routes.py

@@ -0,0 +1,13 @@
+import fastapi
+from src.Web.graph.builder import graph
+router=fastapi.APIRouter()
+
+@router.post("/web_summerizer")
+async def web_sum(url:str):
+
+    res=await graph.ainvoke({
+        "url":url
+    })
+    print(res)
+
+    return {"data": res.get("llm_response", "Error generating response")}

api/main.py

@@ -0,0 +1,55 @@
+from fastapi import FastAPI, Request
+from fastapi.responses import JSONResponse
+from api.MultiRag.routes import chat_route, uploader_route, pages_route
+from api.Web.routes import web_talk_routes
+from api.Blog.routes import page_route_blog,blog_router
+from api.Web.routes import page_route_web
+app = FastAPI()
+
+@app.middleware("http")
+async def check_user_id(request: Request, call_next):
+    # Skip middleware for static files and page routes to allow initial connection
+    public_routes = [
+        "/",
+        "/chat",
+        "/web",
+        "/blog",
+        "/docs",
+        "/redoc",
+        "/openapi.json",
+        "/favicon.ico",
+    ]
+    if request.url.path.startswith("/static") or request.url.path.startswith("/blog/images") or request.url.path in public_routes:
+        return await call_next(request)
+
+    user_id = request.headers.get("user_id") or request.query_params.get("user_id")
+
+    if not user_id:
+        return JSONResponse(
+            status_code=401,
+            content={"message": "user_id header missing"}
+        )
+
+    response = await call_next(request)
+    return response
+
+app.include_router(pages_route.router)
+app.include_router(prefix="/chat", router=chat_route.router)
+app.include_router(prefix="/uploader", router=uploader_route.router)
+
+
+
+
+
+# -------------------- Web -------------------------------
+app.include_router(page_route_web.router)
+app.include_router(prefix="/web",router=web_talk_routes.router)
+
+
+
+
+
+# ------------ Blog --------------------
+app.include_router(page_route_blog.router)
+app.include_router(prefix="/blog",router=blog_router.router)
+

constants/__init__.py

@@ -0,0 +1 @@
+LOGS_DIR="logs"

exception/__init__.py

@@ -0,0 +1,35 @@
+import sys
+import logging
+
+def error_message_detail(error:Exception,error_detail:sys)->str:
+    _, _, exc_tb = error_detail.exc_info()
+
+    # Walk the traceback to find the actual source of the error
+    while exc_tb.tb_next is not None:
+        exc_tb = exc_tb.tb_next
+
+    # Get the file name where the exception occurred
+    file_name = exc_tb.tb_frame.f_code.co_filename
+
+    # Create a formatted error message string with file name, line number, and the actual error
+    line_number = exc_tb.tb_lineno
+    error_message = f"Error occurred in python script: [{file_name}] at line number [{line_number}]: {str(error)}"
+    
+    # Log the error for better tracking
+    logging.error(error_message)
+    
+    return error_message
+
+class MyException(Exception):
+    def __init__(self, error_message: str, error_detail: sys):
+        # Call the base class constructor with the error message
+        super().__init__(error_message)
+
+        # Format the detailed error message using the error_message_detail function
+        self.error_message = error_message_detail(error_message, error_detail)
+
+    def __str__(self) -> str:
+        """
+        Returns the string representation of the error message.
+        """
+        return self.error_message

logger/__init__.py

@@ -0,0 +1,65 @@
+import logging
+import os
+from logging.handlers import RotatingFileHandler
+from datetime import datetime
+from constants import LOGS_DIR
+
+# Log File Configuration
+LOG_FILE = f"{datetime.now().strftime('%m_%d_%Y_%H_%M_%S')}.log"
+MAX_FOLDER_SIZE = 2 * 1024 * 1024  # 2MB
+MAX_LOG_SIZE = 5 * 1024 * 1024    # This is still needed for RotatingFileHandler if a single run exceeds 5MB
+
+log_file_path = os.path.join(LOGS_DIR, LOG_FILE)
+
+def cleanup_logs():
+    """Removes oldest log files if the total folder size exceeds MAX_FOLDER_SIZE."""
+    if not os.path.exists(LOGS_DIR):
+        return
+
+    files = [os.path.join(LOGS_DIR, f) for f in os.listdir(LOGS_DIR) if f.endswith(".log")]
+    files.sort(key=os.path.getmtime)  # Sort by modification time (oldest first)
+
+    total_size = sum(os.path.getsize(f) for f in files)
+
+    while total_size > MAX_FOLDER_SIZE and files:
+        oldest_file = files.pop(0)
+        file_size = os.path.getsize(oldest_file)
+        try:
+            os.remove(oldest_file)
+            total_size -= file_size
+            logging.info(f"Deleted old log file: {oldest_file}")
+        except Exception as e:
+            logging.error(f"Error deleting old log file {oldest_file}: {e}")
+            break
+
+def configure_logger():
+    # Ensure logs directory exists
+    os.makedirs(LOGS_DIR, exist_ok=True)
+    
+    # Run cleanup before creating new log
+    cleanup_logs()
+
+    logger = logging.getLogger()
+    logger.setLevel(logging.DEBUG)
+
+    formatter = logging.Formatter("[ %(asctime)s ] %(name)s - %(levelname)s - %(message)s")
+
+    # File Handler
+    file_handler = RotatingFileHandler(log_file_path, maxBytes=MAX_LOG_SIZE, backupCount=3)
+    file_handler.setFormatter(formatter)
+    file_handler.setLevel(logging.DEBUG)
+
+    # Console Handler
+    console_handler = logging.StreamHandler()
+    console_handler.setFormatter(formatter)
+    console_handler.setLevel(logging.INFO)
+
+    # Avoid duplicate handlers if the logger is re-initialized
+    logger.handlers.clear()
+
+    logger.addHandler(file_handler)
+    logger.addHandler(console_handler)
+
+# Automatically configure on import
+configure_logger()
+logging.info(f"Logger initialized. Logging to {log_file_path}")

main.py

@@ -0,0 +1,25 @@
+from dotenv import load_dotenv
+load_dotenv()   # ← must be FIRST so os.getenv works in all imported modules
+
+import uvicorn as uv
+from api.main import app
+from logger import *
+from fastapi.staticfiles import StaticFiles
+from src.MultiRag.constants import DATA_FOLDER_PATH, DB_FOLDER_PATH
+import os
+app.mount("/static", StaticFiles(directory="static"), name="static")
+os.makedirs("images", exist_ok=True)
+app.mount("/blog/images", StaticFiles(directory="images"), name="blog_images")
+# app.mount("/images", StaticFiles(directory="images"), name="blog_images")
+
+os.makedirs(DATA_FOLDER_PATH, exist_ok=True)
+os.makedirs(DB_FOLDER_PATH, exist_ok=True)
+
+if __name__ == "__main__":
+    uv.run(
+        "main:app",
+        host="0.0.0.0",
+        port=7860,
+        reload=False,
+        reload_excludes=["db/*", "data/*", "logs/*", "vector_db/*", ".venv/*"],
+    )

notebook/README.md

@@ -0,0 +1,3 @@
+# Notebooks
+
+Here goes the notebooks used for research and development.

notebook/image_placeHolder.ipynb

@@ -0,0 +1,376 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "0b9ffe5f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pydantic import BaseModel,Field\n",
+    "from typing import Literal,List\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "cd7bb64d",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from dotenv import load_dotenv\n",
+    "load_dotenv()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "dd8207ef",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class ImageSpec(BaseModel):\n",
+    "    placeholder:str=Field(...,description=\"e.g. [[IMAGE_1]]\")\n",
+    "    filename:str=Field(...,description=\"Save under images/, e.g. qkv_flow.png\")\n",
+    "    prompt:str=Field(...,description=\"Prompt to send to the image model\")\n",
+    "    size:Literal[\"1024x1024\",\"1024x1536\",\"1536x1024\"]=\"1025x1024\"\n",
+    "    quality: Literal[\"low\", \"medium\", \"high\"] = \"medium\"\n",
+    "\n",
+    "\n",
+    "class GlobalImagePlan(BaseModel):\n",
+    "    md_with_placeholders:str\n",
+    "    images:List[ImageSpec]=Field(default_factory=list)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "63f25031",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain_aws import ChatBedrockConverse\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "255a2613",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "LLM_MODEL_ID = \"us.meta.llama3-3-70b-instruct-v1:0\"\n",
+    "LLM_REGION = \"us-east-1\"\n",
+    "llm = ChatBedrockConverse(\n",
+    "    model_id=LLM_MODEL_ID,\n",
+    "    region_name=LLM_REGION\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "849c528a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "placehonder=\"\"\"You are an expert technical blog image planning assistant.\n",
+    "\n",
+    "Your job is to analyze a Markdown blog post and generate a structured image plan.\n",
+    "\n",
+    "You MUST return output strictly matching the Pydantic model `GlobalImagePlan`.\n",
+    "\n",
+    "-----------------------------------------\n",
+    "YOUR TASK\n",
+    "-----------------------------------------\n",
+    "\n",
+    "You will receive a Markdown blog as input.\n",
+    "\n",
+    "You must:\n",
+    "\n",
+    "1. Keep the Markdown EXACTLY the same.\n",
+    "2. DO NOT rewrite, summarize, improve, or modify any text.\n",
+    "3. DO NOT remove or change any formatting.\n",
+    "4. Only insert image placeholders where images would improve clarity.\n",
+    "\n",
+    "-----------------------------------------\n",
+    "WHERE TO INSERT IMAGES\n",
+    "-----------------------------------------\n",
+    "\n",
+    "Insert placeholders only:\n",
+    "- After major section headings (## or ###)\n",
+    "- After complex explanations\n",
+    "- After architecture descriptions\n",
+    "- After workflows\n",
+    "- After comparisons\n",
+    "- Where diagrams would help understanding\n",
+    "- Where visual examples would add clarity\n",
+    "\n",
+    "DO NOT:\n",
+    "- Add images randomly\n",
+    "- Add too many images\n",
+    "- Break code blocks\n",
+    "- Insert placeholders inside code blocks\n",
+    "- Modify existing content\n",
+    "\n",
+    "-----------------------------------------\n",
+    "PLACEHOLDER FORMAT\n",
+    "-----------------------------------------\n",
+    "\n",
+    "Use this exact format:\n",
+    "\n",
+    "[[IMAGE_1]]\n",
+    "[[IMAGE_2]]\n",
+    "[[IMAGE_3]]\n",
+    "\n",
+    "Number them sequentially.\n",
+    "\n",
+    "-----------------------------------------\n",
+    "IMAGE SPEC RULES\n",
+    "-----------------------------------------\n",
+    "\n",
+    "For each placeholder generate an ImageSpec with:\n",
+    "\n",
+    "- placeholder: exact placeholder string (e.g. [[IMAGE_1]])\n",
+    "- filename: save under images/ directory (example: images/attention_flow.png)\n",
+    "- prompt: highly detailed image generation prompt describing what the image should show\n",
+    "- size: choose one of:\n",
+    "    - 1024x1024 (for square diagrams)\n",
+    "    - 1536x1024 (for wide architecture diagrams)\n",
+    "    - 1024x1536 (for vertical infographics)\n",
+    "- quality: \"medium\" unless diagram is complex → use \"high\"\n",
+    "\n",
+    "The prompt must:\n",
+    "- Be descriptive\n",
+    "- Mention diagram style\n",
+    "- Mention labels\n",
+    "- Mention arrows and flow\n",
+    "- Mention clean white background\n",
+    "- Mention professional technical illustration style\n",
+    "\n",
+    "-----------------------------------------\n",
+    "IMPORTANT OUTPUT RULES\n",
+    "-----------------------------------------\n",
+    "\n",
+    "You MUST return ONLY a valid GlobalImagePlan JSON object.\n",
+    "\n",
+    "Do NOT include:\n",
+    "- Explanations\n",
+    "- Extra text\n",
+    "- Markdown fences\n",
+    "- Comments\n",
+    "- Any text before or after the JSON\n",
+    "\n",
+    "-----------------------------------------\n",
+    "OUTPUT FORMAT\n",
+    "-----------------------------------------\n",
+    "\n",
+    "{\n",
+    "  \"md_with_placeholders\": \"...full markdown with inserted placeholders...\",\n",
+    "  \"images\": [\n",
+    "    {\n",
+    "      \"placeholder\": \"[[IMAGE_1]]\",\n",
+    "      \"filename\": \"images/example.png\",\n",
+    "      \"prompt\": \"Detailed image generation prompt...\",\n",
+    "      \"size\": \"1536x1024\",\n",
+    "      \"quality\": \"medium\"\n",
+    "    }\n",
+    "  ]\n",
+    "}\"\"\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "332e03d8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.messages import SystemMessage,HumanMessage"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "1a7a4167",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "markdown=\"\"\"\n",
+    "# State of Multimodal LLMs in 2026\n",
+    "\n",
+    "## Introduction to Multimodal LLMs\n",
+    "Recent developments in multimodal LLMs have shown significant progress, with models now capable of processing and generating multiple forms of data, such as text, images, and audio [Not found in provided sources]. \n",
+    "* Multimodal LLMs have been applied to various tasks, including visual question answering, image captioning, and text-to-image synthesis.\n",
+    "* The impact of multimodal LLMs can be seen in industries like healthcare, education, and entertainment, where they are used for applications such as medical image analysis, interactive learning systems, and content creation [Not found in provided sources].\n",
+    "* Despite the advancements, key challenges in multimodal LLM research remain, including the need for large-scale datasets, improved model architectures, and better evaluation metrics [Not found in provided sources].\n",
+    "\n",
+    "## Recent Advances in Multimodal LLMs\n",
+    "Recent breakthroughs in multimodal LLM architecture have led to significant improvements in the field. \n",
+    "* Multimodal transformers, which combine visual and textual features, have shown promising results in tasks such as visual question answering and image-text retrieval [Not found in provided sources].\n",
+    "* The use of multimodal attention mechanisms has also been explored, allowing models to focus on specific parts of the input data [Not found in provided sources].\n",
+    "\n",
+    "Multimodal LLMs play a crucial role in both computer vision and natural language processing. \n",
+    "They can be used to analyze and understand visual data, such as images and videos, and generate text-based descriptions or summaries.\n",
+    "In natural language processing, multimodal LLMs can be used to improve language understanding and generation tasks, such as machine translation and text summarization.\n",
+    "\n",
+    "The potential applications of multimodal LLMs in healthcare are vast. \n",
+    "They can be used to analyze medical images, such as X-rays and MRIs, and generate text-based diagnoses or recommendations.\n",
+    "Additionally, multimodal LLMs can be used to develop personalized treatment plans and improve patient outcomes [Not found in provided sources].\n",
+    "Overall, the latest advancements in multimodal LLMs have the potential to revolutionize various fields, including healthcare, and improve the way we interact with and understand visual and textual data.\n",
+    "\n",
+    "## Challenges and Limitations\n",
+    "The development of multimodal LLMs has made significant progress, but there are still several challenges and limitations that need to be addressed. \n",
+    "* The limitations of current multimodal LLM models include their inability to fully understand the nuances of human communication, such as sarcasm, idioms, and figurative language [Not found in provided sources].\n",
+    "* Training and deploying multimodal LLMs pose significant challenges, including the need for large amounts of diverse and high-quality training data, as well as the requirement for significant computational resources [Not found in provided sources].\n",
+    "* Further research is needed to improve the performance and robustness of multimodal LLMs, particularly in areas such as common sense reasoning, emotional intelligence, and adaptability to new contexts and domains [Not found in provided sources]. \n",
+    "Overall, addressing these challenges and limitations will be crucial to unlocking the full potential of multimodal LLMs and achieving more effective and engaging human-computer interactions.\n",
+    "\n",
+    "## Future Directions\n",
+    "The future of multimodal LLMs holds great promise, with potential applications in areas such as [virtual assistants](Not found in provided sources) and [human-computer interaction](Not found in provided sources). \n",
+    "* Multimodal LLMs may be used to improve accessibility and user experience in various domains.\n",
+    "* The role of multimodal LLMs in shaping the future of AI is significant, as they can enable more natural and intuitive interactions between humans and machines.\n",
+    "* Continued research in multimodal LLMs is crucial to overcome current limitations and unlock their full potential, driving innovation and progress in the field of AI [Not found in provided sources].\n",
+    "\n",
+    "\"\"\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "796739f7",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "GlobalImagePlan(md_with_placeholders='# State of Multimodal LLMs in 2026\\n## Introduction to Multimodal LLMs\\nRecent developments in multimodal LLMs have shown significant progress, with models now capable of processing and generating multiple forms of data, such as text, images, and audio [Not found in provided sources]. \\n* Multimodal LLMs have been applied to various tasks, including visual question answering, image captioning, and text-to-image synthesis.\\n* The impact of multimodal LLMs can be seen in industries like healthcare, education, and entertainment, where they are used for applications such as medical image analysis, interactive learning systems, and content creation [Not found in provided sources].\\n* Despite the advancements, key challenges in multimodal LLM research remain, including the need for large-scale datasets, improved model architectures, and better evaluation metrics [Not found in provided sources].\\n[[IMAGE_1]]\\n## Recent Advances in Multimodal LLMs\\nRecent breakthroughs in multimodal LLM architecture have led to significant improvements in the field. \\n* Multimodal transformers, which combine visual and textual features, have shown promising results in tasks such as visual question answering and image-text retrieval [Not found in provided sources].\\n* The use of multimodal attention mechanisms has also been explored, allowing models to focus on specific parts of the input data [Not found in provided sources].\\n[[IMAGE_2]]\\nMultimodal LLMs play a crucial role in both computer vision and natural language processing. \\nThey can be used to analyze and understand visual data, such as images and videos, and generate text-based descriptions or summaries.\\nIn natural language processing, multimodal LLMs can be used to improve language understanding and generation tasks, such as machine translation and text summarization.\\n[[IMAGE_3]]\\nThe potential applications of multimodal LLMs in healthcare are vast. \\nThey can be used to analyze medical images, such as X-rays and MRIs, and generate text-based diagnoses or recommendations.\\nAdditionally, multimodal LLMs can be used to develop personalized treatment plans and improve patient outcomes [Not found in provided sources].\\nOverall, the latest advancements in multimodal LLMs have the potential to revolutionize various fields, including healthcare, and improve the way we interact with and understand visual and textual data.\\n[[IMAGE_4]]\\n## Challenges and Limitations\\nThe development of multimodal LLMs has made significant progress, but there are still several challenges and limitations that need to be addressed. \\n* The limitations of current multimodal LLM models include their inability to fully understand the nuances of human communication, such as sarcasm, idioms, and figurative language [Not found in provided sources].\\n* Training and deploying multimodal LLMs pose significant challenges, including the need for large amounts of diverse and high-quality training data, as well as the requirement for significant computational resources [Not found in provided sources].\\n* Further research is needed to improve the performance and robustness of multimodal LLMs, particularly in areas such as common sense reasoning, emotional intelligence, and adaptability to new contexts and domains [Not found in provided sources]. \\nOverall, addressing these challenges and limitations will be crucial to unlocking the full potential of multimodal LLMs and achieving more effective and engaging human-computer interactions.\\n[[IMAGE_5]]\\n## Future Directions\\nThe future of multimodal LLMs holds great promise, with potential applications in areas such as [virtual assistants](Not found in provided sources) and [human-computer interaction](Not found in provided sources). \\n* Multimodal LLMs may be used to improve accessibility and user experience in various domains.\\n* The role of multimodal LLMs in shaping the future of AI is significant, as they can enable more natural and intuitive interactions between humans and machines.\\n* Continued research in multimodal LLMs is crucial to overcome current limitations and unlock their full potential, driving innovation and progress in the field of AI [Not found in provided sources].\\n[[IMAGE_6]]', images=[ImageSpec(placeholder='[[IMAGE_1]]', filename='images/multimodal_llm_architecture.png', prompt='A diagram showing the architecture of a multimodal LLM, with visual and textual features combined, and labels and arrows indicating the flow of data, on a clean white background, in a professional technical illustration style', size='1536x1024', quality='medium'), ImageSpec(placeholder='[[IMAGE_2]]', filename='images/multimodal_transformers.png', prompt='An illustration of multimodal transformers, with visual and textual features combined, and labels and arrows indicating the flow of data, on a clean white background, in a professional technical illustration style', size='1024x1024', quality='medium'), ImageSpec(placeholder='[[IMAGE_3]]', filename='images/multimodal_llm_applications.png', prompt='A diagram showing the various applications of multimodal LLMs, including computer vision and natural language processing, with labels and arrows indicating the relationships between the different applications, on a clean white background, in a professional technical illustration style', size='1024x1536', quality='medium'), ImageSpec(placeholder='[[IMAGE_4]]', filename='images/multimodal_llm_healthcare.png', prompt='An illustration of the potential applications of multimodal LLMs in healthcare, including medical image analysis and personalized treatment plans, with labels and arrows indicating the flow of data, on a clean white background, in a professional technical illustration style', size='1536x1024', quality='medium'), ImageSpec(placeholder='[[IMAGE_5]]', filename='images/multimodal_llm_challenges.png', prompt='A diagram showing the challenges and limitations of multimodal LLMs, including the need for large-scale datasets and improved model architectures, with labels and arrows indicating the relationships between the different challenges, on a clean white background, in a professional technical illustration style', size='1024x1024', quality='medium'), ImageSpec(placeholder='[[IMAGE_6]]', filename='images/multimodal_llm_future.png', prompt='An illustration of the future directions of multimodal LLMs, including potential applications in virtual assistants and human-computer interaction, with labels and arrows indicating the flow of data, on a clean white background, in a professional technical illustration style', size='1024x1536', quality='medium')])"
+      ]
+     },
+     "execution_count": 18,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "output=llm.with_structured_output(GlobalImagePlan)\\\n",
+    ".invoke(\n",
+    "    [\n",
+    "        SystemMessage(content=placehonder),\n",
+    "        HumanMessage(content=markdown)\n",
+    "    ]\n",
+    ")\n",
+    "\n",
+    "output"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "id": "0e44ffd5",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'# State of Multimodal LLMs in 2026\\n## Introduction to Multimodal LLMs\\nRecent developments in multimodal LLMs have shown significant progress, with models now capable of processing and generating multiple forms of data, such as text, images, and audio [Not found in provided sources]. \\n* Multimodal LLMs have been applied to various tasks, including visual question answering, image captioning, and text-to-image synthesis.\\n* The impact of multimodal LLMs can be seen in industries like healthcare, education, and entertainment, where they are used for applications such as medical image analysis, interactive learning systems, and content creation [Not found in provided sources].\\n* Despite the advancements, key challenges in multimodal LLM research remain, including the need for large-scale datasets, improved model architectures, and better evaluation metrics [Not found in provided sources].\\n[[IMAGE_1]]\\n## Recent Advances in Multimodal LLMs\\nRecent breakthroughs in multimodal LLM architecture have led to significant improvements in the field. \\n* Multimodal transformers, which combine visual and textual features, have shown promising results in tasks such as visual question answering and image-text retrieval [Not found in provided sources].\\n* The use of multimodal attention mechanisms has also been explored, allowing models to focus on specific parts of the input data [Not found in provided sources].\\n[[IMAGE_2]]\\nMultimodal LLMs play a crucial role in both computer vision and natural language processing. \\nThey can be used to analyze and understand visual data, such as images and videos, and generate text-based descriptions or summaries.\\nIn natural language processing, multimodal LLMs can be used to improve language understanding and generation tasks, such as machine translation and text summarization.\\n[[IMAGE_3]]\\nThe potential applications of multimodal LLMs in healthcare are vast. \\nThey can be used to analyze medical images, such as X-rays and MRIs, and generate text-based diagnoses or recommendations.\\nAdditionally, multimodal LLMs can be used to develop personalized treatment plans and improve patient outcomes [Not found in provided sources].\\nOverall, the latest advancements in multimodal LLMs have the potential to revolutionize various fields, including healthcare, and improve the way we interact with and understand visual and textual data.\\n[[IMAGE_4]]\\n## Challenges and Limitations\\nThe development of multimodal LLMs has made significant progress, but there are still several challenges and limitations that need to be addressed. \\n* The limitations of current multimodal LLM models include their inability to fully understand the nuances of human communication, such as sarcasm, idioms, and figurative language [Not found in provided sources].\\n* Training and deploying multimodal LLMs pose significant challenges, including the need for large amounts of diverse and high-quality training data, as well as the requirement for significant computational resources [Not found in provided sources].\\n* Further research is needed to improve the performance and robustness of multimodal LLMs, particularly in areas such as common sense reasoning, emotional intelligence, and adaptability to new contexts and domains [Not found in provided sources]. \\nOverall, addressing these challenges and limitations will be crucial to unlocking the full potential of multimodal LLMs and achieving more effective and engaging human-computer interactions.\\n[[IMAGE_5]]\\n## Future Directions\\nThe future of multimodal LLMs holds great promise, with potential applications in areas such as [virtual assistants](Not found in provided sources) and [human-computer interaction](Not found in provided sources). \\n* Multimodal LLMs may be used to improve accessibility and user experience in various domains.\\n* The role of multimodal LLMs in shaping the future of AI is significant, as they can enable more natural and intuitive interactions between humans and machines.\\n* Continued research in multimodal LLMs is crucial to overcome current limitations and unlock their full potential, driving innovation and progress in the field of AI [Not found in provided sources].\\n[[IMAGE_6]]'"
+      ]
+     },
+     "execution_count": 20,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "output.md_with_placeholders"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "id": "00892f27",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[ImageSpec(placeholder='[[IMAGE_1]]', filename='images/multimodal_llm_architecture.png', prompt='A diagram showing the architecture of a multimodal LLM, with visual and textual features combined, and labels and arrows indicating the flow of data, on a clean white background, in a professional technical illustration style', size='1536x1024', quality='medium'),\n",
+       " ImageSpec(placeholder='[[IMAGE_2]]', filename='images/multimodal_transformers.png', prompt='An illustration of multimodal transformers, with visual and textual features combined, and labels and arrows indicating the flow of data, on a clean white background, in a professional technical illustration style', size='1024x1024', quality='medium'),\n",
+       " ImageSpec(placeholder='[[IMAGE_3]]', filename='images/multimodal_llm_applications.png', prompt='A diagram showing the various applications of multimodal LLMs, including computer vision and natural language processing, with labels and arrows indicating the relationships between the different applications, on a clean white background, in a professional technical illustration style', size='1024x1536', quality='medium'),\n",
+       " ImageSpec(placeholder='[[IMAGE_4]]', filename='images/multimodal_llm_healthcare.png', prompt='An illustration of the potential applications of multimodal LLMs in healthcare, including medical image analysis and personalized treatment plans, with labels and arrows indicating the flow of data, on a clean white background, in a professional technical illustration style', size='1536x1024', quality='medium'),\n",
+       " ImageSpec(placeholder='[[IMAGE_5]]', filename='images/multimodal_llm_challenges.png', prompt='A diagram showing the challenges and limitations of multimodal LLMs, including the need for large-scale datasets and improved model architectures, with labels and arrows indicating the relationships between the different challenges, on a clean white background, in a professional technical illustration style', size='1024x1024', quality='medium'),\n",
+       " ImageSpec(placeholder='[[IMAGE_6]]', filename='images/multimodal_llm_future.png', prompt='An illustration of the future directions of multimodal LLMs, including potential applications in virtual assistants and human-computer interaction, with labels and arrows indicating the flow of data, on a clean white background, in a professional technical illustration style', size='1024x1536', quality='medium')]"
+      ]
+     },
+     "execution_count": 21,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "output.images"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "id": "0b4e77e2",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'A diagram showing the architecture of a multimodal LLM, with visual and textual features combined, and labels and arrows indicating the flow of data, on a clean white background, in a professional technical illustration style'"
+      ]
+     },
+     "execution_count": 23,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "output.images[0].prompt"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8666fa58",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "bloggig-Agent (3.12.12)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebook/rag.ipynb

@@ -0,0 +1,379 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "9304576f",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import os\n",
+    "from dotenv import load_dotenv\n",
+    "load_dotenv()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "bbaf9dea",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain_aws import ChatBedrockConverse\n",
+    "LLM_MODEL_ID = \"us.meta.llama3-3-70b-instruct-v1:0\"\n",
+    "LLM_REGION = \"us-east-1\"\n",
+    "\n",
+    "import logging\n",
+    "llm = ChatBedrockConverse(\n",
+    "    model_id=LLM_MODEL_ID,\n",
+    "    region_name=LLM_REGION\n",
+    ")\n",
+    "logging.info(f\"LLM initialized with model_id={LLM_MODEL_ID}, region_name={LLM_REGION}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "751de1c5",
+   "metadata": {},
+   "source": [
+    "# Data Ingestion"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "6dba7a05",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain_community.document_loaders import TextLoader,DirectoryLoader\n",
+    "from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
+    "\n",
+    "from langchain_chroma import Chroma\n",
+    "\n",
+    "from langchain_ollama.embeddings import OllamaEmbeddings"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "68a36c6c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def load_documents(docs_path=\"docs\"):\n",
+    "    \"\"\"Load all text files from the docs directory\"\"\"\n",
+    "    print(f\"Loading documents from {docs_path}...\")\n",
+    "    \n",
+    "    # Check if docs directory exists\n",
+    "    if not os.path.exists(docs_path):\n",
+    "        raise FileNotFoundError(f\"The directory {docs_path} does not exist. Please create it and add your company files.\")\n",
+    "    \n",
+    "    # Load all .txt files from the docs directory\n",
+    "    loader = DirectoryLoader(\n",
+    "        path=docs_path,\n",
+    "        glob=\"*.txt\",\n",
+    "        loader_cls=TextLoader\n",
+    "    )\n",
+    "    \n",
+    "    documents = loader.load()\n",
+    "    \n",
+    "    if len(documents) == 0:\n",
+    "        raise FileNotFoundError(f\"No .txt files found in {docs_path}. Please add your company documents.\")\n",
+    "    \n",
+    "   \n",
+    "    for i, doc in enumerate(documents[:2]):  # Show first 2 documents\n",
+    "        print(f\"\\nDocument {i+1}:\")\n",
+    "        print(f\"  Source: {doc.metadata['source']}\")\n",
+    "        print(f\"  Content length: {len(doc.page_content)} characters\")\n",
+    "        print(f\"  Content preview: {doc.page_content[:100]}...\")\n",
+    "        print(f\"  metadata: {doc.metadata}\")\n",
+    "\n",
+    "    return documents"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d4b05f9c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def split_documents(documents, chunk_size=1000, chunk_overlap=0):\n",
+    "    \"\"\"Split documents into smaller chunks with overlap\"\"\"\n",
+    "    print(\"Splitting documents into chunks...\")\n",
+    "    \n",
+    "    text_splitter = RecursiveCharacterTextSplitter(\n",
+    "        chunk_size=chunk_size, \n",
+    "        chunk_overlap=chunk_overlap\n",
+    "    )\n",
+    "    \n",
+    "    chunks = text_splitter.split_documents(documents)\n",
+    "    \n",
+    "    if chunks:\n",
+    "    \n",
+    "        for i, chunk in enumerate(chunks[:5]):\n",
+    "            print(f\"\\n--- Chunk {i+1} ---\")\n",
+    "            print(f\"Source: {chunk.metadata['source']}\")\n",
+    "            print(f\"Length: {len(chunk.page_content)} characters\")\n",
+    "            print(f\"Content:\")\n",
+    "            print(chunk.page_content)\n",
+    "            print(\"-\" * 50)\n",
+    "        \n",
+    "        if len(chunks) > 5:\n",
+    "            print(f\"\\n... and {len(chunks) - 5} more chunks\")\n",
+    "    \n",
+    "    return chunks"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "e9d2c201",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def create_vector_store(chunks, persist_directory=\"db/chroma_db\"):\n",
+    "    \"\"\"Create and persist ChromaDB vector store\"\"\"\n",
+    "    print(\"Creating embeddings and storing in ChromaDB...\")\n",
+    "        \n",
+    "    embedding_model = OllamaEmbeddings(model=\"gemma2:2b\")\n",
+    "    \n",
+    "    # Create ChromaDB vector store\n",
+    "    print(\"--- Creating vector store ---\")\n",
+    "    vectorstore = Chroma.from_documents(\n",
+    "        documents=chunks,\n",
+    "        embedding=embedding_model,\n",
+    "        persist_directory=persist_directory, \n",
+    "        collection_metadata={\"hnsw:space\": \"cosine\"}\n",
+    "    )\n",
+    "    print(\"--- Finished creating vector store ---\")\n",
+    "    \n",
+    "    print(f\"Vector store created and saved to {persist_directory}\")\n",
+    "    return vectorstore"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "28dd89fe",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def main():\n",
+    "    \"\"\"Main ingestion pipeline\"\"\"\n",
+    "    print(\"=== RAG Document Ingestion Pipeline ===\\n\")\n",
+    "    \n",
+    "    # Define paths\n",
+    "    docs_path = \"data\"\n",
+    "    persistent_directory = \"db/chroma_db\"\n",
+    "    \n",
+    "    # Check if vector store already exists\n",
+    "    if os.path.exists(persistent_directory):\n",
+    "        print(\"✅ Vector store already exists. No need to re-process documents.\")\n",
+    "        \n",
+    "        embedding_model = OllamaEmbeddings(model=\"gemma2:2b\")\n",
+    "        vectorstore = Chroma(\n",
+    "            persist_directory=persistent_directory,\n",
+    "            embedding_function=embedding_model, \n",
+    "            collection_metadata={\"hnsw:space\": \"cosine\"}\n",
+    "        )\n",
+    "        print(f\"Loaded existing vector store with {vectorstore._collection.count()} documents\")\n",
+    "        return vectorstore\n",
+    "    \n",
+    "    print(\"Persistent directory does not exist. Initializing vector store...\\n\")\n",
+    "    \n",
+    "    # Step 1: Load documents\n",
+    "    documents = load_documents(docs_path)  \n",
+    "\n",
+    "    # Step 2: Split into chunks\n",
+    "    chunks = split_documents(documents)\n",
+    "    \n",
+    "    # # Step 3: Create vector store\n",
+    "    vectorstore = create_vector_store(chunks, persistent_directory)\n",
+    "    \n",
+    "    print(\"\\n✅ Ingestion complete! Your documents are now ready for RAG queries.\")\n",
+    "    return vectorstore"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "0bbabc0a",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "=== RAG Document Ingestion Pipeline ===\n",
+      "\n",
+      "Persistent directory does not exist. Initializing vector store...\n",
+      "\n",
+      "Loading documents from data...\n"
+     ]
+    },
+    {
+     "ename": "FileNotFoundError",
+     "evalue": "The directory data does not exist. Please create it and add your company files.",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
+      "\u001b[31mFileNotFoundError\u001b[39m                         Traceback (most recent call last)",
+      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[17]\u001b[39m\u001b[32m, line 1\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m1\u001b[39m vectordb=\u001b[43mmain\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m      2\u001b[39m vectordb\n",
+      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[16]\u001b[39m\u001b[32m, line 25\u001b[39m, in \u001b[36mmain\u001b[39m\u001b[34m()\u001b[39m\n\u001b[32m     22\u001b[39m \u001b[38;5;28mprint\u001b[39m(\u001b[33m\"\u001b[39m\u001b[33mPersistent directory does not exist. Initializing vector store...\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[33m\"\u001b[39m)\n\u001b[32m     24\u001b[39m \u001b[38;5;66;03m# Step 1: Load documents\u001b[39;00m\n\u001b[32m---> \u001b[39m\u001b[32m25\u001b[39m documents = \u001b[43mload_documents\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdocs_path\u001b[49m\u001b[43m)\u001b[49m  \n\u001b[32m     27\u001b[39m \u001b[38;5;66;03m# Step 2: Split into chunks\u001b[39;00m\n\u001b[32m     28\u001b[39m chunks = split_documents(documents)\n",
+      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[7]\u001b[39m\u001b[32m, line 7\u001b[39m, in \u001b[36mload_documents\u001b[39m\u001b[34m(docs_path)\u001b[39m\n\u001b[32m      5\u001b[39m \u001b[38;5;66;03m# Check if docs directory exists\u001b[39;00m\n\u001b[32m      6\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m os.path.exists(docs_path):\n\u001b[32m----> \u001b[39m\u001b[32m7\u001b[39m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mFileNotFoundError\u001b[39;00m(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mThe directory \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mdocs_path\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m does not exist. Please create it and add your company files.\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m      9\u001b[39m \u001b[38;5;66;03m# Load all .txt files from the docs directory\u001b[39;00m\n\u001b[32m     10\u001b[39m loader = DirectoryLoader(\n\u001b[32m     11\u001b[39m     path=docs_path,\n\u001b[32m     12\u001b[39m     glob=\u001b[33m\"\u001b[39m\u001b[33m*.txt\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m     13\u001b[39m     loader_cls=TextLoader\n\u001b[32m     14\u001b[39m )\n",
+      "\u001b[31mFileNotFoundError\u001b[39m: The directory data does not exist. Please create it and add your company files."
+     ]
+    }
+   ],
+   "source": [
+    "vectordb=main()\n",
+    "vectordb"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "cb14033a",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "NameError",
+     "evalue": "name 'embedding_model' is not defined",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
+      "\u001b[31mNameError\u001b[39m                                 Traceback (most recent call last)",
+      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[18]\u001b[39m\u001b[32m, line 11\u001b[39m\n\u001b[32m      4\u001b[39m load_dotenv()\n\u001b[32m      6\u001b[39m persistent_directory = \u001b[33m\"\u001b[39m\u001b[33mdb/chroma_db\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m      9\u001b[39m db = Chroma(\n\u001b[32m     10\u001b[39m     persist_directory=persistent_directory,\n\u001b[32m---> \u001b[39m\u001b[32m11\u001b[39m     embedding_function=\u001b[43membedding_model\u001b[49m,\n\u001b[32m     12\u001b[39m     collection_metadata={\u001b[33m\"\u001b[39m\u001b[33mhnsw:space\u001b[39m\u001b[33m\"\u001b[39m: \u001b[33m\"\u001b[39m\u001b[33mcosine\u001b[39m\u001b[33m\"\u001b[39m}  \n\u001b[32m     13\u001b[39m )\n\u001b[32m     15\u001b[39m \u001b[38;5;66;03m# Search for relevant documents\u001b[39;00m\n\u001b[32m     16\u001b[39m query = \u001b[33m\"\u001b[39m\u001b[33mHow much did Microsoft pay to acquire GitHub?\u001b[39m\u001b[33m\"\u001b[39m\n",
+      "\u001b[31mNameError\u001b[39m: name 'embedding_model' is not defined"
+     ]
+    }
+   ],
+   "source": [
+    "from langchain_chroma import Chroma\n",
+    "from dotenv import load_dotenv\n",
+    "\n",
+    "load_dotenv()\n",
+    "\n",
+    "persistent_directory = \"db/chroma_db\"\n",
+    "\n",
+    "\n",
+    "db = Chroma(\n",
+    "    persist_directory=persistent_directory,\n",
+    "    embedding_function=embedding_model,\n",
+    "    collection_metadata={\"hnsw:space\": \"cosine\"}  \n",
+    ")\n",
+    "\n",
+    "# Search for relevant documents\n",
+    "query = \"How much did Microsoft pay to acquire GitHub?\"\n",
+    "\n",
+    "retriever = db.as_retriever(search_kwargs={\"k\": 5})\n",
+    "\n",
+    "# retriever = db.as_retriever(\n",
+    "#     search_type=\"similarity_score_threshold\",\n",
+    "#     search_kwargs={\n",
+    "#         \"k\": 5,\n",
+    "#         \"score_threshold\": 0.3  # Only return chunks with cosine similarity ≥ 0.3\n",
+    "#     }\n",
+    "# )\n",
+    "\n",
+    "relevant_docs = retriever.invoke(query)\n",
+    "\n",
+    "print(f\"User Query: {query}\")\n",
+    "# Display results\n",
+    "print(\"--- Context ---\")\n",
+    "for i, doc in enumerate(relevant_docs, 1):\n",
+    "    print(f\"Document {i}:\\n{doc.page_content}\\n\")\n",
+    "\n",
+    "\n",
+    "# Synthetic Questions: \n",
+    "\n",
+    "# 1. \"What was NVIDIA's first graphics accelerator called?\"\n",
+    "# 2. \"Which company did NVIDIA acquire to enter the mobile processor market?\"\n",
+    "# 3. \"What was Microsoft's first hardware product release?\"\n",
+    "# 4. \"How much did Microsoft pay to acquire GitHub?\"\n",
+    "# 5. \"In what year did Tesla begin production of the Roadster?\"\n",
+    "# 6. \"Who succeeded Ze'ev Drori as CEO in October 2008?\"\n",
+    "# 7. \"What was the name of the autonomous spaceport drone ship that achieved the first successful sea landing?\"\n",
+    "# 8. \"What was the original name of Microsoft before it became Microsoft?\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "9b97a0c6",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "ImportError",
+     "evalue": "cannot import name 'TransfoXLModel' from 'transformers' (c:\\Users\\vansh\\Projects\\AIAgents\\.venv\\Lib\\site-packages\\transformers\\__init__.py)",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
+      "\u001b[31mImportError\u001b[39m                               Traceback (most recent call last)",
+      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[2]\u001b[39m\u001b[32m, line 1\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m1\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01msummarizer\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Summarizer\n\u001b[32m      3\u001b[39m document = \u001b[33m\"\"\"\u001b[39m\n\u001b[32m      4\u001b[39m \u001b[33mDeep learning is a subset of machine learning that uses neural networks.\u001b[39m\n\u001b[32m      5\u001b[39m \u001b[33mThese networks consist of multiple layers that can learn complex patterns.\u001b[39m\n\u001b[32m      6\u001b[39m \u001b[33mConvolutional neural networks are widely used in image recognition tasks.\u001b[39m\n\u001b[32m      7\u001b[39m \u001b[33m\"\"\"\u001b[39m\n\u001b[32m      9\u001b[39m model = Summarizer()\n",
+      "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\vansh\\Projects\\AIAgents\\.venv\\Lib\\site-packages\\summarizer\\__init__.py:1\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m1\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01msummarizer\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mbert\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Summarizer, TransformerSummarizer\n\u001b[32m      3\u001b[39m __all__ = [\u001b[33m\"\u001b[39m\u001b[33mSummarizer\u001b[39m\u001b[33m\"\u001b[39m, \u001b[33m\"\u001b[39m\u001b[33mTransformerSummarizer\u001b[39m\u001b[33m\"\u001b[39m]\n",
+      "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\vansh\\Projects\\AIAgents\\.venv\\Lib\\site-packages\\summarizer\\bert.py:4\u001b[39m\n\u001b[32m      1\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mfunctools\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m partial\n\u001b[32m      2\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mtyping\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m List, Optional, Union\n\u001b[32m----> \u001b[39m\u001b[32m4\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mtransformers\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m (AlbertModel, AlbertTokenizer, BartModel, BigBirdModel, BigBirdTokenizer,\n\u001b[32m      5\u001b[39m                           BartTokenizer, BertModel, BertTokenizer,\n\u001b[32m      6\u001b[39m                           CamembertModel, CamembertTokenizer, CTRLModel,\n\u001b[32m      7\u001b[39m                           CTRLTokenizer, DistilBertModel, DistilBertTokenizer,\n\u001b[32m      8\u001b[39m                           GPT2Model, GPT2Tokenizer, LongformerModel,\n\u001b[32m      9\u001b[39m                           LongformerTokenizer, OpenAIGPTModel,\n\u001b[32m     10\u001b[39m                           OpenAIGPTTokenizer, PreTrainedModel,\n\u001b[32m     11\u001b[39m                           PreTrainedTokenizer, RobertaModel, RobertaTokenizer,\n\u001b[32m     12\u001b[39m                           TransfoXLModel, TransfoXLTokenizer, XLMModel,\n\u001b[32m     13\u001b[39m                           XLMTokenizer, XLNetModel, XLNetTokenizer)\n\u001b[32m     15\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01msummarizer\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01msummary_processor\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m SummaryProcessor\n\u001b[32m     16\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01msummarizer\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mtext_processors\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01msentence_handler\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m SentenceHandler\n",
+      "\u001b[31mImportError\u001b[39m: cannot import name 'TransfoXLModel' from 'transformers' (c:\\Users\\vansh\\Projects\\AIAgents\\.venv\\Lib\\site-packages\\transformers\\__init__.py)"
+     ]
+    }
+   ],
+   "source": [
+    "from summarizer import Summarizer\n",
+    "\n",
+    "document = \"\"\"\n",
+    "Deep learning is a subset of machine learning that uses neural networks.\n",
+    "These networks consist of multiple layers that can learn complex patterns.\n",
+    "Convolutional neural networks are widely used in image recognition tasks.\n",
+    "\"\"\"\n",
+    "\n",
+    "model = Summarizer()\n",
+    "\n",
+    "summary = model(document, ratio=0.3)\n",
+    "\n",
+    "print(summary)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b538f506",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "AIAgents (3.12.12)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebook/understanding_self_attention.md

@@ -0,0 +1,120 @@
+# Understanding Self Attention
+
+### Introduction to Self Attention
+Self-attention, also known as intra-attention, is a mechanism used in deep learning models to allow the model to attend to different parts of the input data and weigh their importance. It's a key component of the Transformer architecture, introduced in 2017, which revolutionized the field of natural language processing (NLP). Self-attention enables the model to capture long-range dependencies and contextual relationships in the input data, making it particularly useful for sequence-to-sequence tasks such as machine translation, text summarization, and chatbots. The importance of self-attention lies in its ability to handle variable-length input sequences, parallelize computation, and improve model performance by focusing on the most relevant parts of the input data. In this blog, we'll delve deeper into the concept of self-attention, its types, and its applications in deep learning.
+
+### How Self Attention Works
+Self-attention is a mechanism that allows a model to attend to different parts of the input sequence simultaneously and weigh their importance. It's a key component of the Transformer architecture, introduced in the paper "Attention is All You Need" by Vaswani et al.
+
+The self-attention mechanism takes in a set of input vectors, typically the output of an encoder or a previous layer, and computes a weighted sum of these vectors based on their similarity. The weights are learned during training and reflect the relative importance of each input vector.
+
+The mathematical formulation of self-attention can be broken down into three main steps:
+
+1. **Query, Key, and Value Vectors**: The input vectors are first transformed into three different vectors: Query (Q), Key (K), and Value (V). These vectors are obtained by applying linear transformations to the input vectors.
+2. **Attention Scores**: The attention scores are computed by taking the dot product of the Query and Key vectors and applying a scaling factor. The attention scores represent the similarity between the input vectors.
+3. **Weighted Sum**: The attention scores are then used to compute a weighted sum of the Value vectors. The weighted sum is the final output of the self-attention mechanism.
+
+The self-attention mechanism can be formulated mathematically as follows:
+
+`Attention(Q, K, V) = softmax(Q * K^T / sqrt(d)) * V`
+
+where `Q`, `K`, and `V` are the Query, Key, and Value vectors, respectively, `d` is the dimensionality of the input vectors, and `softmax` is the softmax activation function.
+
+The self-attention mechanism has several benefits, including:
+
+* **Parallelization**: Self-attention can be parallelized more easily than recurrent neural networks (RNNs), making it more efficient for long-range dependencies.
+* **Flexibility**: Self-attention can handle variable-length input sequences and can be used for both encoding and decoding tasks.
+* **Interpretability**: The attention scores can provide insights into which parts of the input sequence are most relevant for a particular task.
+
+### Types of Self Attention
+There are several types of self-attention mechanisms that have been proposed in the literature, each with its own strengths and weaknesses. The two main categories of self-attention are local self-attention and global self-attention.
+
+#### Local Self Attention
+Local self-attention, also known as local attention or window-based attention, focuses on a fixed-size window of the input sequence. This type of attention is useful when the relationships between nearby elements in the sequence are more important than the relationships between distant elements. Local self-attention is often used in tasks such as language modeling and machine translation.
+
+#### Global Self Attention
+Global self-attention, on the other hand, considers the entire input sequence when computing the attention weights. This type of attention is useful when the relationships between all elements in the sequence are important, regardless of their distance. Global self-attention is often used in tasks such as question answering and text classification.
+
+#### Other Types of Self Attention
+In addition to local and global self-attention, there are other variants of self-attention that have been proposed, including:
+* **Hierarchical self-attention**: This type of attention uses a hierarchical representation of the input sequence, where the attention weights are computed at multiple levels of granularity.
+* **Graph-based self-attention**: This type of attention is used for graph-structured data, where the attention weights are computed based on the graph structure.
+* **Multi-head self-attention**: This type of attention uses multiple attention heads to capture different types of relationships between the elements in the input sequence.
+
+### Applications of Self Attention
+Self-attention has numerous applications across various fields, including natural language processing, computer vision, and more. Some of the key applications of self-attention are:
+* **Natural Language Processing (NLP)**: Self-attention is widely used in NLP tasks such as language translation, question answering, and text summarization. It helps in understanding the context and relationships between different words in a sentence.
+* **Computer Vision**: Self-attention is used in computer vision tasks such as image classification, object detection, and image generation. It helps in understanding the relationships between different parts of an image.
+* **Speech Recognition**: Self-attention is used in speech recognition tasks to improve the accuracy of speech-to-text models.
+* **Recommendation Systems**: Self-attention is used in recommendation systems to understand the relationships between different items and recommend relevant items to users.
+* **Time Series Forecasting**: Self-attention is used in time series forecasting to understand the relationships between different time steps and predict future values.
+The use of self-attention has led to state-of-the-art results in many of these applications, and its potential continues to be explored in other fields.
+
+### Implementing Self Attention
+Implementing self-attention in a deep learning model involves several key steps. Here's a step-by-step guide to help you get started:
+#### Step 1: Define the Self-Attention Mechanism
+The self-attention mechanism is based on the Query-Key-Value (QKV) framework. You need to define the QKV matrices and calculate the attention weights using the following formula: 
+$$Attention(Q, K, V) = softmax(\frac{Q \cdot K^T}{\sqrt{d_k}}) \cdot V$$
+where $d_k$ is the dimensionality of the key vector.
+
+#### Step 2: Choose the Attention Type
+There are two main types of self-attention: scaled dot-product attention and multi-head attention. Scaled dot-product attention is a basic form of self-attention, while multi-head attention allows the model to jointly attend to information from different representation subspaces.
+
+#### Step 3: Implement the Self-Attention Layer
+You can implement the self-attention layer using popular deep learning frameworks such as PyTorch or TensorFlow. The self-attention layer takes in the input sequence and outputs a weighted sum of the input elements.
+
+#### Step 4: Integrate the Self-Attention Layer into the Model
+Once you have implemented the self-attention layer, you can integrate it into your deep learning model. This typically involves adding the self-attention layer to the model architecture and adjusting the model's parameters accordingly.
+
+#### Step 5: Train the Model
+After integrating the self-attention layer, you need to train the model using a suitable optimizer and loss function. The self-attention mechanism can be trained end-to-end with the rest of the model.
+
+#### Example Code
+Here's an example code snippet in PyTorch that demonstrates how to implement a basic self-attention layer:
+```python
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+class SelfAttention(nn.Module):
+    def __init__(self, embed_dim, num_heads):
+        super(SelfAttention, self).__init__()
+        self.embed_dim = embed_dim
+        self.num_heads = num_heads
+        self.query_linear = nn.Linear(embed_dim, embed_dim)
+        self.key_linear = nn.Linear(embed_dim, embed_dim)
+        self.value_linear = nn.Linear(embed_dim, embed_dim)
+        self.dropout = nn.Dropout(0.1)
+
+    def forward(self, x):
+        # Calculate Q, K, V
+        Q = self.query_linear(x)
+        K = self.key_linear(x)
+        V = self.value_linear(x)
+
+        # Calculate attention weights
+        attention_weights = torch.matmul(Q, K.T) / math.sqrt(self.embed_dim)
+        attention_weights = F.softmax(attention_weights, dim=-1)
+
+        # Calculate output
+        output = torch.matmul(attention_weights, V)
+        output = self.dropout(output)
+        return output
+```
+Note that this is a simplified example, and you may need to modify the code to suit your specific use case.
+
+### Advantages and Limitations of Self Attention
+The self-attention mechanism has several advantages that make it a powerful tool in deep learning models. Some of the key benefits include:
+* **Parallelization**: Self-attention allows for parallelization of sequential data, making it much faster than traditional recurrent neural networks (RNNs) for long sequences.
+* **Flexibility**: Self-attention can handle variable-length input sequences and can be used for both short-term and long-term dependencies.
+* **Interpretability**: The attention weights provide a way to visualize and understand which parts of the input sequence are most relevant for a particular task.
+
+However, self-attention also has some limitations:
+* **Computational Cost**: Self-attention has a high computational cost, especially for long sequences, due to the need to compute attention weights for every pair of elements.
+* **Memory Requirements**: Self-attention requires a significant amount of memory to store the attention weights and the input sequence.
+* **Difficulty in Handling Local Dependencies**: Self-attention can struggle to capture local dependencies, such as those found in images or text with strong spatial relationships.
+
+Despite these limitations, self-attention has the potential for future directions, including:
+* **Improving Efficiency**: Researchers are exploring ways to improve the efficiency of self-attention, such as using sparse attention or hierarchical attention.
+* **Combining with Other Mechanisms**: Self-attention can be combined with other mechanisms, such as convolutional neural networks (CNNs) or RNNs, to create more powerful models.
+* **Applying to New Domains**: Self-attention can be applied to new domains, such as computer vision or speech recognition, to improve performance and efficiency.

notebook/understanding_self_attention_in_deep_learning.md

@@ -0,0 +1,105 @@
+# Understanding Self Attention in Deep Learning
+
+## Introduction to Self Attention
+Self attention is a fundamental concept in deep learning, enabling models to weigh the importance of different input elements relative to each other. It plays a crucial role in deep learning models, particularly in natural language processing and computer vision tasks, by allowing the model to focus on specific parts of the input data.
+
+The traditional attention mechanisms have a limitation - they rely on a fixed-length context, which can be restrictive for sequences with varying lengths. This fixed-length context can lead to information loss or inefficient processing, especially when dealing with long sequences.
+
+To address this, self attention mechanisms are used, which can be implemented using the following minimal code snippet:
+```python
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+class SelfAttention(nn.Module):
+    def __init__(self, embed_dim):
+        super(SelfAttention, self).__init__()
+        self.query_linear = nn.Linear(embed_dim, embed_dim)
+        self.key_linear = nn.Linear(embed_dim, embed_dim)
+        self.value_linear = nn.Linear(embed_dim, embed_dim)
+
+    def forward(self, x):
+        query = self.query_linear(x)
+        key = self.key_linear(x)
+        value = self.value_linear(x)
+        attention_scores = torch.matmul(query, key.T) / math.sqrt(key.size(-1))
+        attention_weights = F.softmax(attention_scores, dim=-1)
+        output = torch.matmul(attention_weights, value)
+        return output
+```
+This code snippet demonstrates a basic self attention implementation, highlighting its importance in deep learning models.
+
+## Implementing Self Attention
+To implement self attention, it's essential to understand the underlying mathematical formulation. The self attention mechanism is based on the concept of attention, which allows the model to focus on specific parts of the input data.
+
+* The mathematical formulation of self attention involves computing the attention weights based on the query, key, and value vectors. This is typically done using the following equation: `Attention(Q, K, V) = softmax(Q * K^T / sqrt(d)) * V`, where `Q`, `K`, and `V` are the query, key, and value vectors, respectively, and `d` is the dimensionality of the input data.
+
+The query-key-value attention mechanism is a core component of self attention. In this mechanism, the query vector represents the context in which the attention is being applied, the key vector represents the input data, and the value vector represents the importance of each input element.
+
+```python
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+class SelfAttention(nn.Module):
+    def __init__(self, embed_dim):
+        super(SelfAttention, self).__init__()
+        self.query_linear = nn.Linear(embed_dim, embed_dim)
+        self.key_linear = nn.Linear(embed_dim, embed_dim)
+        self.value_linear = nn.Linear(embed_dim, embed_dim)
+
+    def forward(self, x):
+        Q = self.query_linear(x)
+        K = self.key_linear(x)
+        V = self.value_linear(x)
+        attention_weights = F.softmax(torch.matmul(Q, K.T) / math.sqrt(x.size(-1)), dim=-1)
+        return torch.matmul(attention_weights, V)
+```
+This code example demonstrates how to implement self attention in PyTorch, a popular deep learning framework. By using this implementation, developers can easily integrate self attention into their own models.
+
+## Applications of Self Attention
+Self attention has numerous applications in various fields. 
+In natural language processing tasks, self attention is used to weigh the importance of different words in a sentence, allowing models to capture long-range dependencies and context.
+
+* Example in computer vision: self attention can be applied to image classification models to focus on specific regions of the image, as shown in this PyTorch code snippet:
+```python
+import torch
+import torch.nn as nn
+
+class SelfAttention(nn.Module):
+    def __init__(self, embed_dim):
+        super(SelfAttention, self).__init__()
+        self.query_linear = nn.Linear(embed_dim, embed_dim)
+        self.key_linear = nn.Linear(embed_dim, embed_dim)
+        self.value_linear = nn.Linear(embed_dim, embed_dim)
+
+    def forward(self, x):
+        query = self.query_linear(x)
+        key = self.key_linear(x)
+        value = self.value_linear(x)
+        attention_weights = torch.matmul(query, key.T) / math.sqrt(x.size(-1))
+        output = torch.matmul(attention_weights, value)
+        return output
+```
+Self attention can also be used in recommender systems to model user-item interactions, allowing for more accurate personalized recommendations by considering the relationships between different items.
+
+## Common Mistakes in Self Attention
+When working with self attention models, several common pitfalls can hinder performance and lead to suboptimal results. 
+
+* Overfitting is a significant problem in self attention models, where the model becomes too specialized to the training data and fails to generalize well to new, unseen data. This can be mitigated by using techniques such as dropout and early stopping, which help to prevent the model from becoming too complex.
+
+Proper initialization and regularization are also crucial when using self attention. Initialization with random weights can lead to slow convergence or getting stuck in local minima, while regularization techniques like L1 and L2 regularization can help to prevent overfitting by adding a penalty term to the loss function.
+
+To debug self attention models, follow these steps:
+* Check the input data for any inconsistencies or missing values
+* Verify that the model is correctly implemented, with attention weights being properly computed and applied
+* Monitor the model's performance on a validation set during training, and adjust hyperparameters as needed to prevent overfitting. 
+By being aware of these common mistakes and taking steps to avoid them, developers can build more effective and reliable self attention models.
+
+## Best Practices for Self Attention
+To ensure effective use of self attention in your projects, follow this checklist for production readiness:
+* Validate input data quality
+* Test model performance on diverse datasets
+* Monitor training time and memory usage
+Monitoring performance metrics, such as accuracy and loss, is crucial for identifying potential issues. 
+For further learning and improvement, refer to the Transformer library documentation and research papers on self attention mechanisms.

notebook/web.ipynb

@@ -0,0 +1,255 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "3d9158b7",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "c:\\Users\\vansh\\Projects\\AIAgents\\.venv\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
+   "source": [
+    "from langchain_aws import ChatBedrockConverse\n",
+    "LLM_MODEL_ID = \"us.meta.llama3-3-70b-instruct-v1:0\"\n",
+    "LLM_REGION = \"us-east-1\"\n",
+    "\n",
+    "import logging\n",
+    "llm = ChatBedrockConverse(\n",
+    "    model_id=LLM_MODEL_ID,\n",
+    "    region_name=LLM_REGION\n",
+    ")\n",
+    "logging.info(f\"LLM initialized with model_id={LLM_MODEL_ID}, region_name={LLM_REGION}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "cad5e68d",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "WARNING:langchain_community.utils.user_agent:USER_AGENT environment variable not set, consider setting it to identify your requests.\n"
+     ]
+    }
+   ],
+   "source": [
+    "from langchain_community.document_loaders import WebBaseLoader"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "2a289acb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "loader=WebBaseLoader(\"https://docs.langchain.com/oss/python/integrations/document_loaders/web_base\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "d1fcbd5f",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[Document(metadata={'source': 'https://docs.langchain.com/oss/python/integrations/document_loaders/web_base', 'title': 'WebBaseLoader integration - Docs by LangChain', 'description': 'Integrate with the WebBaseLoader document loader using LangChain Python.', 'language': 'en'}, page_content='WebBaseLoader integration - Docs by LangChainSkip to main contentDocs by LangChain home pageOpen sourceSearch...⌘KAsk AIGitHubTry LangSmithTry LangSmithSearch...NavigationWebBaseLoader integrationDeep AgentsLangChainLangGraphIntegrationsLearnReferenceContributePythonLangChain integrationsAll providersPopular ProvidersOpenAIAnthropicGoogleAWSHugging FaceMicrosoftOllamaGroqNVIDIAIntegrations by componentChat modelsTools and toolkitsMiddlewareRetrieversText splittersEmbedding modelsVector storesDocument loadersKey-value storesOn this pageOverviewIntegration detailsLoader featuresSetupCredentialsInstallationInitializationInitialization with multiple pagesLoadLoad multiple urls concurrentlyLoading a xml file, or using a different BeautifulSoup parserLazy loadAsyncUsing proxiesAPI referenceWebBaseLoader integrationCopy pageIntegrate with the WebBaseLoader document loader using LangChain Python.Copy pageThis covers how to use WebBaseLoader to load all text from HTML webpages into a document format that we can use downstream. For more custom logic for loading webpages look at some child class examples such as IMSDbLoader, AZLyricsLoader, and CollegeConfidentialLoader.\\nIf you don’t want to worry about website crawling, bypassing JS-blocking sites, and data cleaning, consider using FireCrawlLoader or the faster option SpiderLoader.\\n\\u200bOverview\\n\\u200bIntegration details\\n\\nTODO: Fill in table features.\\nTODO: Remove JS support link if not relevant, otherwise ensure link is correct.\\nTODO: Make sure API reference links are correct.\\n\\nClassPackageLocalSerializableJS supportWebBaseLoaderlangchain-community✅❌❌\\n\\u200bLoader features\\nSourceDocument Lazy LoadingNative Async SupportWebBaseLoader✅✅\\n\\u200bSetup\\n\\u200bCredentials\\nWebBaseLoader does not require any credentials.\\n\\u200bInstallation\\nTo use the WebBaseLoader you first need to install the langchain-community python package.\\nCopypip install -qU langchain-community beautifulsoup4\\n\\n\\u200bInitialization\\nNow we can instantiate our model object and load documents:\\nCopyfrom langchain_community.document_loaders import WebBaseLoader\\n\\nloader = WebBaseLoader(\"https://www.example.com/\")\\n\\nTo bypass SSL verification errors during fetching, you can set the “verify” option:\\nloader.requests_kwargs = {\\'verify\\':False}\\n\\u200bInitialization with multiple pages\\nYou can also pass in a list of pages to load from.\\nCopyloader_multiple_pages = WebBaseLoader(\\n    [\"https://www.example.com/\", \"https://google.com\"]\\n)\\n\\n\\u200bLoad\\nCopydocs = loader.load()\\n\\ndocs[0]\\n\\nCopyDocument(metadata={\\'source\\': \\'https://www.example.com/\\', \\'title\\': \\'Example Domain\\', \\'language\\': \\'No language found.\\'}, page_content=\\'\\\\n\\\\n\\\\nExample Domain\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\nExample Domain\\\\nThis domain is for use in illustrative examples in documents. You may use this\\\\n    domain in literature without prior coordination or asking for permission.\\\\nMore information...\\\\n\\\\n\\\\n\\\\n\\')\\n\\nCopyprint(docs[0].metadata)\\n\\nCopy{\\'source\\': \\'https://www.example.com/\\', \\'title\\': \\'Example Domain\\', \\'language\\': \\'No language found.\\'}\\n\\n\\u200bLoad multiple urls concurrently\\nYou can speed up the scraping process by scraping and parsing multiple urls concurrently.\\nThere are reasonable limits to concurrent requests, defaulting to 2 per second.  If you aren’t concerned about being a good citizen, or you control the server you are scraping and don’t care about load, you can change the requests_per_second parameter to increase the max concurrent requests.  Note, while this will speed up the scraping process, but may cause the server to block you.  Be careful!\\nCopypip install -qU  nest_asyncio\\n\\n# fixes a bug with asyncio and jupyter\\nimport nest_asyncio\\n\\nnest_asyncio.apply()\\n\\nCopyloader = WebBaseLoader([\"https://www.example.com/\", \"https://google.com\"])\\nloader.requests_per_second = 1\\ndocs = loader.aload()\\ndocs\\n\\nCopyFetching pages: 100%|###########################################################################| 2/2 [00:00<00:00,  8.28it/s]\\n\\nCopy[Document(metadata={\\'source\\': \\'https://www.example.com/\\', \\'title\\': \\'Example Domain\\', \\'language\\': \\'No language found.\\'}, page_content=\\'\\\\n\\\\n\\\\nExample Domain\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\nExample Domain\\\\nThis domain is for use in illustrative examples in documents. You may use this\\\\n    domain in literature without prior coordination or asking for permission.\\\\nMore information...\\\\n\\\\n\\\\n\\\\n\\'),\\n Document(metadata={\\'source\\': \\'https://google.com\\', \\'title\\': \\'Google\\', \\'description\\': \"Search the world\\'s information, including webpages, images, videos and more. Google has many special features to help you find exactly what you\\'re looking for.\", \\'language\\': \\'en\\'}, page_content=\\'GoogleSearch Images Maps Play YouTube News Gmail Drive More »Web History | Settings | Sign in\\\\xa0Advanced search5 ways Gemini can help during the HolidaysAdvertisingBusiness SolutionsAbout Google© 2024 - Privacy - Terms  \\')]\\n\\n\\u200bLoading a xml file, or using a different BeautifulSoup parser\\nYou can also look at SitemapLoader for an example of how to load a sitemap file, which is an example of using this feature.\\nCopyloader = WebBaseLoader(\\n    \"https://www.govinfo.gov/content/pkg/CFR-2018-title10-vol3/xml/CFR-2018-title10-vol3-sec431-86.xml\"\\n)\\nloader.default_parser = \"xml\"\\ndocs = loader.load()\\ndocs\\n\\nCopy[Document(metadata={\\'source\\': \\'https://www.govinfo.gov/content/pkg/CFR-2018-title10-vol3/xml/CFR-2018-title10-vol3-sec431-86.xml\\'}, page_content=\\'\\\\n\\\\n10\\\\nEnergy\\\\n3\\\\n2018-01-01\\\\n2018-01-01\\\\nfalse\\\\nUniform test method for the measurement of energy efficiency of commercial packaged boilers.\\\\n§ 431.86\\\\nSection § 431.86\\\\n\\\\nEnergy\\\\nDEPARTMENT OF ENERGY\\\\nENERGY CONSERVATION\\\\nENERGY EFFICIENCY PROGRAM FOR CERTAIN COMMERCIAL AND INDUSTRIAL EQUIPMENT\\\\nCommercial Packaged Boilers\\\\nTest Procedures\\\\n\\\\n\\\\n\\\\n\\\\n§\\\\u2009431.86\\\\nUniform test method for the measurement of energy efficiency of commercial packaged boilers.\\\\n(a) Scope. This section provides test procedures, pursuant to the Energy Policy and Conservation Act (EPCA), as amended, which must be followed for measuring the combustion efficiency and/or thermal efficiency of a gas- or oil-fired commercial packaged boiler.\\\\n(b) Testing and Calculations. Determine the thermal efficiency or combustion efficiency of commercial packaged boilers by conducting the appropriate test procedure(s) indicated in Table 1 of this section.\\\\n\\\\nTable 1—Test Requirements for Commercial Packaged Boiler Equipment Classes\\\\n\\\\nEquipment category\\\\nSubcategory\\\\nCertified rated inputBtu/h\\\\n\\\\nStandards efficiency metric(§\\\\u2009431.87)\\\\n\\\\nTest procedure(corresponding to\\\\nstandards efficiency\\\\nmetric required\\\\nby §\\\\u2009431.87)\\\\n\\\\n\\\\n\\\\nHot Water\\\\nGas-fired\\\\n≥300,000 and ≤2,500,000\\\\nThermal Efficiency\\\\nAppendix A, Section 2.\\\\n\\\\n\\\\nHot Water\\\\nGas-fired\\\\n>2,500,000\\\\nCombustion Efficiency\\\\nAppendix A, Section 3.\\\\n\\\\n\\\\nHot Water\\\\nOil-fired\\\\n≥300,000 and ≤2,500,000\\\\nThermal Efficiency\\\\nAppendix A, Section 2.\\\\n\\\\n\\\\nHot Water\\\\nOil-fired\\\\n>2,500,000\\\\nCombustion Efficiency\\\\nAppendix A, Section 3.\\\\n\\\\n\\\\nSteam\\\\nGas-fired (all*)\\\\n≥300,000 and ≤2,500,000\\\\nThermal Efficiency\\\\nAppendix A, Section 2.\\\\n\\\\n\\\\nSteam\\\\nGas-fired (all*)\\\\n>2,500,000 and ≤5,000,000\\\\nThermal Efficiency\\\\nAppendix A, Section 2.\\\\n\\\\n\\\\n\\\\u2003\\\\n\\\\n>5,000,000\\\\nThermal Efficiency\\\\nAppendix A, Section 2.OR\\\\nAppendix A, Section 3 with Section 2.4.3.2.\\\\n\\\\n\\\\n\\\\nSteam\\\\nOil-fired\\\\n≥300,000 and ≤2,500,000\\\\nThermal Efficiency\\\\nAppendix A, Section 2.\\\\n\\\\n\\\\nSteam\\\\nOil-fired\\\\n>2,500,000 and ≤5,000,000\\\\nThermal Efficiency\\\\nAppendix A, Section 2.\\\\n\\\\n\\\\n\\\\u2003\\\\n\\\\n>5,000,000\\\\nThermal Efficiency\\\\nAppendix A, Section 2.OR\\\\nAppendix A, Section 3. with Section 2.4.3.2.\\\\n\\\\n\\\\n\\\\n*\\\\u2009Equipment classes for commercial packaged boilers as of July 22, 2009 (74 FR 36355) distinguish between gas-fired natural draft and all other gas-fired (except natural draft).\\\\n\\\\n(c) Field Tests. The field test provisions of appendix A may be used only to test a unit of commercial packaged boiler with rated input greater than 5,000,000 Btu/h.\\\\n[81 FR 89305, Dec. 9, 2016]\\\\n\\\\n\\\\nEnergy Efficiency Standards\\\\n\\\\n\\')]\\n\\n\\u200bLazy load\\nYou can use lazy loading to only load one page at a time in order to minimize memory requirements.\\nCopypages = []\\nfor doc in loader.lazy_load():\\n    pages.append(doc)\\n\\nprint(pages[0].page_content[:100])\\nprint(pages[0].metadata)\\n\\nCopy10\\nEnergy\\n3\\n2018-01-01\\n2018-01-01\\nfalse\\nUniform test method for the measurement of energy efficien\\n{\\'source\\': \\'https://www.govinfo.gov/content/pkg/CFR-2018-title10-vol3/xml/CFR-2018-title10-vol3-sec431-86.xml\\'}\\n\\n\\u200bAsync\\nCopypages = []\\nasync for doc in loader.alazy_load():\\n    pages.append(doc)\\n\\nprint(pages[0].page_content[:100])\\nprint(pages[0].metadata)\\n\\nCopyFetching pages: 100%|###########################################################################| 1/1 [00:00<00:00, 10.51it/s]\\n\\nCopy10\\nEnergy\\n3\\n2018-01-01\\n2018-01-01\\nfalse\\nUniform test method for the measurement of energy efficien\\n{\\'source\\': \\'https://www.govinfo.gov/content/pkg/CFR-2018-title10-vol3/xml/CFR-2018-title10-vol3-sec431-86.xml\\'}\\n\\n\\u200bUsing proxies\\nSometimes you might need to use proxies to get around IP blocks. You can pass in a dictionary of proxies to the loader (and requests underneath) to use them.\\nCopyloader = WebBaseLoader(\\n        \"https://www.walmart.com/search?q=parrots\",\\n        proxies={\\n        \"http\": \"http://{username}:{password}:@proxy.service.com:6666/\",\\n        \"https\": \"https://{username}:{password}:@proxy.service.com:6666/\",\\n    },\\n)\\ndocs = loader.load()\\n\\n\\n\\u200bAPI reference\\nFor detailed documentation of all WebBaseLoader features and configurations head to the API reference: python.langchain.com/api_reference/community/document_loaders/langchain_community.document_loaders.web_base.WebBaseLoader.html\\n\\nEdit this page on GitHub or file an issue.Connect these docs to Claude, VSCode, and more via MCP for real-time answers.Was this page helpful?YesNo⌘IDocs by LangChain home pagegithubxlinkedinyoutubeResourcesForumChangelogLangChain AcademyTrust CenterCompanyHomeAboutCareersBloggithubxlinkedinyoutube\\n')]"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "docs=loader.load()\n",
+    "docs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "0ee58d3a",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'WebBaseLoader integration - Docs by LangChainSkip to main contentDocs by LangChain home pageOpen sourceSearch...⌘KAsk AIGitHubTry LangSmithTry LangSmithSearch...NavigationWebBaseLoader integrationDeep AgentsLangChainLangGraphIntegrationsLearnReferenceContributePythonLangChain integrationsAll providersPopular ProvidersOpenAIAnthropicGoogleAWSHugging FaceMicrosoftOllamaGroqNVIDIAIntegrations by componentChat modelsTools and toolkitsMiddlewareRetrieversText splittersEmbedding modelsVector storesDocument loadersKey-value storesOn this pageOverviewIntegration detailsLoader featuresSetupCredentialsInstallationInitializationInitialization with multiple pagesLoadLoad multiple urls concurrentlyLoading a xml file, or using a different BeautifulSoup parserLazy loadAsyncUsing proxiesAPI referenceWebBaseLoader integrationCopy pageIntegrate with the WebBaseLoader document loader using LangChain Python.Copy pageThis covers how to use WebBaseLoader to load all text from HTML webpages into a document format that we can use downstream. For more custom logic for loading webpages look at some child class examples such as IMSDbLoader, AZLyricsLoader, and CollegeConfidentialLoader.\\nIf you don’t want to worry about website crawling, bypassing JS-blocking sites, and data cleaning, consider using FireCrawlLoader or the faster option SpiderLoader.\\n\\u200bOverview\\n\\u200bIntegration details\\n\\nTODO: Fill in table features.\\nTODO: Remove JS support link if not relevant, otherwise ensure link is correct.\\nTODO: Make sure API reference links are correct.\\n\\nClassPackageLocalSerializableJS supportWebBaseLoaderlangchain-community✅❌❌\\n\\u200bLoader features\\nSourceDocument Lazy LoadingNative Async SupportWebBaseLoader✅✅\\n\\u200bSetup\\n\\u200bCredentials\\nWebBaseLoader does not require any credentials.\\n\\u200bInstallation\\nTo use the WebBaseLoader you first need to install the langchain-community python package.\\nCopypip install -qU langchain-community beautifulsoup4\\n\\n\\u200bInitialization\\nNow we can instantiate our model object and load documents:\\nCopyfrom langchain_community.document_loaders import WebBaseLoader\\n\\nloader = WebBaseLoader(\"https://www.example.com/\")\\n\\nTo bypass SSL verification errors during fetching, you can set the “verify” option:\\nloader.requests_kwargs = {\\'verify\\':False}\\n\\u200bInitialization with multiple pages\\nYou can also pass in a list of pages to load from.\\nCopyloader_multiple_pages = WebBaseLoader(\\n    [\"https://www.example.com/\", \"https://google.com\"]\\n)\\n\\n\\u200bLoad\\nCopydocs = loader.load()\\n\\ndocs[0]\\n\\nCopyDocument(metadata={\\'source\\': \\'https://www.example.com/\\', \\'title\\': \\'Example Domain\\', \\'language\\': \\'No language found.\\'}, page_content=\\'\\\\n\\\\n\\\\nExample Domain\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\nExample Domain\\\\nThis domain is for use in illustrative examples in documents. You may use this\\\\n    domain in literature without prior coordination or asking for permission.\\\\nMore information...\\\\n\\\\n\\\\n\\\\n\\')\\n\\nCopyprint(docs[0].metadata)\\n\\nCopy{\\'source\\': \\'https://www.example.com/\\', \\'title\\': \\'Example Domain\\', \\'language\\': \\'No language found.\\'}\\n\\n\\u200bLoad multiple urls concurrently\\nYou can speed up the scraping process by scraping and parsing multiple urls concurrently.\\nThere are reasonable limits to concurrent requests, defaulting to 2 per second.  If you aren’t concerned about being a good citizen, or you control the server you are scraping and don’t care about load, you can change the requests_per_second parameter to increase the max concurrent requests.  Note, while this will speed up the scraping process, but may cause the server to block you.  Be careful!\\nCopypip install -qU  nest_asyncio\\n\\n# fixes a bug with asyncio and jupyter\\nimport nest_asyncio\\n\\nnest_asyncio.apply()\\n\\nCopyloader = WebBaseLoader([\"https://www.example.com/\", \"https://google.com\"])\\nloader.requests_per_second = 1\\ndocs = loader.aload()\\ndocs\\n\\nCopyFetching pages: 100%|###########################################################################| 2/2 [00:00<00:00,  8.28it/s]\\n\\nCopy[Document(metadata={\\'source\\': \\'https://www.example.com/\\', \\'title\\': \\'Example Domain\\', \\'language\\': \\'No language found.\\'}, page_content=\\'\\\\n\\\\n\\\\nExample Domain\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\nExample Domain\\\\nThis domain is for use in illustrative examples in documents. You may use this\\\\n    domain in literature without prior coordination or asking for permission.\\\\nMore information...\\\\n\\\\n\\\\n\\\\n\\'),\\n Document(metadata={\\'source\\': \\'https://google.com\\', \\'title\\': \\'Google\\', \\'description\\': \"Search the world\\'s information, including webpages, images, videos and more. Google has many special features to help you find exactly what you\\'re looking for.\", \\'language\\': \\'en\\'}, page_content=\\'GoogleSearch Images Maps Play YouTube News Gmail Drive More »Web History | Settings | Sign in\\\\xa0Advanced search5 ways Gemini can help during the HolidaysAdvertisingBusiness SolutionsAbout Google© 2024 - Privacy - Terms  \\')]\\n\\n\\u200bLoading a xml file, or using a different BeautifulSoup parser\\nYou can also look at SitemapLoader for an example of how to load a sitemap file, which is an example of using this feature.\\nCopyloader = WebBaseLoader(\\n    \"https://www.govinfo.gov/content/pkg/CFR-2018-title10-vol3/xml/CFR-2018-title10-vol3-sec431-86.xml\"\\n)\\nloader.default_parser = \"xml\"\\ndocs = loader.load()\\ndocs\\n\\nCopy[Document(metadata={\\'source\\': \\'https://www.govinfo.gov/content/pkg/CFR-2018-title10-vol3/xml/CFR-2018-title10-vol3-sec431-86.xml\\'}, page_content=\\'\\\\n\\\\n10\\\\nEnergy\\\\n3\\\\n2018-01-01\\\\n2018-01-01\\\\nfalse\\\\nUniform test method for the measurement of energy efficiency of commercial packaged boilers.\\\\n§ 431.86\\\\nSection § 431.86\\\\n\\\\nEnergy\\\\nDEPARTMENT OF ENERGY\\\\nENERGY CONSERVATION\\\\nENERGY EFFICIENCY PROGRAM FOR CERTAIN COMMERCIAL AND INDUSTRIAL EQUIPMENT\\\\nCommercial Packaged Boilers\\\\nTest Procedures\\\\n\\\\n\\\\n\\\\n\\\\n§\\\\u2009431.86\\\\nUniform test method for the measurement of energy efficiency of commercial packaged boilers.\\\\n(a) Scope. This section provides test procedures, pursuant to the Energy Policy and Conservation Act (EPCA), as amended, which must be followed for measuring the combustion efficiency and/or thermal efficiency of a gas- or oil-fired commercial packaged boiler.\\\\n(b) Testing and Calculations. Determine the thermal efficiency or combustion efficiency of commercial packaged boilers by conducting the appropriate test procedure(s) indicated in Table 1 of this section.\\\\n\\\\nTable 1—Test Requirements for Commercial Packaged Boiler Equipment Classes\\\\n\\\\nEquipment category\\\\nSubcategory\\\\nCertified rated inputBtu/h\\\\n\\\\nStandards efficiency metric(§\\\\u2009431.87)\\\\n\\\\nTest procedure(corresponding to\\\\nstandards efficiency\\\\nmetric required\\\\nby §\\\\u2009431.87)\\\\n\\\\n\\\\n\\\\nHot Water\\\\nGas-fired\\\\n≥300,000 and ≤2,500,000\\\\nThermal Efficiency\\\\nAppendix A, Section 2.\\\\n\\\\n\\\\nHot Water\\\\nGas-fired\\\\n>2,500,000\\\\nCombustion Efficiency\\\\nAppendix A, Section 3.\\\\n\\\\n\\\\nHot Water\\\\nOil-fired\\\\n≥300,000 and ≤2,500,000\\\\nThermal Efficiency\\\\nAppendix A, Section 2.\\\\n\\\\n\\\\nHot Water\\\\nOil-fired\\\\n>2,500,000\\\\nCombustion Efficiency\\\\nAppendix A, Section 3.\\\\n\\\\n\\\\nSteam\\\\nGas-fired (all*)\\\\n≥300,000 and ≤2,500,000\\\\nThermal Efficiency\\\\nAppendix A, Section 2.\\\\n\\\\n\\\\nSteam\\\\nGas-fired (all*)\\\\n>2,500,000 and ≤5,000,000\\\\nThermal Efficiency\\\\nAppendix A, Section 2.\\\\n\\\\n\\\\n\\\\u2003\\\\n\\\\n>5,000,000\\\\nThermal Efficiency\\\\nAppendix A, Section 2.OR\\\\nAppendix A, Section 3 with Section 2.4.3.2.\\\\n\\\\n\\\\n\\\\nSteam\\\\nOil-fired\\\\n≥300,000 and ≤2,500,000\\\\nThermal Efficiency\\\\nAppendix A, Section 2.\\\\n\\\\n\\\\nSteam\\\\nOil-fired\\\\n>2,500,000 and ≤5,000,000\\\\nThermal Efficiency\\\\nAppendix A, Section 2.\\\\n\\\\n\\\\n\\\\u2003\\\\n\\\\n>5,000,000\\\\nThermal Efficiency\\\\nAppendix A, Section 2.OR\\\\nAppendix A, Section 3. with Section 2.4.3.2.\\\\n\\\\n\\\\n\\\\n*\\\\u2009Equipment classes for commercial packaged boilers as of July 22, 2009 (74 FR 36355) distinguish between gas-fired natural draft and all other gas-fired (except natural draft).\\\\n\\\\n(c) Field Tests. The field test provisions of appendix A may be used only to test a unit of commercial packaged boiler with rated input greater than 5,000,000 Btu/h.\\\\n[81 FR 89305, Dec. 9, 2016]\\\\n\\\\n\\\\nEnergy Efficiency Standards\\\\n\\\\n\\')]\\n\\n\\u200bLazy load\\nYou can use lazy loading to only load one page at a time in order to minimize memory requirements.\\nCopypages = []\\nfor doc in loader.lazy_load():\\n    pages.append(doc)\\n\\nprint(pages[0].page_content[:100])\\nprint(pages[0].metadata)\\n\\nCopy10\\nEnergy\\n3\\n2018-01-01\\n2018-01-01\\nfalse\\nUniform test method for the measurement of energy efficien\\n{\\'source\\': \\'https://www.govinfo.gov/content/pkg/CFR-2018-title10-vol3/xml/CFR-2018-title10-vol3-sec431-86.xml\\'}\\n\\n\\u200bAsync\\nCopypages = []\\nasync for doc in loader.alazy_load():\\n    pages.append(doc)\\n\\nprint(pages[0].page_content[:100])\\nprint(pages[0].metadata)\\n\\nCopyFetching pages: 100%|###########################################################################| 1/1 [00:00<00:00, 10.51it/s]\\n\\nCopy10\\nEnergy\\n3\\n2018-01-01\\n2018-01-01\\nfalse\\nUniform test method for the measurement of energy efficien\\n{\\'source\\': \\'https://www.govinfo.gov/content/pkg/CFR-2018-title10-vol3/xml/CFR-2018-title10-vol3-sec431-86.xml\\'}\\n\\n\\u200bUsing proxies\\nSometimes you might need to use proxies to get around IP blocks. You can pass in a dictionary of proxies to the loader (and requests underneath) to use them.\\nCopyloader = WebBaseLoader(\\n        \"https://www.walmart.com/search?q=parrots\",\\n        proxies={\\n        \"http\": \"http://{username}:{password}:@proxy.service.com:6666/\",\\n        \"https\": \"https://{username}:{password}:@proxy.service.com:6666/\",\\n    },\\n)\\ndocs = loader.load()\\n\\n\\n\\u200bAPI reference\\nFor detailed documentation of all WebBaseLoader features and configurations head to the API reference: python.langchain.com/api_reference/community/document_loaders/langchain_community.document_loaders.web_base.WebBaseLoader.html\\n\\nEdit this page on GitHub or file an issue.Connect these docs to Claude, VSCode, and more via MCP for real-time answers.Was this page helpful?YesNo⌘IDocs by LangChain home pagegithubxlinkedinyoutubeResourcesForumChangelogLangChain AcademyTrust CenterCompanyHomeAboutCareersBloggithubxlinkedinyoutube\\n'"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "docs[0].page_content"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "775a77b9",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "1"
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(docs)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "47202522",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "992"
+      ]
+     },
+     "execution_count": 16,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(docs[0].page_content.split())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "46f0d682",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain_community.document_loaders import YoutubeLoader"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "c0cb3d1f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "loader = YoutubeLoader.from_youtube_url(\n",
+    "    \"https://www.youtube.com/watch?v=QsYGlZkevEg\", add_video_info=False\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "496ebb6d",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[Document(metadata={'source': 'QsYGlZkevEg'}, page_content='LADIES AND GENTLEMEN, PEDRO PASCAL! [ CHEERS AND APPLAUSE ] >> THANK YOU, THANK YOU. THANK YOU VERY MUCH. I\\'M SO EXCITED TO BE HERE. THANK YOU. I SPENT THE LAST YEAR SHOOTING A SHOW CALLED \"THE LAST OF US\" ON HBO. FOR SOME HBO SHOES, YOU GET TO SHOOT IN A FIVE STAR ITALIAN RESORT SURROUNDED BY BEAUTIFUL PEOPLE, BUT I SAID, NO, THAT\\'S TOO EASY. I WANT TO SHOOT IN A FREEZING CANADIAN FOREST WHILE BEING CHASED AROUND BY A GUY WHOSE HEAD LOOKS LIKE A GENITAL WART. IT IS AN HONOR BEING A PART OF THESE HUGE FRANCHISEs LIKE \"GAME OF THRONES\" AND \"STAR WARS,\" BUT I\\'M STILL GETTING USED TO PEOPLE RECOGNIZING ME. THE OTHER DAY, A GUY STOPPED ME ON THE STREET AND SAYS, MY SON LOVES \"THE MANDALORIAN\" AND THE NEXT THING I KNOW, I\\'M FACE TIMING WITH A 6-YEAR-OLD WHO HAS NO IDEA WHO I AM BECAUSE MY CHARACTER WEARS A MASK THE ENTIRE SHOW. THE GUY IS LIKE, DO THE MANDO VOICE, BUT IT\\'S LIKE A BEDROOM VOICE. WITHOUT THE MASK, IT JUST SOUNDS PORNY. PEOPLE WALKING BY ON THE STREET SEE ME WHISPERING TO A 6-YEAR-OLD KID. I CAN BRING YOU IN WARM, OR I CAN BRING YOU IN COLD. EVEN THOUGH I CAME TO THE U.S. WHEN I WAS LITTLE, I WAS BORN IN CHILE, AND I HAVE 34 FIRST COUSINS WHO ARE STILL THERE. THEY\\'RE VERY PROUD OF ME. I KNOW THEY\\'RE PROUD BECAUSE THEY GIVE MY PHONE NUMBER TO EVERY PERSON THEY MEET, WHICH MEANS EVERY DAY, SOMEONE IN SANTIAGO WILL TEXT ME STUFF LIKE, CAN YOU COME TO MY WEDDING, OR CAN YOU SING MY PRIEST HAPPY BIRTHDAY, OR IS BABY YODA MEAN IN REAL LIFE. SO I HAVE TO BE LIKE NO, NO, AND HIS NAME IS GROGU. BUT MY COUSINS WEREN\\'T ALWAYS SO PROUD. EARLY IN MY CAREER, I PLAYED SMALL PARTS IN EVERY CRIME SHOW. I EVEN PLAYED TWO DIFFERENT CHARACTERS ON \"LAW AND ORDER.\" TITO CABASSA WHO LOOKED LIKE THIS. AND ONE YEAR LATER, I PLAYED REGGIE LUCKMAN WHO LOOKS LIKE THIS. AND THAT, MY FRIENDS, IS CALLED RANGE. BUT IT IS AMAZING TO BE HERE, LIKE I SAID. I WAS BORN IN CHILE, AND NINE MONTHS LATER, MY PARENTS FLED AND BROUGHT ME AND MY SISTER TO THE U.S. THEY WERE SO BRAVE, AND WITHOUT THEM, I WOULDN\\'T BE HERE IN THIS WONDERFUL COUNTRY, AND I CERTAINLY WOULDN\\'T BE STANDING HERE WITH YOU ALL TONIGHT. SO TO ALL MY FAMILY WATCHING IN CHILE, I WANT TO SAY [ SPEAKING NON-ENGLISH ] WHICH MEANS, I LOVE YOU, I MISS YOU, AND STOP GIVING OUT MY PHONE NUMBER. WE\\'VE GOT AN AMAZING SHOW FOR YOU TONIGHT. COLDPLAY IS HERE, SO STICK')]"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "loader.load()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "3c4920b7",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "HTTPError",
+     "evalue": "HTTP Error 400: Bad Request",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
+      "\u001b[31mHTTPError\u001b[39m                                 Traceback (most recent call last)",
+      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[8]\u001b[39m\u001b[32m, line 4\u001b[39m\n\u001b[32m      1\u001b[39m loader = YoutubeLoader.from_youtube_url(\n\u001b[32m      2\u001b[39m     \u001b[33m\"\u001b[39m\u001b[33mhttps://www.youtube.com/watch?v=QsYGlZkevEg\u001b[39m\u001b[33m\"\u001b[39m, add_video_info=\u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[32m      3\u001b[39m )\n\u001b[32m----> \u001b[39m\u001b[32m4\u001b[39m \u001b[43mloader\u001b[49m\u001b[43m.\u001b[49m\u001b[43mload\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
+      "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\vansh\\Projects\\AIAgents\\.venv\\Lib\\site-packages\\langchain_community\\document_loaders\\youtube.py:258\u001b[39m, in \u001b[36mYoutubeLoader.load\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m    250\u001b[39m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m(\n\u001b[32m    251\u001b[39m         \u001b[33m'\u001b[39m\u001b[33mCould not import \u001b[39m\u001b[33m\"\u001b[39m\u001b[33myoutube_transcript_api\u001b[39m\u001b[33m\"\u001b[39m\u001b[33m Python package. \u001b[39m\u001b[33m'\u001b[39m\n\u001b[32m    252\u001b[39m         \u001b[33m\"\u001b[39m\u001b[33mPlease install it with `pip install youtube-transcript-api`.\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m    253\u001b[39m     )\n\u001b[32m    255\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m.add_video_info:\n\u001b[32m    256\u001b[39m     \u001b[38;5;66;03m# Get more video meta info\u001b[39;00m\n\u001b[32m    257\u001b[39m     \u001b[38;5;66;03m# Such as title, description, thumbnail url, publish_date\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m258\u001b[39m     video_info = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_get_video_info\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    259\u001b[39m     \u001b[38;5;28mself\u001b[39m._metadata.update(video_info)\n\u001b[32m    261\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n",
+      "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\vansh\\Projects\\AIAgents\\.venv\\Lib\\site-packages\\langchain_community\\document_loaders\\youtube.py:336\u001b[39m, in \u001b[36mYoutubeLoader._get_video_info\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m    330\u001b[39m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m(\n\u001b[32m    331\u001b[39m         \u001b[33m'\u001b[39m\u001b[33mCould not import \u001b[39m\u001b[33m\"\u001b[39m\u001b[33mpytube\u001b[39m\u001b[33m\"\u001b[39m\u001b[33m Python package. \u001b[39m\u001b[33m'\u001b[39m\n\u001b[32m    332\u001b[39m         \u001b[33m\"\u001b[39m\u001b[33mPlease install it with `pip install pytube`.\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m    333\u001b[39m     )\n\u001b[32m    334\u001b[39m yt = YouTube(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mhttps://www.youtube.com/watch?v=\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m.video_id\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m)\n\u001b[32m    335\u001b[39m video_info = {\n\u001b[32m--> \u001b[39m\u001b[32m336\u001b[39m     \u001b[33m\"\u001b[39m\u001b[33mtitle\u001b[39m\u001b[33m\"\u001b[39m: \u001b[43myt\u001b[49m\u001b[43m.\u001b[49m\u001b[43mtitle\u001b[49m \u001b[38;5;129;01mor\u001b[39;00m \u001b[33m\"\u001b[39m\u001b[33mUnknown\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m    337\u001b[39m     \u001b[33m\"\u001b[39m\u001b[33mdescription\u001b[39m\u001b[33m\"\u001b[39m: yt.description \u001b[38;5;129;01mor\u001b[39;00m \u001b[33m\"\u001b[39m\u001b[33mUnknown\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m    338\u001b[39m     \u001b[33m\"\u001b[39m\u001b[33mview_count\u001b[39m\u001b[33m\"\u001b[39m: yt.views \u001b[38;5;129;01mor\u001b[39;00m \u001b[32m0\u001b[39m,\n\u001b[32m    339\u001b[39m     \u001b[33m\"\u001b[39m\u001b[33mthumbnail_url\u001b[39m\u001b[33m\"\u001b[39m: yt.thumbnail_url \u001b[38;5;129;01mor\u001b[39;00m \u001b[33m\"\u001b[39m\u001b[33mUnknown\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m    340\u001b[39m     \u001b[33m\"\u001b[39m\u001b[33mpublish_date\u001b[39m\u001b[33m\"\u001b[39m: yt.publish_date.strftime(\u001b[33m\"\u001b[39m\u001b[33m%\u001b[39m\u001b[33mY-\u001b[39m\u001b[33m%\u001b[39m\u001b[33mm-\u001b[39m\u001b[38;5;132;01m%d\u001b[39;00m\u001b[33m \u001b[39m\u001b[33m%\u001b[39m\u001b[33mH:\u001b[39m\u001b[33m%\u001b[39m\u001b[33mM:\u001b[39m\u001b[33m%\u001b[39m\u001b[33mS\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m    341\u001b[39m     \u001b[38;5;28;01mif\u001b[39;00m yt.publish_date\n\u001b[32m    342\u001b[39m     \u001b[38;5;28;01melse\u001b[39;00m \u001b[33m\"\u001b[39m\u001b[33mUnknown\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m    343\u001b[39m     \u001b[33m\"\u001b[39m\u001b[33mlength\u001b[39m\u001b[33m\"\u001b[39m: yt.length \u001b[38;5;129;01mor\u001b[39;00m \u001b[32m0\u001b[39m,\n\u001b[32m    344\u001b[39m     \u001b[33m\"\u001b[39m\u001b[33mauthor\u001b[39m\u001b[33m\"\u001b[39m: yt.author \u001b[38;5;129;01mor\u001b[39;00m \u001b[33m\"\u001b[39m\u001b[33mUnknown\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m    345\u001b[39m }\n\u001b[32m    346\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m video_info\n",
+      "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\vansh\\Projects\\AIAgents\\.venv\\Lib\\site-packages\\pytube\\__main__.py:341\u001b[39m, in \u001b[36mYouTube.title\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m    338\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m._title\n\u001b[32m    340\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m341\u001b[39m     \u001b[38;5;28mself\u001b[39m._title = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mvid_info\u001b[49m[\u001b[33m'\u001b[39m\u001b[33mvideoDetails\u001b[39m\u001b[33m'\u001b[39m][\u001b[33m'\u001b[39m\u001b[33mtitle\u001b[39m\u001b[33m'\u001b[39m]\n\u001b[32m    342\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m:\n\u001b[32m    343\u001b[39m     \u001b[38;5;66;03m# Check_availability will raise the correct exception in most cases\u001b[39;00m\n\u001b[32m    344\u001b[39m     \u001b[38;5;66;03m#  if it doesn't, ask for a report.\u001b[39;00m\n\u001b[32m    345\u001b[39m     \u001b[38;5;28mself\u001b[39m.check_availability()\n",
+      "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\vansh\\Projects\\AIAgents\\.venv\\Lib\\site-packages\\pytube\\__main__.py:246\u001b[39m, in \u001b[36mYouTube.vid_info\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m    242\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m._vid_info\n\u001b[32m    244\u001b[39m innertube = InnerTube(use_oauth=\u001b[38;5;28mself\u001b[39m.use_oauth, allow_cache=\u001b[38;5;28mself\u001b[39m.allow_oauth_cache)\n\u001b[32m--> \u001b[39m\u001b[32m246\u001b[39m innertube_response = \u001b[43minnertube\u001b[49m\u001b[43m.\u001b[49m\u001b[43mplayer\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mvideo_id\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    247\u001b[39m \u001b[38;5;28mself\u001b[39m._vid_info = innertube_response\n\u001b[32m    248\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m._vid_info\n",
+      "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\vansh\\Projects\\AIAgents\\.venv\\Lib\\site-packages\\pytube\\innertube.py:448\u001b[39m, in \u001b[36mInnerTube.player\u001b[39m\u001b[34m(self, video_id)\u001b[39m\n\u001b[32m    444\u001b[39m query = {\n\u001b[32m    445\u001b[39m     \u001b[33m'\u001b[39m\u001b[33mvideoId\u001b[39m\u001b[33m'\u001b[39m: video_id,\n\u001b[32m    446\u001b[39m }\n\u001b[32m    447\u001b[39m query.update(\u001b[38;5;28mself\u001b[39m.base_params)\n\u001b[32m--> \u001b[39m\u001b[32m448\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_call_api\u001b[49m\u001b[43m(\u001b[49m\u001b[43mendpoint\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mquery\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mbase_data\u001b[49m\u001b[43m)\u001b[49m\n",
+      "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\vansh\\Projects\\AIAgents\\.venv\\Lib\\site-packages\\pytube\\innertube.py:390\u001b[39m, in \u001b[36mInnerTube._call_api\u001b[39m\u001b[34m(self, endpoint, query, data)\u001b[39m\n\u001b[32m    386\u001b[39m         headers[\u001b[33m'\u001b[39m\u001b[33mAuthorization\u001b[39m\u001b[33m'\u001b[39m] = \u001b[33mf\u001b[39m\u001b[33m'\u001b[39m\u001b[33mBearer \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m.access_token\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m'\u001b[39m\n\u001b[32m    388\u001b[39m headers.update(\u001b[38;5;28mself\u001b[39m.header)\n\u001b[32m--> \u001b[39m\u001b[32m390\u001b[39m response = \u001b[43mrequest\u001b[49m\u001b[43m.\u001b[49m\u001b[43m_execute_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m    391\u001b[39m \u001b[43m    \u001b[49m\u001b[43mendpoint_url\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    392\u001b[39m \u001b[43m    \u001b[49m\u001b[33;43m'\u001b[39;49m\u001b[33;43mPOST\u001b[39;49m\u001b[33;43m'\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[32m    393\u001b[39m \u001b[43m    \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    394\u001b[39m \u001b[43m    \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdata\u001b[49m\n\u001b[32m    395\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    396\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m json.loads(response.read())\n",
+      "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\vansh\\Projects\\AIAgents\\.venv\\Lib\\site-packages\\pytube\\request.py:37\u001b[39m, in \u001b[36m_execute_request\u001b[39m\u001b[34m(url, method, headers, data, timeout)\u001b[39m\n\u001b[32m     35\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m     36\u001b[39m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[33m\"\u001b[39m\u001b[33mInvalid URL\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m---> \u001b[39m\u001b[32m37\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m~\\AppData\\Roaming\\uv\\python\\cpython-3.12.12-windows-x86_64-none\\Lib\\urllib\\request.py:215\u001b[39m, in \u001b[36murlopen\u001b[39m\u001b[34m(url, data, timeout, cafile, capath, cadefault, context)\u001b[39m\n\u001b[32m    213\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m    214\u001b[39m     opener = _opener\n\u001b[32m--> \u001b[39m\u001b[32m215\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mopener\u001b[49m\u001b[43m.\u001b[49m\u001b[43mopen\u001b[49m\u001b[43m(\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m~\\AppData\\Roaming\\uv\\python\\cpython-3.12.12-windows-x86_64-none\\Lib\\urllib\\request.py:521\u001b[39m, in \u001b[36mOpenerDirector.open\u001b[39m\u001b[34m(self, fullurl, data, timeout)\u001b[39m\n\u001b[32m    519\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m processor \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m.process_response.get(protocol, []):\n\u001b[32m    520\u001b[39m     meth = \u001b[38;5;28mgetattr\u001b[39m(processor, meth_name)\n\u001b[32m--> \u001b[39m\u001b[32m521\u001b[39m     response = \u001b[43mmeth\u001b[49m\u001b[43m(\u001b[49m\u001b[43mreq\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mresponse\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    523\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m response\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m~\\AppData\\Roaming\\uv\\python\\cpython-3.12.12-windows-x86_64-none\\Lib\\urllib\\request.py:630\u001b[39m, in \u001b[36mHTTPErrorProcessor.http_response\u001b[39m\u001b[34m(self, request, response)\u001b[39m\n\u001b[32m    627\u001b[39m \u001b[38;5;66;03m# According to RFC 2616, \"2xx\" code indicates that the client's\u001b[39;00m\n\u001b[32m    628\u001b[39m \u001b[38;5;66;03m# request was successfully received, understood, and accepted.\u001b[39;00m\n\u001b[32m    629\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[32m200\u001b[39m <= code < \u001b[32m300\u001b[39m):\n\u001b[32m--> \u001b[39m\u001b[32m630\u001b[39m     response = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mparent\u001b[49m\u001b[43m.\u001b[49m\u001b[43merror\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m    631\u001b[39m \u001b[43m        \u001b[49m\u001b[33;43m'\u001b[39;49m\u001b[33;43mhttp\u001b[39;49m\u001b[33;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mresponse\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcode\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmsg\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mhdrs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    633\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m response\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m~\\AppData\\Roaming\\uv\\python\\cpython-3.12.12-windows-x86_64-none\\Lib\\urllib\\request.py:559\u001b[39m, in \u001b[36mOpenerDirector.error\u001b[39m\u001b[34m(self, proto, *args)\u001b[39m\n\u001b[32m    557\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m http_err:\n\u001b[32m    558\u001b[39m     args = (\u001b[38;5;28mdict\u001b[39m, \u001b[33m'\u001b[39m\u001b[33mdefault\u001b[39m\u001b[33m'\u001b[39m, \u001b[33m'\u001b[39m\u001b[33mhttp_error_default\u001b[39m\u001b[33m'\u001b[39m) + orig_args\n\u001b[32m--> \u001b[39m\u001b[32m559\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_call_chain\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m~\\AppData\\Roaming\\uv\\python\\cpython-3.12.12-windows-x86_64-none\\Lib\\urllib\\request.py:492\u001b[39m, in \u001b[36mOpenerDirector._call_chain\u001b[39m\u001b[34m(self, chain, kind, meth_name, *args)\u001b[39m\n\u001b[32m    490\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m handler \u001b[38;5;129;01min\u001b[39;00m handlers:\n\u001b[32m    491\u001b[39m     func = \u001b[38;5;28mgetattr\u001b[39m(handler, meth_name)\n\u001b[32m--> \u001b[39m\u001b[32m492\u001b[39m     result = \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    493\u001b[39m     \u001b[38;5;28;01mif\u001b[39;00m result \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m    494\u001b[39m         \u001b[38;5;28;01mreturn\u001b[39;00m result\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m~\\AppData\\Roaming\\uv\\python\\cpython-3.12.12-windows-x86_64-none\\Lib\\urllib\\request.py:639\u001b[39m, in \u001b[36mHTTPDefaultErrorHandler.http_error_default\u001b[39m\u001b[34m(self, req, fp, code, msg, hdrs)\u001b[39m\n\u001b[32m    638\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mhttp_error_default\u001b[39m(\u001b[38;5;28mself\u001b[39m, req, fp, code, msg, hdrs):\n\u001b[32m--> \u001b[39m\u001b[32m639\u001b[39m     \u001b[38;5;28;01mraise\u001b[39;00m HTTPError(req.full_url, code, msg, hdrs, fp)\n",
+      "\u001b[31mHTTPError\u001b[39m: HTTP Error 400: Bad Request"
+     ]
+    }
+   ],
+   "source": [
+    "loader = YoutubeLoader.from_youtube_url(\n",
+    "    \"https://www.youtube.com/watch?v=QsYGlZkevEg\", add_video_info=True\n",
+    ")\n",
+    "loader.load()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7c635f42",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

pyproject.toml

@@ -0,0 +1,28 @@
+[project]
+name = "aiagents"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "bert-extractive-summarizer>=0.10.1",
+    "dotenv>=0.9.9",
+    "faiss-cpu>=1.13.2",
+    "fastapi>=0.135.1",
+    "keybert>=0.9.0",
+    "langchain>=1.2.10",
+    "langchain-aws>=1.3.1",
+    "langchain-chroma>=1.1.0",
+    "langchain-community>=0.4.1",
+    "langchain-core>=1.2.17",
+    "langchain-google-genai>=4.2.1",
+    "langchain-huggingface>=1.2.1",
+    "langchain-ollama>=1.0.1",
+    "langgraph>=1.0.10",
+    "pillow>=12.1.1",
+    "python-multipart>=0.0.22",
+    "sentence-transformers>=5.2.3",
+    "transformers>=5.3.0",
+    "unstructured>=0.21.5",
+    "youtube-transcript-api>=1.2.4",
+]

requirements.txt

@@ -0,0 +1 @@
+-e .

results/Attention is All You Need Paper Explained.md

@@ -0,0 +1,53 @@
+# Attention is All You Need Paper Explained
+## Introduction to Attention is All You Need
+The concept of attention in deep learning refers to the ability of a model to focus on specific parts of the input data that are relevant for the task at hand. 
+* Introduce the concept of attention in deep learning: Attention allows models to selectively concentrate on certain inputs or features, improving performance and efficiency.
+* Explain the limitations of traditional sequence-to-sequence models: Traditional sequence-to-sequence models rely on recurrent neural networks (RNNs) or long short-term memory (LSTM) networks, which can be limited by their sequential processing and fixed-length context.
+* Highlight the key contributions of the Attention is All You Need paper: The Attention is All You Need paper introduced a novel architecture that relies entirely on self-attention mechanisms, eliminating the need for RNNs and LSTMs, and achieving state-of-the-art results in machine translation tasks.
+![attention mechanism](../images/attention_mechanism.png)
+## The Transformer Model Architecture
+The Transformer model, introduced in the "Attention is All You Need" paper, revolutionized the field of natural language processing. At its core, the Transformer model consists of an encoder-decoder structure. 
+* The encoder takes in a sequence of tokens, such as words or characters, and generates a continuous representation of the input sequence.
+* The decoder then uses this representation to generate the output sequence, one token at a time.
+
+Self-attention mechanisms play a crucial role in the Transformer model, allowing it to weigh the importance of different tokens in the input sequence relative to each other. This is particularly useful for tasks such as machine translation, where the context of a word can greatly affect its translation. 
+
+The Transformer model also relies on positional encoding to preserve the order of the input sequence. Since the self-attention mechanism is permutation-invariant, the model would not be able to distinguish between different token orders without some form of positional information. 
+Positional encoding adds a fixed vector to each token's representation, based on its position in the sequence, allowing the model to capture sequential relationships between tokens. 
+This combination of self-attention and positional encoding enables the Transformer model to effectively process sequential data, making it a powerful tool for a wide range of NLP tasks.
+![transformer architecture](../images/transformer_architecture.png)
+## Applying the Transformer Model to Real-World Examples
+The Transformer model, introduced in the "Attention is All You Need" paper, has been widely adopted in various NLP tasks. To apply this model to real-world examples, it's essential to understand its implementation and applications.
+* A minimal code sketch of a Transformer model implementation can be represented as follows:
+```python
+import torch
+import torch.nn as nn
+import torch.optim as optim
+
+class TransformerModel(nn.Module):
+    def __init__(self):
+        super(TransformerModel, self).__init__()
+        self.encoder = nn.TransformerEncoderLayer(d_model=512, nhead=8)
+        self.decoder = nn.TransformerDecoderLayer(d_model=512, nhead=8)
+
+    def forward(self, src, tgt):
+        encoder_output = self.encoder(src)
+        decoder_output = self.decoder(tgt, encoder_output)
+        return decoder_output
+```
+* The Transformer model has been highly effective in machine translation tasks, allowing for parallelization of the decoding process and improving overall translation quality.
+* The Transformer model can also be applied to other NLP tasks, such as text classification, sentiment analysis, and question answering, by modifying the model architecture and training objectives to suit the specific task requirements.
+![transformer application](../images/transformer_application.png)
+## Common Mistakes and Challenges
+When implementing the Transformer model, several common pitfalls can hinder its performance. 
+* Proper hyperparameter tuning is crucial, as it directly affects the model's ability to learn and generalize.
+* Training large Transformer models can be challenging due to their complexity and computational requirements, often leading to issues like overfitting or slow training times.
+* Careful evaluation metrics are necessary to accurately assess the model's performance, as misleading metrics can lead to suboptimal results or incorrect conclusions about the model's effectiveness. 
+By being aware of these potential issues, developers can take steps to mitigate them and ensure successful implementation of the Transformer model.
+![common mistakes](../images/common_mistakes.png)
+## Conclusion
+The Attention is All You Need paper made significant contributions to the field of NLP, introducing a novel architecture that relies entirely on self-attention mechanisms. 
+* The main contributions of the paper include the proposal of a transformer model that replaces traditional recurrent neural network (RNN) and convolutional neural network (CNN) architectures.
+* The paper's impact on NLP has been substantial, enabling state-of-the-art results in various tasks such as machine translation and text generation.
+* Future directions for research and application include exploring the use of attention mechanisms in other areas of NLP, such as question answering and text summarization, and applying the transformer model to other domains like computer vision.
+![conclusion](../images/conclusion.png)

results/Introduction to Machine Learning.md

@@ -0,0 +1,75 @@
+# Introduction to Machine Learning
+
+![machine learning overview](../images/machine_learning_overview.png)
+
+## What is Machine Learning?
+Machine learning is a subset of artificial intelligence that involves training algorithms to learn from data and make predictions or decisions without being explicitly programmed. 
+* Define machine learning: It is a field of study that focuses on the use of algorithms and statistical models to enable machines to perform a specific task.
+* Explain types of machine learning: There are several types, including supervised, unsupervised, and reinforcement learning, each with its own strengths and weaknesses.
+* Provide examples of machine learning applications: Examples include image recognition, natural language processing, and predictive analytics, which are used in various industries such as healthcare, finance, and transportation. 
+Machine learning has numerous applications and is a key driver of innovation in many fields, with its performance and edge cases being crucial considerations for developers.
+
+![ml workflow diagram](../images/ml_workflow_diagram.png)
+
+## Machine Learning Workflow
+The machine learning workflow is a series of steps that help developers build and deploy effective machine learning models. To start a machine learning project, several key steps are involved:
+* Data collection: This is the initial step where data relevant to the problem is gathered. The quality and quantity of the data collected have a significant impact on the performance of the model.
+* Data preprocessing: After collecting the data, it needs to be cleaned and preprocessed to ensure it is in a suitable format for the model. This step involves handling missing values, removing duplicates, and scaling the data.
+* Model selection: With the preprocessed data, the next step is to choose a suitable machine learning algorithm. The choice of model depends on the type of problem, the size of the dataset, and the desired outcome.
+* Model training: Once the model is selected, it is trained using the preprocessed data. The goal of this step is to find the optimal parameters that result in the best performance.
+* Model evaluation: After training the model, its performance is evaluated using various metrics such as accuracy, precision, and recall. This step helps to identify the strengths and weaknesses of the model and provides insights for further improvement. 
+Considering edge cases and performance considerations, developers should be aware of potential issues that may arise during these steps, such as overfitting or underfitting, and take necessary measures to address them. By following these basic steps and being mindful of potential pitfalls, developers can build effective machine learning models that deliver reliable results.
+
+![ml common mistakes](../images/ml_common_mistakes.png)
+
+## Common Mistakes in Machine Learning
+When working on machine learning projects, it's essential to be aware of common mistakes that can hinder performance and accuracy. These mistakes can be easily avoided with a good understanding of the underlying concepts. 
+* Overfitting occurs when a model is too complex and learns the training data too well, resulting in poor performance on new, unseen data.
+* Underfitting happens when a model is too simple and fails to capture the underlying patterns in the training data, leading to subpar performance.
+* Data leakage is another common issue, where information from the test data is inadvertently used during training, causing the model to perform unrealistically well on the test set, but poorly on real-world data. 
+By recognizing these potential pitfalls, developers can take steps to mitigate them and develop more robust and accurate machine learning models.
+
+![ml model example](../images/ml_model_example.png)
+
+## Machine Learning Example
+To implement a simple machine learning model, several key steps must be followed. The process begins with selecting a suitable dataset. 
+* Choose a dataset: This is a crucial step as the quality and relevance of the data will directly impact the performance of the model. A dataset should be chosen based on the problem you are trying to solve, and it should be large enough to train the model effectively.
+
+Once the dataset is chosen, the next step is to select a suitable model. 
+* Select a model: There are various machine learning models available, including linear regression, decision trees, and neural networks. The choice of model depends on the type of problem you are trying to solve and the characteristics of your dataset.
+
+After selecting the model, the next step is to train it. 
+* Train the model: Training a model involves feeding it with the chosen dataset and adjusting its parameters to minimize the error between predicted and actual outputs. This can be done using various algorithms, such as gradient descent.
+
+Finally, the trained model needs to be evaluated to determine its performance. 
+* Evaluate the model: Evaluation involves testing the model on a separate dataset to determine its accuracy and reliability. This can be done using metrics such as mean squared error or accuracy score.
+
+Here's a minimal code sketch in Python to illustrate the process:
+```python
+from sklearn import datasets
+from sklearn.model_selection import train_test_split
+from sklearn.linear_model import LinearRegression
+from sklearn.metrics import mean_squared_error
+
+# Load the dataset
+dataset = datasets.load_diabetes()
+
+# Split the dataset into training and testing sets
+X_train, X_test, y_train, y_test = train_test_split(dataset.data, dataset.target, test_size=0.2, random_state=42)
+
+# Create and train a linear regression model
+model = LinearRegression()
+model.fit(X_train, y_train)
+
+# Make predictions and evaluate the model
+y_pred = model.predict(X_test)
+mse = mean_squared_error(y_test, y_pred)
+print(f"Mean squared error: {mse}")
+```
+This code sketch demonstrates how to choose a dataset, select a model, train the model, and evaluate its performance. It also highlights the importance of considering edge cases and performance considerations when implementing machine learning models.
+
+## Conclusion
+In this introduction to machine learning, key points include understanding the basics of machine learning and its applications. 
+* Summarize key points: Machine learning is a subset of artificial intelligence that involves training algorithms to make predictions or decisions based on data.
+* Provide resources for further learning: For developers looking to dive deeper, online courses and tutorials are available to explore machine learning concepts and implementation. 
+Machine learning has various applications, and its performance considerations are crucial for optimal results.

setup_folder.py

@@ -0,0 +1,71 @@
+import os
+
+PROJECT_NAME = "."
+
+folders = [
+    "src/ingestion",
+    "src/chunking",
+    "src/embeddings",
+    "src/vectorstore",
+    "src/retrievers",
+    "src/chains",
+    "src/prompts",
+    "src/llm",
+    "src/evaluation",
+    "src/utils",
+    "src/config",
+
+    "api",
+    "data/raw",
+    "data/processed",
+    "vector_db",
+    "experiments",
+    "monitoring",
+    "notebooks",
+    "tests",
+
+    "deployment",
+    "configs"
+]
+
+files = {
+    "src/ingestion/ingest_pipeline.py": "",
+    "src/chunking/text_chunker.py": "",
+    "src/embeddings/embedding_model.py": "",
+    "src/vectorstore/vector_db.py": "",
+    "src/retrievers/retriever.py": "",
+    "src/chains/rag_chain.py": "",
+    "src/prompts/prompt_templates.py": "",
+    "src/llm/llm_loader.py": "",
+    "src/evaluation/rag_metrics.py": "",
+    "src/utils/helpers.py": "",
+    "src/config/settings.py": "",
+
+    "api/routes.py": "",
+    "api/main.py": "",
+
+    "deployment/Dockerfile": "",
+    "deployment/docker-compose.yml": "",
+
+    "configs/rag_config.yaml": "",
+
+    ".env": "",
+    "requirements.txt": "",
+    "README.md": ""
+}
+
+# Create project root
+os.makedirs(PROJECT_NAME, exist_ok=True)
+
+# Create folders
+for folder in folders:
+    path = os.path.join(PROJECT_NAME, folder)
+    os.makedirs(path, exist_ok=True)
+
+# Create files
+for file_path, content in files.items():
+    full_path = os.path.join(PROJECT_NAME, file_path)
+    with open(full_path, "w") as f:
+        f.write(content)
+
+print(f"✅ RAG LLMOps project '{PROJECT_NAME}' created successfully!")

src/Blog/components/image_generation.py

@@ -0,0 +1,16 @@
+import os
+from huggingface_hub import InferenceClient
+
+class ImageGeneration:
+    def __init__(self):
+        self.client = InferenceClient(
+        provider="nscale",
+        api_key=os.environ["HF_TOKEN"],
+    )
+        
+    async def generateImage(self,prompt:str):
+        image = self.client.text_to_image(
+            prompt,
+            model="stabilityai/stable-diffusion-xl-base-1.0",
+        )
+        return image

src/Blog/components/taivily_search.py

@@ -0,0 +1,35 @@
+import logging
+import sys
+from typing import List
+from langchain_community.tools.tavily_search import TavilySearchResults
+from exception import MyException
+from utils.asyncHandler import asyncHandler
+
+
+class Taivily_search:
+    def __init__(self):
+        pass
+
+    @asyncHandler
+    async def _tavily_search(self, query: str, max_results: int = 5) -> List[dict]:
+        logging.info(f"Using Tavily to search for: {query}")
+        try:
+            tool = TavilySearchResults(max_results=max_results)
+            results = await tool.ainvoke({"query": query})
+
+            normalized: List[dict] = []
+            for r in results or []:
+                normalized.append(
+                    {
+                        "title": r.get("title") or "",
+                        "url": r.get("url") or "",
+                        "snippet": r.get("content") or r.get("snippet") or "",
+                        "published_at": r.get("published_date") or r.get("published_at"),
+                        "source": r.get("source"),
+                    }
+                )
+            logging.debug(f"Tavily search returned {len(normalized)} results")
+            return normalized
+        except Exception as e:
+            logging.error(f"Error in Tavily_search: {str(e)}")
+            raise MyException(e, sys)

src/Blog/constants/__init__.py

@@ -0,0 +1,2 @@
+import os
+FOLDER_PATH_TO_SAVE_MD="results"

src/Blog/graph/Compile_graph.py

@@ -0,0 +1,74 @@
+import logging
+import asyncio
+from dotenv import load_dotenv
+from langgraph.graph import StateGraph,START,END
+from src.Blog.models.State_model import State
+from src.Blog.graph.nodes.router_node import router_node,route_next
+from src.Blog.graph.nodes.reducer_node import reducer_node
+from src.Blog.graph.nodes.search_node import research_node
+from src.Blog.graph.nodes.orchaster_node import orchestrator_node
+from src.Blog.graph.nodes.worker_node import worker_node
+from src.Blog.graph.nodes.fanout_node import fanout
+load_dotenv()
+
+g = StateGraph(State)
+g.add_node("router", router_node)
+g.add_node("research", research_node)
+g.add_node("orchestrator", orchestrator_node)
+g.add_node("worker", worker_node)
+g.add_node("reducer", reducer_node)
+
+g.add_edge(START, "router")
+g.add_conditional_edges("router", route_next, {"research": "research", "orchestrator": "orchestrator"})
+g.add_edge("research", "orchestrator")
+
+g.add_conditional_edges("orchestrator", fanout, ["worker"])
+g.add_edge("worker", "reducer")
+g.add_edge("reducer", END)
+
+app = g.compile()
+
+
+png_data = app.get_graph().draw_mermaid_png()
+with open("graph.png", "wb") as f:
+    f.write(png_data)
+async def run(topic: str):
+    logging.info(f"Starting blog generation for topic: {topic}")
+    try:
+        # out = await app.ainvoke(
+        #     {
+        #         "topic": topic,
+        #         "mode": "",
+        #         "needs_research": False,
+        #         "queries": [],
+        #         "evidence": [],
+        #         "plan": None,
+        #         "sections": [],
+        #         "final": "",
+        #     }
+        # )
+        async for step in app.astream(
+            {
+                "topic": topic,
+                "mode": "",
+                "needs_research": False,
+                "queries": [],
+                "evidence": [],
+                "plan": None,
+                "sections": [],
+                "final": "",
+            },
+            stream_mode="values"   # important
+        ):
+            # print("Current Step:", step)
+            yield step
+        logging.info("Blog generation completed successfully")
+        return
+    except Exception as e:
+        logging.error(f"Error during graph execution: {str(e)}")
+        raise
+
+if __name__ == "__main__":
+    from logger import *
+    out=asyncio.run(run("State of Multimodal LLMs in 2026"))
+    print(out)

src/Blog/graph/graphs/reducer_subgraph.py

@@ -0,0 +1,22 @@
+import logging
+from langgraph.graph import StateGraph,START,END
+from src.Blog.models.ImageSpec_model import State
+from src.Blog.graph.nodes.reducer_sub_node import reducer_sub_image,reducer_sub_llm,merge_images_and_md
+app=StateGraph(State)
+
+
+app.add_node("reducer_sub_llm",reducer_sub_llm)
+app.add_node("reducer_sub_image",reducer_sub_image)
+app.add_node("merge_images_and_md",merge_images_and_md)
+
+app.add_edge(START,"reducer_sub_llm")
+app.add_edge("reducer_sub_llm","reducer_sub_image")
+app.add_edge("reducer_sub_image","merge_images_and_md")
+app.add_edge("merge_images_and_md",END)
+
+
+
+
+app=app.compile()
+logging.info("Reducer subgraph compiled successfully")
+

src/Blog/graph/nodes/fanout_node.py

@@ -0,0 +1,23 @@
+import logging
+from src.Blog.models.State_model import State
+from langgraph.types import Send
+
+
+def fanout(state: State):
+    logging.info("Entering fanout")
+    tasks = state["plan"].tasks
+    logging.debug(f"Fanning out {len(tasks)} tasks")
+    
+    return [
+        Send(
+            "worker",
+            {
+                "task": task.model_dump(),
+                "topic": state["topic"],
+                "mode": state["mode"],
+                "plan": state["plan"].model_dump(),
+                "evidence": [e.model_dump() for e in state.get("evidence", [])],
+            },
+        )
+        for task in tasks
+    ]

src/Blog/graph/nodes/orchaster_node.py

@@ -0,0 +1,40 @@
+import logging
+import sys
+from src.Blog.models.State_model import State
+from src.Blog.llm import llm
+from exception import MyException
+from src.Blog.models.Plan_model import Plan
+from src.Blog.prompts import ORCH_SYSTEM
+from langchain_core.messages import SystemMessage, HumanMessage
+from utils.asyncHandler import asyncHandler
+
+
+@asyncHandler
+async def orchestrator_node(state: State) -> dict:
+    logging.info("Entering orchestrator_node")
+    try:
+        planner = llm.with_structured_output(Plan)
+
+        evidence = state.get("evidence", [])
+        mode = state.get("mode", "closed_book")
+        logging.debug(f"Mode: {mode}, Evidence count: {len(evidence)}")
+
+        plan = await planner.ainvoke(
+            [
+                SystemMessage(content=ORCH_SYSTEM),
+                HumanMessage(
+                    content=(
+                        f"Topic: {state['topic']}\n"
+                        f"Mode: {mode}\n\n"
+                        f"Evidence (ONLY use for fresh claims; may be empty):\n"
+                        f"{[e.model_dump() for e in evidence][:16]}"
+                    )
+                ),
+            ]
+        )
+
+        logging.info(f"Orchestrator plan created: {plan.blog_title} with {len(plan.tasks)} tasks.")
+        return {"plan": plan}
+    except Exception as e:
+        logging.error(f"Error in orchestrator_node: {str(e)}")
+        raise MyException(e, sys)

src/Blog/graph/nodes/reducer_node.py

@@ -0,0 +1,36 @@
+import logging
+import sys
+from pathlib import Path
+from src.Blog.models.State_model import State
+from exception import MyException
+from utils.asyncHandler import asyncHandler
+from src.Blog.constants import FOLDER_PATH_TO_SAVE_MD
+import os
+from src.Blog.graph.graphs.reducer_subgraph import app
+@asyncHandler
+async def reducer_node(state: State) -> dict:
+    logging.info("Entering reducer_node")
+    try:
+        plan = state["plan"]
+
+        ordered_sections = [md for _, md in sorted(state["sections"], key=lambda x: x[0])]
+        body = "\n\n".join(ordered_sections).strip()
+        final_md = f"# {plan.blog_title}\n\n{body}\n"
+
+        filename = f"{plan.blog_title}.md"
+        logging.debug(f"Writing final blog to {filename}")
+
+        logging.info("Starting image generation and merging via subgraph")
+        red_f_ob=await app.ainvoke({"prompt_markdown":final_md})
+        final_md=red_f_ob["final_md"]
+        
+        logging.debug(f"Final MD size after merging: {len(final_md)} characters")
+        os.makedirs(FOLDER_PATH_TO_SAVE_MD,exist_ok=True)
+        file_path=os.path.join(FOLDER_PATH_TO_SAVE_MD,filename)
+        Path(file_path).write_text(final_md, encoding="utf-8")
+
+        logging.info(f"Reducer node completed successfully, blog saved to {file_path}")
+        return {"final": final_md}
+    except Exception as e:
+        logging.error(f"Error in reducer_node: {str(e)}")
+        raise MyException(e, sys)

src/Blog/graph/nodes/reducer_sub_node.py

@@ -0,0 +1,66 @@
+import logging
+import os
+from src.Blog.models.ImageSpec_model import State, GlobalImagePlan
+from utils.asyncHandler import asyncHandler
+from langchain.messages import SystemMessage, HumanMessage
+from src.Blog.prompts import IMAGE_PLACEHOLDER_GENERATION
+from exception import MyException
+import sys
+from src.Blog.components.image_generation import ImageGeneration
+from src.Blog.llm import image_planner_llm
+
+
+@asyncHandler
+async def reducer_sub_llm(state: State) -> State:
+    logging.info("Calling Claude Haiku for image placeholder planning (structured output)")
+    output = await image_planner_llm.with_structured_output(GlobalImagePlan).ainvoke(
+        [
+            SystemMessage(content=IMAGE_PLACEHOLDER_GENERATION),
+            HumanMessage(content=state["prompt_markdown"]),
+        ]
+    )
+    if not output:
+        logging.error("LLM failed to return a valid image placeholder plan (output is None)")
+        raise MyException("Failed to generate image placeholder plan from LLM", sys)
+
+    state["output"] = output
+    logging.info("Successfully generated image placeholder plan")
+    return state
+
+
+@asyncHandler
+async def reducer_sub_image(state: State) -> State:
+    output = state['output']
+    image_generator = ImageGeneration()
+    if not output:
+        raise MyException("output from reducer_sub not found", sys)
+
+    os.makedirs("images", exist_ok=True)
+
+    logging.info(f"Starting image generation for {len(output.images)} images")
+    for image_con in output.images:
+        logging.debug(f"Generating image: {image_con.filename} with prompt: {image_con.prompt[:50]}...")
+        image = await image_generator.generateImage(prompt=image_con.prompt)
+        image.save(image_con.filename)
+    logging.info("All images generated successfully")
+    return state
+
+
+@asyncHandler
+async def merge_images_and_md(state: State) -> State:
+    output = state["output"]
+    md = output.md_with_placeholders
+
+    logging.info(f"Merging {len(output.images)} images into Markdown")
+    for im in output.images:
+        alt_text = (
+            im.filename.split("/")[-1]
+            .replace(".png", "")
+            .replace("_", " ")
+        )
+        md_image_tag = f"![{alt_text}](../{im.filename})"
+        md = md.replace(im.placeholder, md_image_tag)
+
+    state["final_md"] = md
+    logging.info("Markdown merging completed")
+    return state

src/Blog/graph/nodes/router_node.py

@@ -0,0 +1,102 @@
+import logging
+import sys
+import json
+import re
+from src.Blog.models.RouterDecision_model import RouterDecision
+from langchain_core.messages import SystemMessage, HumanMessage
+from src.Blog.models.State_model import State
+from src.Blog.llm import llm
+from src.Blog.prompts import ROUTER_SYSTEM
+from exception import MyException
+from utils.asyncHandler import asyncHandler
+
+
+@asyncHandler
+async def router_node(state: State):
+    logging.info("Entering router_node")
+    topic = state['topic']
+    logging.debug(f"Topic: {topic}")
+    
+    try:
+        try:
+            runnable = llm.with_structured_output(RouterDecision)
+            decision = await runnable.ainvoke(
+                [
+                    SystemMessage(content=ROUTER_SYSTEM),
+                    HumanMessage(content=f"Topic: {topic}")
+                ]
+            )
+            if decision:
+                logging.info(f"Router decision (structured): needs_research={decision.needs_research}, mode={decision.mode}")
+                return {
+                    "needs_research": decision.needs_research,
+                    "mode": decision.mode,
+                    "queries": decision.queries,
+                }
+        except Exception as e:
+            logging.warning(f"Structured output failed: {str(e)}. Attempting manual parse.")
+
+        raw_response = await llm.ainvoke(
+            [
+                SystemMessage(content=ROUTER_SYSTEM + "\n\nCRITICAL: You MUST return a valid JSON object. Do not include any text before or after the JSON."),
+                HumanMessage(content=f"Topic: {topic}")
+            ]
+        )
+        content = raw_response.content
+        logging.debug(f"Raw LLM content for fallback: {content}")
+
+        json_str = ""
+        markdown_match = re.search(r'```json\s*(.*?)\s*```', content, re.DOTALL)
+        if markdown_match:
+            json_str = markdown_match.group(1)
+        else:
+            start = content.find('{')
+            end = content.rfind('}')
+            if start != -1 and end != -1:
+                json_str = content[start:end+1]
+        
+        if json_str:
+            try:
+                data = json.loads(json_str)
+            except json.JSONDecodeError:
+                # Progressive truncation fallback
+                success = False
+                temp_str = json_str
+                while '}' in temp_str:
+                    try:
+                        data = json.loads(temp_str)
+                        success = True
+                        break
+                    except json.JSONDecodeError:
+                        last_brace = temp_str.rfind('}')
+                        if last_brace == -1: break
+                        temp_str = temp_str[:last_brace]
+                
+                if not success:
+                    raise ValueError("Failed to parse JSON even after structural truncation")
+
+            needs_res = str(data.get("needs_research", "")).lower() in ["true", "1", "yes"]
+            
+            decision = RouterDecision(
+                needs_research=needs_res,
+                mode=data.get("mode", "open_book"),
+                queries=data.get("queries", [])
+            )
+            logging.info(f"Router decision (manual): needs_research={decision.needs_research}, mode={decision.mode}")
+            return {
+                "needs_research": decision.needs_research,
+                "mode": decision.mode,
+                "queries": decision.queries,
+            }
+
+        logging.error("Failed to extract JSON from LLM response")
+        raise ValueError("LLM failed to return a valid RouterDecision. Please check prompts or model output.")
+
+    except Exception as e:
+        logging.error(f"Error in router_node: {str(e)}")
+        raise
+
+def route_next(state: State) -> str:
+    needs_research = state.get("needs_research", False)
+    logging.info(f"Routing next based on research need: {needs_research}")
+    return "research" if needs_research else "orchestrator"

src/Blog/graph/nodes/search_node.py

@@ -0,0 +1,61 @@
+import logging
+import sys
+import asyncio
+import json
+import re
+from typing import List
+from src.Blog.models.State_model import State
+from src.Blog.components.taivily_search import Taivily_search
+from exception import MyException
+from src.Blog.llm import llm
+from langchain_core.messages import SystemMessage, HumanMessage
+from src.Blog.prompts import RESEARCH_SYSTEM
+from src.Blog.models.Evidence_model import EvidencePack, EvidenceItem
+from utils.asyncHandler import asyncHandler
+
+
+@asyncHandler
+async def research_node(state: State) -> dict:
+    logging.info("Entering research_node")
+    try:
+        taivily_search = Taivily_search()
+        queries = state.get("queries", []) or []
+        logging.debug(f"Queries for research: {queries}")
+        max_results = 6
+
+        raw_results: List[dict] = []
+
+        for q in queries:
+            logging.debug(f"Queuing tavily search for query: {q}")
+            raw_results.extend(await taivily_search._tavily_search(q, max_results=max_results))
+        
+        if not raw_results:
+            logging.warning("No raw results found during research")
+
+            return {"evidence": []}
+
+       
+
+        logging.info(f"Extracted {len(raw_results)} raw results. Processing with LLM for EvidencePack.")
+        
+        try:
+            extractor = llm.with_structured_output(EvidencePack)
+            pack = await extractor.ainvoke(
+                [
+                    SystemMessage(content=RESEARCH_SYSTEM),
+                    HumanMessage(content=f"Raw results:\n{raw_results}"),
+                ]
+            )
+            if pack:
+                logging.info(f"EvidencePack extracted (structured). Evidence count: {len(pack.evidence)}")
+                dedup={}
+                for e in pack.evidence:
+                    dedup[e.url]=e
+                return {"evidence": list(dedup.values())}
+        except Exception as e:
+            logging.warning(f"Structured output failed in research_node: {str(e)}. Attempting manual parse.")
+
+
+    except Exception as e:
+        logging.error(f"Error in research_node: {str(e)}")
+        raise MyException(e, sys)

src/Blog/graph/nodes/worker_node.py

@@ -0,0 +1,65 @@
+import logging
+import sys
+from src.Blog.models.Task_models import Task
+from src.Blog.models.Plan_model import Plan
+from src.Blog.models.Evidence_model import EvidenceItem
+from src.Blog.llm import llm
+from langchain_core.messages import SystemMessage, HumanMessage
+from src.Blog.prompts import WORKER_SYSTEM
+from exception import MyException
+from utils.asyncHandler import asyncHandler
+
+
+@asyncHandler
+async def worker_node(payload: dict) -> dict:
+    logging.info("Entering worker_node")
+    try:
+        task = Task(**payload["task"])
+        plan = Plan(**payload["plan"])
+        evidence = [EvidenceItem(**e) for e in payload.get("evidence", [])]
+        topic = payload["topic"]
+        mode = payload.get("mode", "closed_book")
+
+        logging.debug(f"Task: {task.title}, Mode: {mode}")
+
+        bullets_text = "\n- " + "\n- ".join(task.bullets)
+
+        evidence_text = ""
+        if evidence:
+            evidence_text = "\n".join(
+                f"- {e.title} | {e.url} | {e.published_at or 'date:unknown'}".strip()
+                for e in evidence[:20]
+            )
+
+        response = await llm.ainvoke(
+            [
+                SystemMessage(content=WORKER_SYSTEM),
+                HumanMessage(
+                    content=(
+                        f"Blog title: {plan.blog_title}\n"
+                        f"Audience: {plan.audience}\n"
+                        f"Tone: {plan.tone}\n"
+                        f"Blog kind: {plan.blog_kind}\n"
+                        f"Constraints: {plan.constraints}\n"
+                        f"Topic: {topic}\n"
+                        f"Mode: {mode}\n\n"
+                        f"Section title: {task.title}\n"
+                        f"Goal: {task.goal}\n"
+                        f"Target words: {task.target_words}\n"
+                        f"Tags: {task.tags}\n"
+                        f"requires_research: {task.requires_research}\n"
+                        f"requires_citations: {task.requires_citations}\n"
+                        f"requires_code: {task.requires_code}\n"
+                        f"Bullets:{bullets_text}\n\n"
+                        f"Evidence (ONLY use these URLs when citing):\n{evidence_text}\n"
+                    )
+                ),
+            ]
+        )
+        section_md = response.content.strip()
+
+        logging.info(f"Worker node completed task: {task.title}")
+        return {"sections": [(task.id, section_md)]}
+    except Exception as e:
+        logging.error(f"Error in worker_node: {str(e)}")
+        raise MyException(e, sys)

src/Blog/llm/__init__.py

@@ -0,0 +1,13 @@
+from langchain_aws import ChatBedrockConverse
+LLM_MODEL_ID = "us.meta.llama3-3-70b-instruct-v1:0"
+LLM_REGION = "us-east-1"
+llm = ChatBedrockConverse(
+    model_id=LLM_MODEL_ID,
+    region_name=LLM_REGION
+)
+
+
+image_planner_llm = ChatBedrockConverse(
+    model_id="us.anthropic.claude-3-5-haiku-20241022-v1:0",
+    region_name="us-east-1",
+)

src/Blog/models/Evidence_model.py

@@ -0,0 +1,13 @@
+from typing import Optional,List,Literal
+from pydantic import BaseModel,Field
+class EvidenceItem(BaseModel):
+    title: str
+    url: str
+    published_at: Optional[str] = None  # keep if Tavily provides; DO NOT rely on it
+    snippet: Optional[str] = None
+    source: Optional[str] = None
+
+
+
+class EvidencePack(BaseModel):
+    evidence: List[EvidenceItem] = Field(default_factory=list)    

src/Blog/models/ImageSpec_model.py

@@ -0,0 +1,20 @@
+from pydantic import BaseModel,Field
+from typing import List,Literal,TypedDict
+class ImageSpec(BaseModel):
+    placeholder:str=Field(...,description="e.g. [[IMAGE_1]]")
+    filename:str=Field(...,description="Save under images/, e.g. qkv_flow.png")
+    prompt:str=Field(...,description="Prompt to send to the image model")
+    size:Literal["1024x1024","1024x1536","1536x1024"]="1025x1024"
+    quality: Literal["low", "medium", "high"] = "medium"
+
+
+class GlobalImagePlan(BaseModel):
+    md_with_placeholders:str
+    images:List[ImageSpec]=Field(default_factory=list)
+
+
+
+class State(TypedDict):
+    prompt_markdown:str
+    final_md:str
+    output: GlobalImagePlan