project init

.env

@@ -0,0 +1,17 @@
+HF_TOKEN=""
+WEATHER_API_KEY="9e50616b95574a30dbc5a01579aa2b9f"
+LANGCHAIN_TRACING_V2=true
+LANGCHAIN_ENDPOINT='https://api.smith.langchain.com'
+LANGCHAIN_API_KEY='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'
+LANGCHAIN_PROJECT='Default'
+
+TWILIO_ACCOUNT_SID="ACfafc0d2d007bdf14b21bb3e14a7a7b31"
+TWILIO_AUTH_TOKEN="ed15fa98748c8c3d3d02cb54e431a187"
+TWILIO_PHONE_NUMBER="+14343375085"
+
+LIVEKIT_URL=wss://demo-wqwzjgsv.livekit.cloud
+LIVEKIT_API_KEY=APIesfzMFdhmrb6
+LIVEKIT_API_SECRET=kb7jLghH6Q3qLXxUHoYwREpYJdgX8qgAOHBDOG7q40G
+
+GROQ_API_KEY=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
+CARTESIA_API_KEY=sk_car_h3oyy6jPSJzx8KnEGJ1m5f

.gitattributes

@@ -1,35 +1,42 @@
+# ===== Archives =====
 *.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
 *.bz2 filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+
+# ===== Machine Learning / Model Files =====
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
 *.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
 *.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+
+# ===== Data / Arrays / Artifacts =====
 *.npy filter=lfs diff=lfs merge=lfs -text
 *.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
 *.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
 *.pickle filter=lfs diff=lfs merge=lfs -text
 *.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
+
+# ===== Large Binary / Misc =====
+*.bin filter=lfs diff=lfs merge=lfs -text
 *.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+
+# ===== TensorFlow / Training Logs =====
 *tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,11 @@
+# Database files (SQLite temp/journal files)
+*.db
+*.db-shm
+*.db-wal
+
+# Python bytecode
+*.pyc
+**/__pycache__/
+
+# Binary files
+*.bin

app.py

@@ -0,0 +1,95 @@
+from fastapi import FastAPI
+from fastapi.responses import StreamingResponse
+from contextlib import asynccontextmanager
+from pydantic import BaseModel
+from core.backend import AIBackend
+import uvicorn, json, os
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi import FastAPI, WebSocket, WebSocketDisconnect
+from services.stt import StreamingSTT
+from services.tts import text_to_speech_stream
+from fastapi.staticfiles import StaticFiles
+
+
+chatbot_obj = AIBackend()
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    await chatbot_obj.async_setup()
+    yield
+    if chatbot_obj.conn:
+        await chatbot_obj.conn.close()
+
+app = FastAPI(lifespan=lifespan)
+
+class UserRequest(BaseModel):
+    user_id: str
+    user_query: str
+
+@app.post("/chat")
+async def chat(request: UserRequest):
+    stream = await chatbot_obj.main(
+        user_id=request.user_id,
+        user_query=request.user_query,
+    )
+    return StreamingResponse(stream, media_type="text/event-stream")
+
+@app.websocket("/ws/chat")
+async def websocket_chat(websocket: WebSocket):
+    await websocket.accept()
+    try:
+        while True:
+            # receive frontend message
+            data = await websocket.receive_text()
+            payload = json.loads(data)
+            
+            user_id = payload["user_id"]
+            user_query = payload["user_query"]
+            
+            # stream AI response
+            stream = await chatbot_obj.main(
+                user_id=user_id,
+                user_query=user_query
+            )
+            
+            async for chunk in stream:
+                await websocket.send_text(chunk)
+            
+            # notify frontend response finished
+            await websocket.send_text("[[END]]")
+    except WebSocketDisconnect:
+        print("Client disconnected")
+
+@app.websocket("/ws/voice")
+async def voice_ws(websocket: WebSocket):
+    await websocket.accept()
+    stt = StreamingSTT()
+    try:
+        while True:
+            message = await websocket.receive()
+            # 🎤 AUDIO INPUT
+            if "bytes" in message:
+                audio_chunk = message["bytes"]
+                stt.add_audio(audio_chunk)
+                text = stt.transcribe_if_ready()
+                if not text:
+                    continue
+                await websocket.send_text(f"[STT]: {text}")
+                # 🤖 LLM STREAM
+                stream = chatbot_obj.main(
+                    user_id="voice_user",
+                    user_query=text
+                )
+                full_response = ""
+                async for token in stream:
+                    full_response += token
+                    await websocket.send_text(f"[LLM]: {token}")
+                # 🔊 TTS STREAM
+                async for audio_chunk in text_to_speech_stream(full_response):
+                    await websocket.send_bytes(audio_chunk)
+                await websocket.send_text("[END]")
+    except WebSocketDisconnect:
+        print("Voice client disconnected")
+
+if __name__ == "__main__":
+    uvicorn.run("app:app", host="127.0.0.1", port=8679, reload=True)

core/backend.py

@@ -0,0 +1,433 @@
+from langgraph.graph import StateGraph, START, END
+from typing import TypedDict, Annotated
+from langchain_core.messages import BaseMessage
+from langgraph.graph.message import add_messages
+from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
+from langchain_ollama import ChatOllama
+from langgraph.prebuilt import ToolNode, tools_condition
+from langchain_community.tools import DuckDuckGoSearchRun
+from langchain_core.tools import tool
+from langchain_core.messages import HumanMessage, AIMessage, ToolMessage, RemoveMessage, SystemMessage
+import aiosqlite, uuid, os, httpx, asyncio
+from twilio.rest import Client
+from dotenv import load_dotenv
+import json, pytz
+from datetime import datetime
+
+######################### STATE #########################
+class ChatState(TypedDict):
+    messages: Annotated[list[BaseMessage], add_messages]
+    summary: str
+
+######################### TOOLS #########################
+def get_db_path():
+    return os.path.join(os.path.dirname(__file__), "daa.db")
+
+def send_sms(to_number: str, message: str):
+    client = Client(os.getenv("TWILIO_ACCOUNT_SID"), os.getenv("TWILIO_AUTH_TOKEN"))
+    client.messages.create(
+        body=message,
+        from_=os.getenv("TWILIO_PHONE_NUMBER"),
+        to=to_number
+    )
+
+def format_bd_number(num: str) -> str:
+    num = num.strip().replace(" ", "")
+    if num.startswith("01") and len(num) == 11:
+        return "+88" + num
+    if num.startswith("8801"):
+        return "+" + num
+    return num  # already formatted or unknown
+
+@tool
+def get_bd_time() -> str:
+    """
+    Get current Bangladesh time (Asia/Dhaka) with weekday name
+    """
+    tz = pytz.timezone("Asia/Dhaka")
+    now = datetime.now(tz)
+    return now.strftime("%Y-%m-%d %H:%M:%S (%A, Bangladesh Time)")
+
+@tool
+async def search_doctor(name: str = "", category: str = "", visiting_days: str = "") -> str:
+    """
+    Search doctors by name, category, or visiting_days from SQLite database.
+    Any combination of filters is supported (OR logic for each field).
+    """
+    db_path = get_db_path()
+    query = "SELECT * FROM doctors WHERE 1=1"
+    params = []
+    conditions = []
+    
+    if name:
+        conditions.append("LOWER(doctor_name) LIKE ?")
+        params.append(f"%{name.lower()}%")
+    
+    if category:
+        conditions.append("LOWER(category) LIKE ?")
+        params.append(f"%{category.lower()}%")
+    
+    if visiting_days:
+        conditions.append("LOWER(visiting_days) LIKE ?")
+        params.append(f"%{visiting_days.lower()}%")
+    
+    if conditions:
+        query += " AND (" + " OR ".join(conditions) + ")"
+    
+    async with aiosqlite.connect(db_path) as db:
+        db.row_factory = aiosqlite.Row
+        cursor = await db.execute(query, params)
+        rows = await cursor.fetchall()
+    
+    if not rows:
+        return json.dumps({
+            "success": False,
+            "message": "No doctors found matching your search.",
+            "data": []
+        })
+    
+    return json.dumps({
+        "success": True,
+        "count": len(rows),
+        "data": [dict(r) for r in rows]
+    })
+
+@tool
+async def search_appointment_by_phone(patient_num: str) -> str:
+    """
+    Search all appointments using patient phone number.
+    """
+    db_path = get_db_path()
+    patient_num = format_bd_number(patient_num)
+    
+    async with aiosqlite.connect(db_path) as db:
+        db.row_factory = aiosqlite.Row
+        
+        cursor = await db.execute("""
+            SELECT * FROM patients
+            WHERE patient_num = ?
+            ORDER BY visiting_date ASC
+        """, (patient_num,))
+        
+        rows = await cursor.fetchall()
+    
+    if not rows:
+        return json.dumps({
+            "success": False,
+            "message": "No appointments found for this phone number.",
+            "data": []
+        })
+    
+    return json.dumps({
+        "success": True,
+        "count": len(rows),
+        "data": [dict(r) for r in rows]
+    })
+
+@tool
+async def book_appointment(doctor_id: int, patient_name: str, patient_age: str, patient_num: str, visiting_date: str) -> str:
+    """
+    Book a doctor appointment and save it to the patients table.
+    
+    Args:
+        doctor_id: Doctor's ID from search_doctor results.
+        patient_name: Full name of the patient.
+        patient_age: Age of the patient (e.g. "32").
+        patient_num: Contact phone number of the patient.
+        visiting_date: Date of visit in YYYY-MM-DD format (e.g. 2025-06-15).
+    
+    Returns a booking confirmation with the new record ID.
+    """
+    db_path = get_db_path()
+    
+    async with aiosqlite.connect(db_path) as db:
+        db.row_factory = aiosqlite.Row
+        patient_num = format_bd_number(patient_num)
+        
+        # Verify doctor exists
+        cursor = await db.execute("SELECT * FROM doctors WHERE id = ?", (doctor_id,))
+        doctor = await cursor.fetchone()
+        if not doctor:
+            return f"No doctor found with ID {doctor_id}. Please search for a doctor first."
+        
+        doctor_data = dict(doctor)
+        doctor_name = doctor_data.get("doctor_name", "Unknown")
+        doctor_category = doctor_data.get("doctor_category", "Unknown")
+        
+        # Check for conflicting booking (same doctor + same date)
+        cursor = await db.execute(
+            """SELECT id FROM patients
+            WHERE doctor_name = ? AND visiting_date = ? AND patient_num = ?""",
+            (doctor_name, visiting_date, patient_num),
+        )
+        conflict = await cursor.fetchone()
+        if conflict:
+            return (
+                f"A booking for {patient_name} with Dr. {doctor_name} "
+                f"on {visiting_date} already exists."
+            )
+        
+        # Insert into patients table
+        cursor = await db.execute(
+            """INSERT INTO patients (doctor_name, doctor_category, patient_name, patient_age, patient_num, visiting_date)
+            VALUES (?, ?, ?, ?, ?, ?)""",
+            (doctor_name, doctor_category, patient_name, patient_age, patient_num, visiting_date),
+        )
+        await db.commit()
+    
+    # Send SMS confirmation
+    sms_message = (
+        f"✅ Appointment Confirmed!\n"
+        f"Doctor     : {doctor_name}\n"
+        f"Patient    : {patient_name}\n"
+        f"Visit Date : {visiting_date}\n"
+        f"Please arrive 10 minutes early."
+    )
+    # try:
+    #     send_sms(to_number=patient_num, message=sms_message)
+    #     sms_status = "📱 SMS confirmation sent."
+    # except Exception as e:
+    #     sms_status = f"⚠️ SMS failed: {str(e)}"
+    
+    return (
+        f"✅ Appointment Booked!\n"
+        f"━━━━━━━━━━━━━━━━━━━━━━\n"
+        f"Doctor       : {doctor_name}\n"
+        f"Patient      : {patient_name}\n"
+        f"Age          : {patient_age}\n"
+        f"Date         : {visiting_date}\n"
+        f"Contact      : {patient_num}\n"
+        f"━━━━━━━━━━━━━━━━━━━━━━\n"
+        f"Please arrive 10 minutes early."
+        # f"{sms_status}"
+    )
+
+async def delete_appointment(patient_num: str, doctor_name: str) -> str:
+    """
+    Delete an appointment using patient phone number and doctor name.
+    """
+    db_path = get_db_path()
+    # normalize phone number
+    patient_num = format_bd_number(patient_num)
+    
+    async with aiosqlite.connect(db_path) as db:
+        db.row_factory = aiosqlite.Row
+        
+        # check if appointment exists first
+        cursor = await db.execute("""
+            SELECT * FROM patients
+            WHERE patient_num = ?
+            AND LOWER(doctor_name) = LOWER(?)
+        """, (patient_num, doctor_name))
+        
+        row = await cursor.fetchone()
+        if not row:
+            return json.dumps({
+                "success": False,
+                "message": "No matching appointment found to delete."
+            })
+        
+        # delete appointment
+        await db.execute("""
+            DELETE FROM patients
+            WHERE patient_num = ?
+            AND LOWER(doctor_name) = LOWER(?)
+        """, (patient_num, doctor_name))
+        
+        await db.commit()
+    
+    return json.dumps({
+        "success": True,
+        "message": f"Appointment with Dr. {doctor_name} deleted successfully."
+    })
+
+######################### MAIN AGENT CLASS #########################
+class AIBackend:
+    def __init__(self):
+        load_dotenv()
+        os.environ["LANGCHAIN_PROJECT"] = "Doctor Appointment Automation"
+        self.llm = ChatOllama(model="gemma4:e4b", streaming=True) # qwen2.5:3b, gemma4:e4b
+        self.tools = [search_doctor, book_appointment, get_bd_time, search_appointment_by_phone, delete_appointment]
+        self.tool_node = ToolNode(self.tools)
+        self.llm_with_tools = self.llm.bind_tools(self.tools)
+    
+    async def async_setup(self):
+        db_path = os.path.join(os.path.dirname(__file__), "daa.db")
+        self.conn = await aiosqlite.connect(db_path)
+        self.checkpointer = AsyncSqliteSaver(self.conn)
+        await self._create_user_table()
+        self.graph = self._build_graph()
+        self.summary_graph = self._build_summary_graph()
+    
+    async def _create_user_table(self):
+        await self.conn.execute("""
+            CREATE TABLE IF NOT EXISTS userid_threadid (
+                userId  TEXT UNIQUE NOT NULL,
+                threadId TEXT UNIQUE NOT NULL
+            )
+            """)
+        await self.conn.commit()
+    
+    ######################### SUMMARIZE NODE #########################
+    async def summarize_conversation(self, state: ChatState):
+        existing_summary = state.get("summary", "")
+        messages = state["messages"]
+        prompt = (
+                f"""
+                You are maintaining a long-term conversation memory for a chatbot.
+                
+                Existing summary:
+                {existing_summary}
+                
+                Update and extend the summary using ONLY the new conversation messages above.
+                
+                Instructions:
+                - Preserve important existing context.
+                - Add new facts, decisions, preferences, goals, issues, and ongoing tasks.
+                - Keep technical details concise but meaningful.
+                - Track unresolved problems or follow-up actions.
+                - Avoid repetition and remove outdated or redundant information when appropriate.
+                - Maintain chronological consistency.
+                - Write the summary in clear bullet points.
+                - Focus on information useful for future conversations and contextual continuity.
+                - Do NOT include casual greetings or temporary small talk unless important.
+                - Keep the summary compact but information-dense.
+                """
+                    if existing_summary
+                    else
+                    """
+                You are creating a long-term conversation memory summary for a chatbot.
+                
+                Summarize the conversation above.
+                
+                Instructions:
+                - Capture important user information, goals, preferences, projects, and decisions.
+                - Include technical issues, debugging progress, and solutions discussed.
+                - Track ongoing tasks or unresolved questions.
+                - Ignore casual greetings and low-value chatter.
+                - Write concise, structured bullet points.
+                - Keep the summary compact but highly informative for future context retention.
+                """
+                )
+        messages_for_summary = messages + [HumanMessage(content=prompt)]
+        response = await self.llm.ainvoke(messages_for_summary)
+        return {
+            "summary": response.content,
+            "messages": [RemoveMessage(id=m.id) for m in messages[:-2]],
+        }
+    
+    async def should_summarize(self, state: ChatState):
+        if len(state["messages"]) > 10:
+            return "summarize_node"
+        return "chat_node"
+    
+    ######################### CHAT NODE #########################
+    async def chat_node(self, state: ChatState):
+        summary = state.get("summary", "")
+        messages = state["messages"]
+        
+        print('#'*50)
+        print(">>>>>>>>>> CHAT NODE START <<<<<<<<<<")
+        if summary:
+            print(f"[SUMMARY]:\n{summary}\n")
+        else:
+            print("[NO SUMMARY YET]\n")
+        
+        print('$'*50)
+        print("[MESSAGES]:")
+        for m in messages:
+            role = m.__class__.__name__
+            print(f"  [{role}]: {m.content[:200]}")
+        print('$'*50,'\n')
+        
+        if summary:
+            summary_message = SystemMessage(
+                content=(
+                        "You are provided with a condensed memory of previous conversations.\n\n"
+                        f"Conversation Memory:\n{summary}\n\n"
+                        "Instructions:\n"
+                        "- Use this memory as long-term conversational context.\n"
+                        "- Maintain continuity with the user's previous discussions, projects, goals, and preferences.\n"
+                        "- Prioritize recent and relevant information when generating responses.\n"
+                        "- Do not repeat the summary unless necessary.\n"
+                        "- If new information conflicts with old memory, prefer the latest context.\n"
+                        "- Use the memory naturally to improve personalization, reasoning, and follow-up responses.\n"
+                        "- Treat unresolved issues, active projects, and pending tasks as ongoing unless stated otherwise."
+                        )
+                    )
+            messages = [summary_message] + messages
+        response = await self.llm_with_tools.ainvoke(messages)
+        print(f"Final [{response.__class__.__name__}]: {response.content[:200]}")
+        print(">>>>>>>>>> CHAT NODE END <<<<<<<<<<")
+        print('#'*50)
+        return {"messages": [response]}
+    
+    ######################### GRAPH #########################
+    def _build_graph(self):
+        g = StateGraph(ChatState)
+        g.add_node("chat_node", self.chat_node)
+        g.add_node("tools", self.tool_node)
+        
+        g.add_edge(START, "chat_node")
+        g.add_conditional_edges("chat_node", tools_condition)
+        g.add_edge("tools", "chat_node")
+        
+        return g.compile(checkpointer=self.checkpointer)
+    
+    def _build_summary_graph(self):
+        g = StateGraph(ChatState)
+        g.add_node("summarize_node", self.summarize_conversation)
+        g.add_edge(START, "summarize_node")
+        g.add_edge("summarize_node", END)
+        return g.compile(checkpointer=self.checkpointer)
+    
+    ######################### STREAMING #########################
+    async def ai_only_stream(self, initial_state: dict, config: dict):
+        async for message_chunk, metadata in self.graph.astream(initial_state, config=config, stream_mode="messages"):
+            if isinstance(message_chunk, AIMessage) and message_chunk.content:
+                yield message_chunk.content
+        
+        # Auto Summarization Execute
+        current_state = await self.graph.aget_state(config)
+        if len(current_state.values.get("messages", [])) > 10:
+            asyncio.create_task(
+                self.summary_graph.ainvoke(current_state.values, config=config)
+            )
+            print('@'*20,'Summarization Execute','@'*20)
+    
+    ######################### THREAD ID #########################
+    @staticmethod
+    def generate_thread_id() -> str:
+        return str(uuid.uuid4())
+    
+    ######################### RETRIEVE ALL THREADS #########################
+    async def retrieve_all_threads(self):
+        all_threads = set()
+        async for checkpoint in self.checkpointer.alist(None):
+            all_threads.add(checkpoint.config["configurable"]["thread_id"])
+        return list(all_threads)
+    
+    ######################### MAIN ENTRY POINT #########################
+    async def main(self, user_id: str, user_query: str):
+        async with self.conn.execute(
+            "SELECT userId, threadId FROM userid_threadid WHERE userId = ?", (user_id,)
+        ) as cursor:
+            result = await cursor.fetchone()
+        
+        if result is None:
+            thread_id = user_id + self.generate_thread_id()
+            await self.conn.execute(
+                "INSERT INTO userid_threadid (userId, threadId) VALUES (?, ?)",
+                (user_id, thread_id),
+            )
+            await self.conn.commit()
+        else:
+            thread_id = result[1]
+        
+        initial_state = {"messages": [HumanMessage(content=user_query)]}
+        config = {
+            "configurable": {"thread_id": thread_id},
+            "metadata": {"thread_id": thread_id},
+            "run_name": "chat_turn",
+        }
+        return self.ai_only_stream(initial_state, config)

frontend/index.html

@@ -0,0 +1,47 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8" />
+<meta name="viewport" content="width=device-width, initial-scale=1.0"/>
+<title>Realtime AI Voice Assistant</title>
+
+<link rel="stylesheet" href="style.css" />
+</head>
+<body>
+
+<div class="container">
+
+    <div class="topbar">
+        <h1>🎙️ AI Voice Assistant</h1>
+    </div>
+
+    <div id="chat-box"></div>
+
+    <div class="controls">
+
+        <div class="text-section">
+            <input
+                type="text"
+                id="text-input"
+                placeholder="Type your message..."
+            />
+
+            <button id="send-btn">
+                Send
+            </button>
+        </div>
+
+        <div class="voice-section">
+            <button id="mic-btn">
+                🎤 Start Voice
+            </button>
+        </div>
+
+    </div>
+
+</div>
+
+<script src="https://cdn.jsdelivr.net/npm/marked/marked.min.js"></script>
+<script src="script.js"></script>
+</body>
+</html>

frontend/script.js

@@ -0,0 +1,166 @@
+const chatBox = document.getElementById("chat-box");
+const sendBtn = document.getElementById("send-btn");
+const textInput = document.getElementById("text-input");
+const micBtn = document.getElementById("mic-btn");
+
+const userId = "walid";
+
+
+// =======================
+// CHAT WEBSOCKET
+// =======================
+
+const chatSocket = new WebSocket("ws://127.0.0.1:8679/ws/chat");
+
+chatSocket.onmessage = (event) => {
+
+    const data = event.data;
+
+    if (data === "[[END]]") {
+        return;
+    }
+
+    appendMessage(data, "ai");
+};
+
+
+sendBtn.onclick = () => {
+    sendTextMessage();
+};
+
+textInput.addEventListener("keydown", (e) => {
+    if (e.key === "Enter") {
+        sendTextMessage();
+    }
+});
+
+
+function sendTextMessage() {
+
+    const message = textInput.value.trim();
+
+    if (!message) return;
+
+    appendMessage(message, "user");
+
+    chatSocket.send(JSON.stringify({
+        user_id: userId,
+        user_query: message
+    }));
+
+    textInput.value = "";
+}
+
+
+// =======================
+// VOICE WEBSOCKET
+// =======================
+
+const voiceSocket = new WebSocket("ws://127.0.0.1:8679/ws/voice");
+
+voiceSocket.binaryType = "arraybuffer";
+
+let mediaRecorder;
+let audioChunks = [];
+let isRecording = false;
+
+voiceSocket.onmessage = async (event) => {
+
+    // TEXT MESSAGE
+    if (typeof event.data === "string") {
+
+        const text = event.data;
+
+        if (text.startsWith("[STT]:")) {
+            appendMessage("🎤 " + text.replace("[STT]:", ""), "user");
+        }
+
+        else if (text.startsWith("[LLM]:")) {
+            appendMessage(
+                text.replace("[LLM]:", ""),
+                "ai"
+            );
+        }
+
+        return;
+    }
+
+    // AUDIO MESSAGE
+    const audioBlob = new Blob([event.data], { type: "audio/mp3" });
+
+    const audioUrl = URL.createObjectURL(audioBlob);
+
+    const audio = new Audio(audioUrl);
+
+    audio.play();
+};
+
+
+micBtn.onclick = async () => {
+
+    if (!isRecording) {
+        startRecording();
+    } else {
+        stopRecording();
+    }
+};
+
+
+async function startRecording() {
+
+    const stream = await navigator.mediaDevices.getUserMedia({
+        audio: true
+    });
+
+    mediaRecorder = new MediaRecorder(stream, {
+        mimeType: "audio/webm"
+    });
+
+    mediaRecorder.start(250);
+
+    mediaRecorder.ondataavailable = async (event) => {
+
+        if (event.data.size > 0 &&
+            voiceSocket.readyState === WebSocket.OPEN) {
+
+            const arrayBuffer = await event.data.arrayBuffer();
+
+            voiceSocket.send(arrayBuffer);
+        }
+    };
+
+    isRecording = true;
+
+    micBtn.innerText = "⏹ Stop Voice";
+    micBtn.classList.add("recording");
+}
+
+
+function stopRecording() {
+
+    mediaRecorder.stop();
+
+    isRecording = false;
+
+    micBtn.innerText = "🎤 Start Voice";
+    micBtn.classList.remove("recording");
+}
+
+
+// =======================
+// UI
+// =======================
+
+function appendMessage(text, sender) {
+
+    const div = document.createElement("div");
+
+    div.classList.add("message");
+    div.classList.add(sender);
+
+    div.innerHTML = marked.parse(text);
+
+    chatBox.appendChild(div);
+
+    chatBox.scrollTop = chatBox.scrollHeight;
+}

frontend/style.css

@@ -0,0 +1,152 @@
+* {
+    margin: 0;
+    padding: 0;
+    box-sizing: border-box;
+}
+
+body {
+    background: #0f172a;
+    color: white;
+    font-family: Arial, Helvetica, sans-serif;
+    height: 100vh;
+    display: flex;
+    justify-content: center;
+    align-items: center;
+}
+
+.container {
+    width: 90%;
+    max-width: 900px;
+    height: 90vh;
+    background: #111827;
+    border-radius: 20px;
+    overflow: hidden;
+    display: flex;
+    flex-direction: column;
+}
+
+.topbar {
+    padding: 20px;
+    background: #1e293b;
+    border-bottom: 1px solid #334155;
+}
+
+.topbar h1 {
+    font-size: 24px;
+}
+
+#chat-box {
+    flex: 1;
+    overflow-y: auto;
+    padding: 20px;
+}
+
+/* .message {
+    margin-bottom: 16px;
+    padding: 12px 16px;
+    border-radius: 14px;
+    width: fit-content;
+    max-width: 80%;
+    line-height: 1.5;
+} */
+
+.user {
+    background: #2563eb;
+    margin-left: auto;
+}
+
+.ai {
+    background: #374151;
+}
+
+.controls {
+    padding: 20px;
+    border-top: 1px solid #334155;
+}
+
+.text-section {
+    display: flex;
+    gap: 10px;
+}
+
+#text-input {
+    flex: 1;
+    padding: 14px;
+    border-radius: 12px;
+    border: none;
+    outline: none;
+    background: #1e293b;
+    color: white;
+    font-size: 16px;
+}
+
+button {
+    padding: 14px 20px;
+    border: none;
+    border-radius: 12px;
+    cursor: pointer;
+    background: #2563eb;
+    color: white;
+    font-size: 16px;
+}
+
+button:hover {
+    opacity: 0.9;
+}
+
+.voice-section {
+    margin-top: 15px;
+}
+
+#mic-btn.recording {
+    background: red;
+}
+
+.message {
+    max-width: 80%;
+    padding: 12px 14px;
+    margin: 8px 0;
+    border-radius: 12px;
+
+    line-height: 1.6;
+    font-size: 15px;
+
+    word-wrap: break-word;
+    overflow-wrap: break-word;
+
+    white-space: normal;
+}
+
+.message.ai {
+    background: #2d3748;
+    color: #fff;
+    text-align: left;
+}
+
+.message.user {
+    background: #4a5568;
+    color: #fff;
+    text-align: left;
+    margin-left: auto;
+}
+
+.message ul,
+.message ol {
+    padding-left: 20px;
+    margin: 8px 0;
+}
+
+.message li {
+    margin-bottom: 6px;
+}
+
+.message p {
+    margin: 6px 0;
+}
+
+#chat-box {
+    display: flex;
+    flex-direction: column;
+    padding: 10px;
+    gap: 6px;
+}

requirements.txt

@@ -0,0 +1,35 @@
+python-dotenv
+fastapi
+uvicorn
+requests
+langchain
+langchain-chroma
+langchain-classic
+langchain-community
+langchain-core
+langchain-experimental
+langchain-google-genai
+langchain-huggingface
+langchain-mcp-adapters
+langchain-ollama
+langchain-openai
+langchain-protocol
+langchain-text-splitters
+langgraph
+langgraph-checkpoint
+langgraph-checkpoint-sqlite
+langgraph-prebuilt
+langgraph-sdk
+langsmith
+aiosqlite
+colorama
+faster-whisper
+mcp
+numpy
+ollama
+pydantic
+twilio
+uuid_utils
+uv
+uvicorn
+

services/stt.py

@@ -0,0 +1,73 @@
+# from faster_whisper import WhisperModel
+# import tempfile
+
+# model = WhisperModel("small", device="cpu", compute_type="int8")
+# class StreamingSTT:
+#     def __init__(self):
+#         self.audio_buffer = bytearray()
+    
+#     def add_audio(self, chunk: bytes):
+#         self.audio_buffer.extend(chunk)
+    
+#     def transcribe_if_ready(self):
+#         # simple chunk trigger (1.5–3 sec buffer recommended)
+#         if len(self.audio_buffer) < 48000 * 2 * 2:
+#             return None
+        
+#         with tempfile.NamedTemporaryFile(suffix=".wav", delete=True) as f:
+#             f.write(self.audio_buffer)
+#             f.flush()
+#             segments, _ = model.transcribe(f.name, language="bn", task="translate", beam_size=1)
+#             text = " ".join([s.text for s in segments])
+        
+#         self.audio_buffer.clear()
+#         return text
+
+
+
+
+from faster_whisper import WhisperModel
+import tempfile
+
+model = WhisperModel(
+    "small",
+    device="cpu",
+    compute_type="int8"
+)
+
+class StreamingSTT:
+
+    def __init__(self):
+        self.audio_buffer = bytearray()
+
+    def add_audio(self, chunk: bytes):
+        self.audio_buffer.extend(chunk)
+
+    def transcribe_if_ready(self):
+        # wait enough audio
+        if len(self.audio_buffer) < 50000:
+            return None
+
+        # SAVE AS WEBM
+        with tempfile.NamedTemporaryFile(
+            suffix=".webm",
+            delete=True
+        ) as f:
+
+            f.write(self.audio_buffer)
+            f.flush()
+
+            segments, _ = model.transcribe(
+                f.name,
+                language="bn",
+                beam_size=1
+            )
+
+            text = " ".join(
+                [segment.text for segment in segments]
+            )
+            
+
+        self.audio_buffer.clear()
+
+        return text.strip()

services/tts.py

@@ -0,0 +1,12 @@
+import edge_tts
+import asyncio
+import tempfile
+
+VOICE = "en-US-AriaNeural"
+
+async def text_to_speech_stream(text: str):
+    communicate = edge_tts.Communicate(text, VOICE)
+
+    async for chunk in communicate.stream():
+        if chunk["type"] == "audio":
+            yield chunk["data"]