add something

.dockerignore

@@ -0,0 +1,94 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+env/
+venv/
+ENV/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Environment
+.env
+.env.local
+.env.*.local
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# Logs
+*.log
+logs/
+*.log.*
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+.tox/
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Git
+.git/
+.gitignore
+.gitattributes
+
+# Documentation
+README.md
+*.md
+
+# Test files
+test_*.py
+*_test.py
+tests/
+test_urls.txt
+
+# Agent files (large binary)
+agent
+
+# Specific log files
+agent_direct_run.log
+agent_test.log
+agent_uvicorn_new.log
+agent_uvicorn.log
+ingestion.log
+local_test.log
+retrieve.log
+test_run.log
+test_simple.log
+validate_rag.log
+
+# Scripts not needed in production
+check_collection.py
+check_detailed.py
+extract_sitemap.py
+test_local.py
+validate_rag.py
+validate.py
+retrieve.py
+main.py
+
+# Data files
+sitemap_urls.txt

.env

@@ -0,0 +1,19 @@
+# Cohere API key for embedding generation
+COHERE_API_KEY="fYDA4euHeuxeMn2FmfxGqLqWyG8PN16BjLT0N4O9"
+
+# Qdrant Cloud connection details
+QDRANT_URL="https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333"
+QDRANT_API_KEY="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3MiOiJtIn0.xIHDfcROMhK4TPiIr4Q-P5yKHaofGf41M0FmQbxvJdE"
+QDRANT_COLLECTION=book_embeddings
+
+# Deployment vercel
+DEPLOY_VERCEL_URL="https://humanoid-ai-robotics-book-1.vercel.app"     # /sitemap.xml
+
+
+# ⚠️ IMPORTANT: Replace with your own OpenAI API key with available quota
+# Current key has insufficient quota - get a new key from: https://platform.openai.com/api-keys
+# OPENAI_API_KEY="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"
+# AGENT_MODEL="gpt-4o-mini"
+
+
+# OPENROUTER_API_KEY="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"

.env.example

@@ -0,0 +1,22 @@
+# Cohere API key for embedding generation
+# Get your key from: https://dashboard.cohere.com/api-keys
+COHERE_API_KEY="fYDA4euHeuxeMn2FmfxGqLqWyG8PN16BjLT0N4O9"
+
+# Qdrant Cloud connection details
+# Get these from your Qdrant Cloud dashboard
+QDRANT_URL="https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333"
+QDRANT_API_KEY="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3MiOiJtIn0.xIHDfcROMhK4TPiIr4Q-P5yKHaofGf41M0FmQbxvJdE"
+QDRANT_COLLECTION="book_embeddings"
+
+# Deployment URL (optional - used for documentation references)
+DEPLOY_VERCEL_URL="https://humanoid-ai-robotics-book-1.vercel.app"     # /sitemap.xml
+
+
+# OpenAI API key for the agent
+# Get your key from: https://platform.openai.com/api-keys
+# ⚠️ Ensure your key has available quota (not expired or exhausted)
+# OPENAI_API_KEY="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"
+# AGENT_MODEL="gpt-4o-mini"
+
+
+# OPENROUTER_API_KEY="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"

Dockerfile

@@ -0,0 +1,39 @@
+# Use Python 3.11 slim image for smaller size
+FROM python:3.11-slim
+
+# Set working directory
+WORKDIR /app
+
+# Set environment variables
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PIP_NO_CACHE_DIR=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy requirements first for better caching
+COPY requirements.txt .
+
+# Install Python dependencies
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy application code
+COPY . .
+
+# Create necessary directories
+RUN mkdir -p /app/logs
+
+# Expose port 7860 (Hugging Face Spaces default)
+EXPOSE 7860
+
+# Health check
+HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \
+    CMD curl -f http://localhost:7860/health || exit 1
+
+# Run the FastAPI application
+CMD ["uvicorn", "api:app", "--host", "0.0.0.0", "--port", "7860"]

agent.py

@@ -0,0 +1,361 @@
+"""
+RAG Agent FastAPI Server using OpenAI Agents SDK
+
+Provides POST /chat endpoint for grounded Q&A using OpenAI Agents SDK
+and retrieval from Qdrant via Spec-2's retrieve.py module.
+"""
+
+import os
+import sys
+import uuid
+import asyncio
+from datetime import datetime
+from typing import List, Dict, Any, Optional
+
+from fastapi import FastAPI, HTTPException
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel, Field, validator
+from dotenv import load_dotenv
+
+from agents import OpenAIChatCompletionsModel
+from openai import AsyncOpenAI
+
+OPENROUTER_API_KEY = (
+    "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"
+)
+
+client = AsyncOpenAI(
+    api_key=OPENROUTER_API_KEY,
+    base_url="https://openrouter.ai/api/v1",
+)
+
+third_party_model = OpenAIChatCompletionsModel(
+    openai_client=client, model="stepfun/step-3.5-flash:free"
+)
+
+# Make backend package importable
+current_dir = os.path.dirname(os.path.abspath(__file__))
+backend_parent = os.path.dirname(current_dir)
+if backend_parent not in sys.path:
+    sys.path.insert(0, backend_parent)
+
+# Import backend modules (support both module and script execution)
+try:
+    from .config import get_config
+    from .retrieve import search as retrieve_search
+    from .logging_config import setup_logging
+except ImportError as e:
+    try:
+        from backend.config import get_config
+        from backend.retrieve import search as retrieve_search
+        from backend.logging_config import setup_logging
+    except ImportError as e2:
+        raise ImportError(f"Failed to import backend modules: {e2}")
+
+# Import OpenAI Agents SDK (must be installed separately)
+try:
+    from agents import Agent, Runner, function_tool, ModelSettings, ToolCallOutputItem
+except ImportError:
+    raise ImportError(
+        "openai-agents package required. Install: pip install openai-agents"
+    )
+
+# Load environment
+load_dotenv()
+
+# Setup logging
+logger = setup_logging("agent")
+
+# Initialize FastAPI app
+app = FastAPI(
+    title="RAG Book Chatbot API",
+    version="1.0.0",
+    description="Chatbot for humanoid robotics book using OpenAI Agents SDK",
+)
+
+# ============ Pydantic Models ============
+
+
+class ChatRequest(BaseModel):
+    question: str = Field(..., min_length=1, max_length=1000)
+
+    @validator("question")
+    def validate_question(cls, v):
+        if not v or not v.strip():
+            raise ValueError("Question cannot be empty")
+        return v.strip()
+
+
+class Source(BaseModel):
+    url: str
+    chunk_index: int
+    text_snippet: str
+
+
+class ChatResponse(BaseModel):
+    answer: str
+    sources: List[Source]
+    tokens_used: int
+    agent_trace: Optional[str] = None
+
+
+class HealthStatus(BaseModel):
+    status: str
+    qdrant: str
+    openai: str
+    timestamp: str
+
+
+# ============ Retrieval Tool ============
+
+
+@function_tool
+def retrieve_chunks(query: str, top_k: int = 5) -> List[Dict[str, Any]]:
+    """
+    Retrieve relevant book chunks from Qdrant.
+
+    Args:
+        query: User's question
+        top_k: Number of chunks to retrieve (default: 5, max: 10)
+
+    Returns:
+        List of chunks with url, chunk_index, text, score, and source_number
+    """
+    logger.info(
+        f"[Tool] retrieve_chunks called: query='{query[:100]}...', top_k={top_k}"
+    )
+
+    try:
+        import cohere
+        from qdrant_client import QdrantClient
+
+        cfg = get_config()
+        cohere_client = cohere.ClientV2(api_key=cfg["cohere_api_key"])
+        qdrant_client = QdrantClient(
+            url=cfg["qdrant_url"], api_key=cfg["qdrant_api_key"]
+        )
+        collection_name = cfg["qdrant_collection"]
+
+        results = retrieve_search(
+            query_text=query,
+            cohere_client=cohere_client,
+            qdrant_client=qdrant_client,
+            collection_name=collection_name,
+            top_k=top_k,
+        )
+
+        chunks = []
+        for i, result in enumerate(results):
+            payload = result.get("payload", {})
+            chunks.append(
+                {
+                    "url": payload.get("url", ""),
+                    "chunk_index": payload.get("chunk_index", i),
+                    "text": payload.get("text", ""),
+                    "score": result.get("score", 0.0),
+                    "source_number": i + 1,
+                }
+            )
+
+        logger.info(f"[Tool] Retrieved {len(chunks)} chunks")
+        return chunks
+
+    except Exception as e:
+        logger.error(f"[Tool] Retrieval failed: {e}", exc_info=True)
+        raise
+
+
+# ============ Agent Definition ============
+
+
+def get_agent_instructions() -> str:
+    return """You are a helpful assistant answering questions about a humanoid robotics book.
+
+IMPORTANT GROUNDING RULES:
+1. Answer ONLY using the retrieved book content provided by the retrieve_chunks tool.
+2. Do NOT use external knowledge or make up information.
+3. If the retrieved content does not contain relevant information, say "I couldn't find relevant information in the book."
+4. Always cite your sources using the format [Source 1], [Source 2], etc. Each source number corresponds to the chunk number from the tool.
+5. Be concise and accurate.
+
+Your responses should be helpful, clear, and grounded exclusively in the provided context."""
+
+
+def create_agent():
+    return Agent(
+        name="RAG Book Assistant",
+        instructions=get_agent_instructions(),
+        tools=[retrieve_chunks],
+        model=third_party_model,
+        model_settings=ModelSettings(temperature=0.7, max_tokens=500),
+    )
+
+
+_agent_instance = None
+
+
+def get_agent():
+    """Lazy singleton agent instance."""
+    global _agent_instance
+    if _agent_instance is None:
+        _agent_instance = create_agent()
+    return _agent_instance
+
+
+# ============ Health Checks ============
+
+
+def check_qdrant_health() -> str:
+    try:
+        from backend.config import get_config
+        from qdrant_client import QdrantClient
+
+        cfg = get_config()
+        client = QdrantClient(url=cfg["qdrant_url"], api_key=cfg["qdrant_api_key"])
+        client.get_collection(cfg["qdrant_collection"])
+        return "connected"
+    except Exception as e:
+        logger.warning(f"Qdrant health check failed: {e}")
+        return "disconnected"
+
+
+def check_openai_health() -> str:
+    try:
+        api_key = os.getenv("OPENAI_API_KEY")
+        if not api_key:
+            return "disconnected"
+        import openai
+
+        client = openai.OpenAI(api_key=api_key)
+        # Simple models.list call to verify API connectivity
+        client.models.list()
+        return "connected"
+    except Exception as e:
+        logger.warning(f"OpenAI health check failed: {e}")
+        return "disconnected"
+
+
+# ============ FastAPI Endpoints ============
+
+
+@app.post("/chat")
+async def chat_endpoint(request: ChatRequest):
+    request_id = str(uuid.uuid4())[:8]
+    question = request.question.strip()
+
+    logger.info(f"[{request_id}] Received chat: {question[:100]}...")
+
+    try:
+        agent = get_agent()
+
+        # Use async Runner.run (native async, no blocking)
+        result = await asyncio.wait_for(
+            Runner.run(agent, question),
+            timeout=20.0,  # Increased from 10s to accommodate full workflow
+        )
+
+        # Extract sources from tool call outputs
+        sources = []
+        if result.new_items:
+            for item in result.new_items:
+                if isinstance(item, ToolCallOutputItem):
+                    output = item.output
+                    if isinstance(output, list):
+                        for chunk in output:
+                            sources.append(
+                                Source(
+                                    url=chunk.get("url", ""),
+                                    chunk_index=chunk.get("chunk_index", 0),
+                                    text_snippet=chunk.get("text", "")[:200],
+                                )
+                            )
+
+        # Get token usage
+        tokens_used = 0
+        if result.context_wrapper and hasattr(result.context_wrapper, "usage"):
+            tokens_used = result.context_wrapper.usage.total_tokens
+
+        response = ChatResponse(
+            answer=result.final_output,
+            sources=sources,
+            tokens_used=tokens_used,
+            agent_trace=f"{request_id}: completed",
+        )
+
+        logger.info(
+            f"[{request_id}] Completed: tokens={tokens_used}, sources={len(sources)}"
+        )
+        return response
+
+    except asyncio.TimeoutError:
+        logger.error(f"[{request_id}] Timeout after 10s")
+        return JSONResponse(
+            status_code=504,
+            content={
+                "error": "timeout",
+                "message": "Request timed out. Please try again.",
+            },
+        )
+
+    except Exception as e:
+        logger.error(f"[{request_id}] Error: {e}", exc_info=True)
+        status_code = 503 if "openai" in str(e).lower() else 500
+        error_code = "openai_failed" if status_code == 503 else "internal_error"
+        return JSONResponse(
+            status_code=status_code, content={"error": error_code, "message": str(e)}
+        )
+
+
+@app.get("/health", response_model=HealthStatus)
+async def health_check():
+    request_id = str(uuid.uuid4())[:8]
+    qdrant = check_qdrant_health()
+    openai = check_openai_health()  # sync call
+    status = (
+        "healthy" if qdrant == "connected" and openai == "connected" else "degraded"
+    )
+    return HealthStatus(
+        status=status,
+        qdrant=qdrant,
+        openai=openai,
+        timestamp=datetime.utcnow().isoformat() + "Z",
+    )
+
+
+@app.on_event("startup")
+async def startup_event():
+    logger.info("=" * 60)
+    logger.info("RAG Agent FastAPI Server Starting")
+    logger.info("=" * 60)
+
+    if not os.getenv("OPENAI_API_KEY"):
+        logger.warning("OPENAI_API_KEY not set")
+
+    # Test retrieval
+    try:
+        import cohere
+        from qdrant_client import QdrantClient
+
+        cfg = get_config()
+        cohere_client = cohere.ClientV2(api_key=cfg["cohere_api_key"])
+        qdrant_client = QdrantClient(
+            url=cfg["qdrant_url"], api_key=cfg["qdrant_api_key"]
+        )
+        test_result = retrieve_search(
+            query_text="test",
+            cohere_client=cohere_client,
+            qdrant_client=qdrant_client,
+            collection_name=cfg["qdrant_collection"],
+            top_k=1,
+        )
+        logger.info(f"Retrieval test OK: {len(test_result)} results")
+    except Exception as e:
+        logger.error(f"Retrieval test failed: {e}")
+
+    logger.info("Server startup complete")
+
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(app, host="0.0.0.0", port=8000)

agent_direct_run.log

@@ -0,0 +1,26 @@
+/mnt/e/7. Low Code Agentic AI/Hackathon I/humanoid-ai-robotics-book-chatbot/backend/agent.py:63: PydanticDeprecatedSince20: Pydantic V1 style `@validator` validators are deprecated. You should migrate to Pydantic V2 style `@field_validator` validators, see the migration guide for more details. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.12/migration/
+  @validator('question')
+/mnt/e/7. Low Code Agentic AI/Hackathon I/humanoid-ai-robotics-book-chatbot/backend/agent.py:291: DeprecationWarning: 
+        on_event is deprecated, use lifespan event handlers instead.
+
+        Read more about it in the
+        [FastAPI docs for Lifespan Events](https://fastapi.tiangolo.com/advanced/events/).
+        
+  @app.on_event("startup")
+INFO:     Started server process [13533]
+INFO:     Waiting for application startup.
+05:10:47 - INFO - root - ============================================================
+05:10:47 - INFO - root - RAG Agent FastAPI Server Starting
+05:10:47 - INFO - root - ============================================================
+05:10:49 - INFO - httpx - HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333 "HTTP/1.1 200 OK"
+05:10:49 - INFO - backend.retrieve - Embedding query: 'test...' (top_k=1)
+05:10:49 - INFO - httpx - HTTP Request: POST https://api.cohere.com/v2/embed "HTTP/1.1 200 OK"
+05:10:50 - INFO - httpx - HTTP Request: POST https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333/collections/book_embeddings/points/query "HTTP/1.1 200 OK"
+05:10:50 - INFO - backend.retrieve - Search completed in 0.62s, returned 1 results
+05:10:50 - INFO - backend.retrieve - Total query time: 1.05s
+05:10:50 - INFO - root - Retrieval test OK: 1 results
+05:10:50 - INFO - root - Server startup complete
+INFO:     Application startup complete.
+ERROR:    [Errno 98] error while attempting to bind on address ('0.0.0.0', 8000): address already in use
+INFO:     Waiting for application shutdown.
+INFO:     Application shutdown complete.

agent_test.log

@@ -0,0 +1,4 @@
+Traceback (most recent call last):
+  File "/mnt/e/7. Low Code Agentic AI/Hackathon I/humanoid-ai-robotics-book-chatbot/backend/agent.py", line 22, in <module>
+    from .config import get_config
+ImportError: attempted relative import with no known parent package

agent_uvicorn.log

@@ -0,0 +1,96 @@
+INFO:     Started server process [9071]
+INFO:     Waiting for application startup.
+INFO: ============================================================
+INFO: RAG Agent FastAPI Server Starting
+INFO: ============================================================
+INFO: HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333 "HTTP/1.1 200 OK"
+INFO: Embedding query: 'test...' (top_k=1)
+INFO: HTTP Request: POST https://api.cohere.com/v2/embed "HTTP/1.1 200 OK"
+INFO: HTTP Request: POST https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333/collections/book_embeddings/points/query "HTTP/1.1 200 OK"
+INFO: Search completed in 0.61s, returned 1 results
+INFO: Total query time: 1.14s
+INFO: Retrieval test successful: 1 results
+INFO: Server startup complete
+INFO:     Application startup complete.
+INFO:     Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit)
+INFO: HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333 "HTTP/1.1 200 OK"
+INFO: HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333/collections/book_embeddings "HTTP/1.1 200 OK"
+INFO: HTTP Request: GET https://api.openai.com/v1/models "HTTP/1.1 200 OK"
+INFO:     127.0.0.1:46502 - "GET /health HTTP/1.1" 200 OK
+INFO:     127.0.0.1:58298 - "GET /docs HTTP/1.1" 200 OK
+INFO:     127.0.0.1:58298 - "GET /openapi.json HTTP/1.1" 200 OK
+INFO: [e87ec0e7] Received chat request: What is ROS 2?...
+INFO: [e87ec0e7] Starting agent execution for question: What is ROS 2?...
+INFO: Retrieving chunks for query: What is ROS 2?...
+INFO: HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333 "HTTP/1.1 200 OK"
+INFO: Embedding query: 'What is ROS 2?...' (top_k=5)
+INFO: HTTP Request: POST https://api.cohere.com/v2/embed "HTTP/1.1 200 OK"
+INFO: HTTP Request: POST https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333/collections/book_embeddings/points/query "HTTP/1.1 200 OK"
+INFO: Search completed in 0.63s, returned 5 results
+INFO: Total query time: 1.04s
+INFO: Retrieved 5 chunks
+INFO: HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 429 Too Many Requests"
+INFO: Retrying request to /chat/completions in 0.439727 seconds
+INFO: HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 429 Too Many Requests"
+INFO: Retrying request to /chat/completions in 0.899512 seconds
+INFO: HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 429 Too Many Requests"
+INFO: HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 429 Too Many Requests"
+INFO: Retrying request to /chat/completions in 0.428043 seconds
+INFO: HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 429 Too Many Requests"
+INFO: Retrying request to /chat/completions in 0.881976 seconds
+ERROR: [e87ec0e7] Request timeout after 10s
+INFO:     127.0.0.1:34080 - "POST /chat HTTP/1.1" 504 Gateway Timeout
+INFO: HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 429 Too Many Requests"
+INFO: HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 429 Too Many Requests"
+INFO: Retrying request to /chat/completions in 0.403914 seconds
+INFO: HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 429 Too Many Requests"
+INFO: Retrying request to /chat/completions in 0.922430 seconds
+INFO: HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 429 Too Many Requests"
+ERROR: [e87ec0e7] Agent failed after 12.08s: Error code: 429 - {'error': {'message': 'You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.', 'type': 'insufficient_quota', 'param': None, 'code': 'insufficient_quota'}}
+Traceback (most recent call last):
+  File "/mnt/e/7. Low Code Agentic AI/Hackathon I/humanoid-ai-robotics-book-chatbot/backend/agent.py", line 224, in run
+    response = retry_with_backoff(call_openai, max_retries=3)
+               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/mnt/e/7. Low Code Agentic AI/Hackathon I/humanoid-ai-robotics-book-chatbot/backend/utils.py", line 49, in retry_with_backoff
+    return func()
+           ^^^^^^
+  File "/mnt/e/7. Low Code Agentic AI/Hackathon I/humanoid-ai-robotics-book-chatbot/backend/agent.py", line 216, in call_openai
+    response = self.client.chat.completions.create(
+               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/m-ahmad-official/venv/lib/python3.12/site-packages/openai/_utils/_utils.py", line 286, in wrapper
+    return func(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^
+  File "/home/m-ahmad-official/venv/lib/python3.12/site-packages/openai/resources/chat/completions/completions.py", line 1192, in create
+    return self._post(
+           ^^^^^^^^^^^
+  File "/home/m-ahmad-official/venv/lib/python3.12/site-packages/openai/_base_client.py", line 1297, in post
+    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))
+                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/m-ahmad-official/venv/lib/python3.12/site-packages/openai/_base_client.py", line 1070, in request
+    raise self._make_status_error_from_response(err.response) from None
+openai.RateLimitError: Error code: 429 - {'error': {'message': 'You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.', 'type': 'insufficient_quota', 'param': None, 'code': 'insufficient_quota'}}
+INFO:     127.0.0.1:48658 - "GET /docs HTTP/1.1" 200 OK
+INFO:     127.0.0.1:48658 - "GET /openapi.json HTTP/1.1" 200 OK
+INFO: HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333 "HTTP/1.1 200 OK"
+INFO: HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333/collections/book_embeddings "HTTP/1.1 200 OK"
+INFO: HTTP Request: GET https://api.openai.com/v1/models "HTTP/1.1 200 OK"
+INFO:     127.0.0.1:42400 - "GET /health HTTP/1.1" 200 OK
+INFO: [19abeba2] Received chat request: What is ROS 2?...
+INFO: [19abeba2] Starting agent execution for question: What is ROS 2?...
+INFO: Retrieving chunks for query: What is ROS 2?...
+INFO: HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333 "HTTP/1.1 200 OK"
+INFO: Embedding query: 'What is ROS 2?...' (top_k=5)
+INFO: HTTP Request: POST https://api.cohere.com/v2/embed "HTTP/1.1 200 OK"
+INFO: HTTP Request: POST https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333/collections/book_embeddings/points/query "HTTP/1.1 200 OK"
+INFO: Search completed in 0.60s, returned 5 results
+INFO: Total query time: 1.07s
+INFO: Retrieved 5 chunks
+INFO: HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK"
+INFO: [19abeba2] Agent completed in 4.17s, tokens=1360, sources=5
+INFO:     127.0.0.1:36912 - "POST /chat HTTP/1.1" 200 OK
+INFO:     127.0.0.1:56750 - "GET /docs HTTP/1.1" 200 OK
+INFO:     127.0.0.1:56750 - "GET /openapi.json HTTP/1.1" 200 OK
+INFO:     Shutting down
+INFO:     Waiting for application shutdown.
+INFO:     Application shutdown complete.
+INFO:     Finished server process [9071]

agent_uvicorn_new.log

@@ -0,0 +1,87 @@
+INFO:     Started server process [13073]
+INFO:     Waiting for application startup.
+05:05:21 - INFO - root - ============================================================
+05:05:21 - INFO - root - RAG Agent FastAPI Server Starting
+05:05:21 - INFO - root - ============================================================
+05:05:23 - INFO - httpx - HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333 "HTTP/1.1 200 OK"
+05:05:23 - INFO - backend.retrieve - Embedding query: 'test...' (top_k=1)
+05:05:23 - INFO - httpx - HTTP Request: POST https://api.cohere.com/v2/embed "HTTP/1.1 200 OK"
+05:05:24 - INFO - httpx - HTTP Request: POST https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333/collections/book_embeddings/points/query "HTTP/1.1 200 OK"
+05:05:24 - INFO - backend.retrieve - Search completed in 0.61s, returned 1 results
+05:05:24 - INFO - backend.retrieve - Total query time: 1.14s
+05:05:24 - INFO - root - Retrieval test OK: 1 results
+05:05:24 - INFO - root - Server startup complete
+INFO:     Application startup complete.
+INFO:     Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit)
+05:05:53 - INFO - httpx - HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333 "HTTP/1.1 200 OK"
+05:05:54 - INFO - httpx - HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333/collections/book_embeddings "HTTP/1.1 200 OK"
+05:05:56 - INFO - httpx - HTTP Request: POST https://api.openai.com/v1/responses "HTTP/1.1 200 OK"
+INFO:     127.0.0.1:56784 - "GET /health HTTP/1.1" 200 OK
+05:06:01 - INFO - httpx - HTTP Request: POST https://api.openai.com/v1/traces/ingest "HTTP/1.1 204 No Content"
+05:06:46 - INFO - httpx - HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333 "HTTP/1.1 200 OK"
+05:06:46 - INFO - httpx - HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333/collections/book_embeddings "HTTP/1.1 200 OK"
+05:06:47 - INFO - httpx - HTTP Request: POST https://api.openai.com/v1/traces/ingest "HTTP/1.1 204 No Content"
+05:06:48 - INFO - httpx - HTTP Request: POST https://api.openai.com/v1/responses "HTTP/1.1 200 OK"
+INFO:     127.0.0.1:60030 - "GET /health HTTP/1.1" 200 OK
+05:06:54 - INFO - httpx - HTTP Request: POST https://api.openai.com/v1/traces/ingest "HTTP/1.1 204 No Content"
+05:07:01 - INFO - root - [c8a20a2e] Received chat: What is ROS 2?...
+05:07:02 - INFO - httpx - HTTP Request: POST https://api.openai.com/v1/responses "HTTP/1.1 200 OK"
+05:07:02 - INFO - root - [Tool] retrieve_chunks called: query='What is ROS 2?...', top_k=5
+05:07:03 - INFO - httpx - HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333 "HTTP/1.1 200 OK"
+05:07:03 - INFO - backend.retrieve - Embedding query: 'What is ROS 2?...' (top_k=5)
+05:07:03 - INFO - httpx - HTTP Request: POST https://api.cohere.com/v2/embed "HTTP/1.1 200 OK"
+05:07:04 - INFO - httpx - HTTP Request: POST https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333/collections/book_embeddings/points/query "HTTP/1.1 200 OK"
+05:07:04 - INFO - backend.retrieve - Search completed in 0.65s, returned 5 results
+05:07:04 - INFO - backend.retrieve - Total query time: 1.06s
+05:07:04 - INFO - root - [Tool] Retrieved 5 chunks
+05:07:05 - INFO - httpx - HTTP Request: POST https://api.openai.com/v1/traces/ingest "HTTP/1.1 204 No Content"
+05:07:05 - INFO - httpx - HTTP Request: POST https://api.openai.com/v1/traces/ingest "HTTP/1.1 204 No Content"
+05:07:09 - INFO - httpx - HTTP Request: POST https://api.openai.com/v1/responses "HTTP/1.1 200 OK"
+05:07:09 - INFO - root - [c8a20a2e] Completed: tokens=2006, sources=5
+INFO:     127.0.0.1:33240 - "POST /chat HTTP/1.1" 200 OK
+05:07:11 - INFO - httpx - HTTP Request: POST https://api.openai.com/v1/traces/ingest "HTTP/1.1 204 No Content"
+05:07:44 - INFO - httpx - HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333 "HTTP/1.1 200 OK"
+05:07:45 - INFO - httpx - HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333/collections/book_embeddings "HTTP/1.1 200 OK"
+05:07:46 - INFO - httpx - HTTP Request: POST https://api.openai.com/v1/responses "HTTP/1.1 200 OK"
+INFO:     127.0.0.1:54614 - "GET /health HTTP/1.1" 200 OK
+05:07:47 - INFO - httpx - HTTP Request: POST https://api.openai.com/v1/traces/ingest "HTTP/1.1 204 No Content"
+05:07:58 - INFO - root - [6ad72fe1] Received chat: What is ROS 2?...
+05:07:59 - INFO - httpx - HTTP Request: POST https://api.openai.com/v1/responses "HTTP/1.1 200 OK"
+05:07:59 - INFO - root - [Tool] retrieve_chunks called: query='What is ROS 2?...', top_k=5
+05:08:00 - INFO - httpx - HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333 "HTTP/1.1 200 OK"
+05:08:00 - INFO - backend.retrieve - Embedding query: 'What is ROS 2?...' (top_k=5)
+05:08:01 - INFO - httpx - HTTP Request: POST https://api.cohere.com/v2/embed "HTTP/1.1 200 OK"
+05:08:01 - INFO - httpx - HTTP Request: POST https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333/collections/book_embeddings/points/query "HTTP/1.1 200 OK"
+05:08:01 - INFO - backend.retrieve - Search completed in 0.65s, returned 5 results
+05:08:01 - INFO - backend.retrieve - Total query time: 1.09s
+05:08:01 - INFO - root - [Tool] Retrieved 5 chunks
+05:08:03 - INFO - httpx - HTTP Request: POST https://api.openai.com/v1/traces/ingest "HTTP/1.1 204 No Content"
+05:08:06 - INFO - httpx - HTTP Request: POST https://api.openai.com/v1/responses "HTTP/1.1 200 OK"
+05:08:06 - INFO - root - [6ad72fe1] Completed: tokens=1957, sources=5
+INFO:     127.0.0.1:51824 - "POST /chat HTTP/1.1" 200 OK
+05:08:08 - INFO - httpx - HTTP Request: POST https://api.openai.com/v1/traces/ingest "HTTP/1.1 204 No Content"
+05:11:23 - INFO - httpx - HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333 "HTTP/1.1 200 OK"
+05:11:23 - INFO - httpx - HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333/collections/book_embeddings "HTTP/1.1 200 OK"
+05:11:24 - INFO - httpx - HTTP Request: POST https://api.openai.com/v1/responses "HTTP/1.1 200 OK"
+INFO:     127.0.0.1:56880 - "GET /health HTTP/1.1" 200 OK
+05:11:27 - INFO - httpx - HTTP Request: POST https://api.openai.com/v1/traces/ingest "HTTP/1.1 204 No Content"
+05:11:36 - INFO - root - [5177fa40] Received chat: What is ROS 2?...
+05:11:37 - INFO - httpx - HTTP Request: POST https://api.openai.com/v1/traces/ingest "HTTP/1.1 204 No Content"
+05:11:39 - INFO - httpx - HTTP Request: POST https://api.openai.com/v1/responses "HTTP/1.1 200 OK"
+05:11:39 - INFO - root - [Tool] retrieve_chunks called: query='What is ROS 2?...', top_k=5
+05:11:40 - INFO - httpx - HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333 "HTTP/1.1 200 OK"
+05:11:40 - INFO - backend.retrieve - Embedding query: 'What is ROS 2?...' (top_k=5)
+05:11:40 - INFO - httpx - HTTP Request: POST https://api.cohere.com/v2/embed "HTTP/1.1 200 OK"
+05:11:38 - INFO - httpx - HTTP Request: POST https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333/collections/book_embeddings/points/query "HTTP/1.1 200 OK"
+05:11:38 - INFO - backend.retrieve - Search completed in -1.87s, returned 5 results
+05:11:38 - INFO - backend.retrieve - Total query time: -1.18s
+05:11:38 - INFO - root - [Tool] Retrieved 5 chunks
+05:11:43 - INFO - httpx - HTTP Request: POST https://api.openai.com/v1/traces/ingest "HTTP/1.1 204 No Content"
+05:11:43 - INFO - httpx - HTTP Request: POST https://api.openai.com/v1/responses "HTTP/1.1 200 OK"
+05:11:43 - INFO - root - [5177fa40] Completed: tokens=1979, sources=5
+INFO:     127.0.0.1:54308 - "POST /chat HTTP/1.1" 200 OK
+05:11:49 - INFO - httpx - HTTP Request: POST https://api.openai.com/v1/traces/ingest "HTTP/1.1 204 No Content"
+INFO:     Shutting down
+INFO:     Waiting for application shutdown.
+INFO:     Application shutdown complete.
+INFO:     Finished server process [13073]

api.py

@@ -0,0 +1,194 @@
+"""
+FastAPI wrapper for the RAG Book Assistant agent.
+
+This module provides a standalone FastAPI application that exposes the
+/chat endpoint using the agent defined in agent.py. It is separate from
+agent.py to allow independent deployment and testing.
+"""
+
+import os
+import sys
+import uuid
+import asyncio
+from datetime import datetime
+from typing import List, Dict, Any, Optional
+
+from fastapi import FastAPI, HTTPException
+from fastapi.responses import JSONResponse
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel, Field, validator
+from dotenv import load_dotenv
+
+# Make backend package importable
+current_dir = os.path.dirname(os.path.abspath(__file__))
+if current_dir not in sys.path:
+    sys.path.insert(0, current_dir)
+
+# Load environment
+load_dotenv()
+
+# Import agent components
+try:
+    from agent import get_agent, Runner
+    from agent import ToolCallOutputItem, Source as AgentSource
+except ImportError as e:
+    raise ImportError(f"Failed to import agent module: {e}")
+
+# Initialize FastAPI app
+app = FastAPI(
+    title="RAG Chatbot API",
+    version="1.0.0",
+    description="FastAPI wrapper for RAG Book Assistant"
+)
+
+# ============ CORS Configuration ============
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=[
+        "http://localhost:3000",
+        "http://127.0.0.1:3000",
+        "https://hackathon-1-humanoid-ai-robotics.vercel.app",
+        "https://*.vercel.app",
+    ],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# ============ Pydantic Models ============
+
+
+class ChatRequest(BaseModel):
+    question: str = Field(..., min_length=1, max_length=1000)
+
+    @validator('question')
+    def validate_question(cls, v):
+        if not v or not v.strip():
+            raise ValueError("Question cannot be empty")
+        return v.strip()
+
+
+class Source(BaseModel):
+    url: str
+    chunk_index: int
+    text_snippet: str
+
+
+class ChatResponse(BaseModel):
+    answer: str
+    sources: List[Source]
+    tokens_used: int
+    agent_trace: Optional[str] = None
+
+
+class HealthStatus(BaseModel):
+    status: str
+    qdrant: str
+    openai: str
+    timestamp: str
+
+
+# ============ Health Check ============
+
+def check_qdrant_health() -> str:
+    try:
+        from backend.config import get_config
+        from qdrant_client import QdrantClient
+        cfg = get_config()
+        client = QdrantClient(url=cfg["qdrant_url"], api_key=cfg["qdrant_api_key"])
+        client.get_collection(cfg["qdrant_collection"])
+        return "connected"
+    except Exception as e:
+        return "disconnected"
+
+
+def check_openai_health() -> str:
+    try:
+        api_key = os.getenv("OPENAI_API_KEY")
+        if not api_key:
+            return "disconnected"
+        import openai
+        client = openai.OpenAI(api_key=api_key)
+        client.models.list()
+        return "connected"
+    except Exception:
+        return "disconnected"
+
+
+@app.get("/health", response_model=HealthStatus)
+async def health_check():
+    qdrant = check_qdrant_health()
+    openai = check_openai_health()
+    status = "healthy" if qdrant == "connected" and openai == "connected" else "degraded"
+    return HealthStatus(
+        status=status,
+        qdrant=qdrant,
+        openai=openai,
+        timestamp=datetime.utcnow().isoformat() + "Z"
+    )
+
+
+# ============ Chat Endpoint ============
+
+@app.post("/chat")
+async def chat_endpoint(request: ChatRequest):
+    request_id = str(uuid.uuid4())[:8]
+    question = request.question.strip()
+
+    try:
+        agent = get_agent()
+
+        # Run agent with timeout (20s to accommodate full workflow)
+        result = await asyncio.wait_for(
+            Runner.run(agent, question),
+            timeout=20.0
+        )
+
+        # Extract sources from tool call outputs
+        sources = []
+        if result.new_items:
+            for item in result.new_items:
+                if isinstance(item, ToolCallOutputItem):
+                    output = item.output
+                    if isinstance(output, list):
+                        for chunk in output:
+                            sources.append(Source(
+                                url=chunk.get("url", ""),
+                                chunk_index=chunk.get("chunk_index", 0),
+                                text_snippet=chunk.get("text", "")[:200]
+                            ))
+
+        # Get token usage
+        tokens_used = 0
+        if result.context_wrapper and hasattr(result.context_wrapper, 'usage'):
+            tokens_used = result.context_wrapper.usage.total_tokens
+
+        return ChatResponse(
+            answer=result.final_output,
+            sources=sources,
+            tokens_used=tokens_used,
+            agent_trace=f"{request_id}: completed"
+        )
+
+    except asyncio.TimeoutError:
+        return JSONResponse(
+            status_code=504,
+            content={"error": "timeout", "message": "The chatbot is taking too long to respond. Please try a shorter question."}
+        )
+
+    except Exception as e:
+        if "openai" in str(e).lower() or "rate limit" in str(e).lower():
+            return JSONResponse(
+                status_code=503,
+                content={"error": "openai_failed", "message": "The AI service is currently unavailable. Please try again in a few minutes."}
+            )
+        return JSONResponse(
+            status_code=500,
+            content={"error": "internal_error", "message": "An unexpected error occurred. Please refresh the page and try again."}
+        )
+
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=8000)

check_collection.py

@@ -0,0 +1,61 @@
+"""
+Check contents of the Qdrant collection.
+"""
+import sys
+from pathlib import Path
+sys.path.insert(0, str(Path(__file__).parent))
+
+import config
+from qdrant_client import QdrantClient
+
+cfg = config.get_config()
+client = QdrantClient(url=cfg['qdrant_url'], api_key=cfg['qdrant_api_key'])
+
+# Get collection info
+info = client.get_collection(cfg['qdrant_collection'])
+print(f"Collection: {cfg['qdrant_collection']}")
+print(f"Total points: {info.points_count}")
+print(f"Vector size: {info.config.params.vectors.size}")
+print()
+
+# Scroll through all points
+if info.points_count > 0:
+    records = client.scroll(
+        collection_name=cfg['qdrant_collection'],
+        limit=min(10, info.points_count),
+        with_payload=True,
+        with_vectors=False
+    )[0]
+
+    print(f"Showing {len(records)} sample points:")
+    print("=" * 80)
+
+    for i, record in enumerate(records, 1):
+        print(f"\nPoint {i}:")
+        print(f"  ID: {record.id}")
+        payload = record.payload or {}
+        print(f"  URL: {payload.get('url', 'N/A')}")
+        print(f"  Title: {payload.get('title', 'N/A')}")
+        print(f"  Chunk index: {payload.get('chunk_index', 'N/A')}")
+        text = payload.get('text', '')
+        print(f"  Text length: {len(text)} chars")
+        print(f"  Text preview: {text[:200]}...")
+
+    print("\n" + "=" * 80)
+
+    # Check for unique URLs
+    all_records = client.scroll(
+        collection_name=cfg['qdrant_collection'],
+        limit=1000,
+        with_payload=True,
+        with_vectors=False
+    )[0]
+
+    unique_urls = set()
+    for rec in all_records:
+        if rec.payload and 'url' in rec.payload:
+            unique_urls.add(rec.payload['url'])
+
+    print(f"Unique URLs in collection: {len(unique_urls)}")
+    for url in unique_urls:
+        print(f"  - {url}")

check_detailed.py

@@ -0,0 +1,36 @@
+"""
+Detailed check of all points in the collection.
+"""
+import sys
+from pathlib import Path
+sys.path.insert(0, str(Path(__file__).parent))
+
+import config
+from qdrant_client import QdrantClient
+
+cfg = config.get_config()
+client = QdrantClient(url=cfg['qdrant_url'], api_key=cfg['qdrant_api_key'])
+
+# Scroll through all points with full details
+all_points = client.scroll(
+    collection_name=cfg['qdrant_collection'],
+    limit=1000,
+    with_payload=True,
+    with_vectors=False
+)[0]
+
+print(f"Total points in collection: {len(all_points)}")
+print("=" * 100)
+
+for i, point in enumerate(all_points, 1):
+    print(f"\nPoint {i}:")
+    print(f"  ID: {point.id}")
+    payload = point.payload or {}
+    print(f"  URL: {payload.get('url', 'N/A')}")
+    print(f"  Title: {payload.get('title', 'N/A')}")
+    print(f"  Section: {payload.get('section', 'N/A')}")
+    print(f"  Chunk index: {payload.get('chunk_index', 'N/A')}")
+    text = payload.get('text', '')
+    print(f"  Text length: {len(text)} chars")
+    print(f"  Text (full):\n{text}")
+    print("-" * 100)

config.py

@@ -0,0 +1,27 @@
+"""
+Configuration loader for the ingestion pipeline.
+Reads environment variables with .env support.
+"""
+import os
+from dotenv import load_dotenv
+
+# Load .env file if present
+load_dotenv()
+
+def get_config():
+    """Get configuration from environment variables."""
+    return {
+        "cohere_api_key": os.getenv("COHERE_API_KEY"),
+        "qdrant_url": os.getenv("QDRANT_URL"),
+        "qdrant_api_key": os.getenv("QDRANT_API_KEY"),
+        "qdrant_collection": os.getenv("QDRANT_COLLECTION", "book_embeddings"),
+        "openai_api_key": os.getenv("OPENAI_API_KEY"),
+    }
+
+def validate_config(config):
+    """Validate that all required config values are set."""
+    required = ["cohere_api_key", "qdrant_url", "qdrant_api_key"]
+    missing = [key for key in required if not config.get(key)]
+    if missing:
+        raise ValueError(f"Missing required environment variables: {', '.join(missing)}")
+    return True

exceptions.py

@@ -0,0 +1,26 @@
+"""
+Custom exceptions for the RAG retrieval pipeline.
+"""
+from typing import Optional
+
+
+class ConfigurationError(Exception):
+    """Raised when required configuration is missing or invalid."""
+    pass
+
+
+class CollectionNotFoundError(Exception):
+    """Raised when the specified Qdrant collection does not exist."""
+    pass
+
+
+class DimensionMismatchError(Exception):
+    """Raised when embedding dimension doesn't match collection vector size."""
+    pass
+
+
+class APIError(Exception):
+    """Raised when an external API (Cohere or Qdrant) call fails after retries."""
+    def __init__(self, message: str, original_exception: Optional[Exception] = None):
+        super().__init__(message)
+        self.original_exception = original_exception

extract_sitemap.py

@@ -0,0 +1,32 @@
+"""
+Fetch sitemap.xml and extract all page URLs.
+"""
+import sys
+from pathlib import Path
+import httpx
+import re
+
+sys.path.insert(0, str(Path(__file__).parent))
+
+sitemap_url = "https://humanoid-ai-robotics-book-1.vercel.app/sitemap.xml"
+
+print(f"Fetching sitemap from: {sitemap_url}")
+response = httpx.get(sitemap_url, timeout=30.0)
+response.raise_for_status()
+
+# Use regex to extract <loc> URLs from XML
+content = response.text
+urls = re.findall(r'<loc[^>]*>(.*?)</loc>', content, re.DOTALL)
+urls = [url.strip() for url in urls if url.strip()]
+
+print(f"Found {len(urls)} URLs in sitemap:")
+for url in urls:
+    print(f"  - {url}")
+
+# Save to file
+output_file = Path(__file__).parent / 'sitemap_urls.txt'
+with open(output_file, 'w') as f:
+    for url in urls:
+        f.write(url + '\n')
+
+print(f"\nSaved to: {output_file}")

history/prompts/frontend-chat-integration/4-implementation.tasks.prompt.md

@@ -0,0 +1,56 @@
+---
+id: 4
+title: Implement frontend-backend chat integration
+stage: tasks
+date: 2025-02-18
+surface: agent
+model: claude-sonnet-4-5-20250929
+feature: frontend-chat-integration
+branch: 8-frontend-chat-integration
+user: m-ahmad-official
+command: /sp.implement
+labels: [implementation, backend, frontend, fastapi, react, docusaurus]
+links:
+  spec: specs/8-frontend-chat-integration/spec.md
+  plan: specs/8-frontend-chat-integration/plan.md
+  tasks: specs/8-frontend-chat-integration/tasks.md
+  adr: null
+  pr: null
+files:
+  - backend/api.py
+  - frontend_book/src/components/ChatBot.tsx
+  - frontend_book/src/components/ChatBot.module.css
+  - frontend_book/src/theme/Root.tsx
+  - frontend_book/.env
+tests: []
+---
+
+## Prompt
+
+/sp.implement (execute all tasks)
+
+## Response snapshot
+
+Executed all 28 tasks from tasks.md across 6 phases:
+
+**Phase 1 (Setup)**: Verified backend agent.py, frontend_book structure, created .env with VITE_API_BASE_URL, installed npm dependencies
+**Phase 2 (Foundational)**: Created backend/api.py with FastAPI app, CORS middleware, health check
+**Phase 3 (US1-P1)**: Implemented POST /chat with agent invocation, 20s timeout, source extraction, token tracking, error handling; tested with curl (success)
+**Phase 4 (US2-P2)**: Created ChatBot.module.css with responsive styles: desktop (380px fixed panel) and mobile (full-screen overlay)
+**Phase 5 (US3-P3)**: Implemented localStorage persistence in ChatBot.tsx with versioned schema
+**Phase 6 (Polish)**: Integrated globally via src/theme/Root.tsx, added character count (FR-015) and Clear Chat button
+
+All files created and validated.
+
+## Outcome
+
+- ✅ Impact: Complete implementation; backend/frontend integrated and ready for testing
+- 🧪 Tests: End-to-end manual testing recommended; /chat endpoint verified
+- 📁 Files: api.py, ChatBot.tsx, ChatBot.module.css, Root.tsx, .env created
+- 🔁 Next prompts: Manual integration testing; consider /sp.test if applicable
+- 🧠 Reflection: Minimal, isolated changes respect implementation boundaries. Backend wrapper cleanly reuses agent.py; frontend component self-contained.
+
+## Evaluation notes (flywheel)
+
+- Failure modes: None
+- Next experiment: N/A

local_test.log

@@ -0,0 +1,23 @@
+INFO: Extracted 781 characters from fixture
+Text preview: Sample Book Page - Introduction
+Chapter 1: Introduction
+This is the first paragraph of the introduction. It provides an overview of the topic.
+The second paragraph discusses the background and context...
+INFO: Chunked into 2 segments
+First chunk: Sample Book Page - Introduction
+Chapter 1: Introduction
+This is the first paragraph of the introduct...
+INFO: HTTP Request: POST https://api.cohere.com/v2/embed "HTTP/1.1 200 OK"
+INFO: Generated 6 embeddings
+Embedding dimension: 2
+INFO: HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333 "HTTP/1.1 200 OK"
+INFO: HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333/collections "HTTP/1.1 200 OK"
+INFO: Collection 'book_embeddings' already exists
+ERROR: Test failed: 'PosixPath' object has no attribute 'path'
+Traceback (most recent call last):
+  File "/mnt/e/7. Low Code Agentic AI/Hackathon I/humanoid-ai-robotics-book-chatbot/backend/test_local.py", line 93, in <module>
+    test_with_fixture()
+  File "/mnt/e/7. Low Code Agentic AI/Hackathon I/humanoid-ai-robotics-book-chatbot/backend/test_local.py", line 67, in test_with_fixture
+    'title': Path(url).path,
+             ^^^^^^^^^^^^^^
+AttributeError: 'PosixPath' object has no attribute 'path'

logging_config.py

@@ -0,0 +1,77 @@
+"""
+Logging configuration for the ingestion pipeline.
+"""
+import logging
+import sys
+import json
+from pathlib import Path
+from datetime import datetime
+
+def setup_logging(log_file: str = "agent.log", console_level: str = None):
+    """
+    Configure logging to both file (JSON structured) and console (human-readable).
+
+    Args:
+        log_file: Path to log file (default: agent.log)
+        console_level: Log level for console output (defaults to LOG_LEVEL env var or INFO)
+    """
+    import os
+
+    logger = logging.getLogger()
+    logger.setLevel(logging.DEBUG)
+
+    # Clear any existing handlers
+    logger.handlers = []
+
+    # Determine log level from environment or default
+    log_level_env = os.getenv("LOG_LEVEL", "INFO").upper()
+    console_level = console_level or log_level_env
+
+    # Validate log level
+    valid_levels = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
+    if console_level not in valid_levels:
+        console_level = "INFO"
+
+    # ============ File Handler - JSON Structured Format ============
+    file_handler = logging.FileHandler(log_file, mode='a', encoding='utf-8')
+    file_handler.setLevel(logging.DEBUG)
+
+    class JSONFormatter(logging.Formatter):
+        """Format log records as JSON for structured logging."""
+        def format(self, record):
+            log_object = {
+                "timestamp": datetime.utcfromtimestamp(record.created).isoformat() + "Z",
+                "level": record.levelname,
+                "name": record.name,
+                "message": record.getMessage(),
+                "module": record.module,
+                "lineno": record.lineno,
+            }
+            # Include exception info if present
+            if record.exc_info:
+                log_object["exc_info"] = self.formatException(record.exc_info)
+            if record.stack_info:
+                log_object["stack_info"] = record.stack_info
+            # Include extra fields
+            for key, value in record.__dict__.items():
+                if key not in ['name', 'msg', 'args', 'created', 'filename', 'module',
+                               'levelno', 'levelname', 'lineno', 'funcName', 'relativeCreated',
+                               'msecs', 'abs_path', 'exc_info', 'exc_text', 'stack_info',
+                               'message']:
+                    log_object[key] = value
+            return json.dumps(log_object)
+
+    file_formatter = JSONFormatter()
+    file_handler.setFormatter(file_formatter)
+
+    # ============ Console Handler - Human-Readable ============
+    console_handler = logging.StreamHandler(sys.stdout)
+    console_handler.setLevel(getattr(logging, console_level))
+    console_formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(name)s - %(message)s',
+                                          datefmt='%H:%M:%S')
+    console_handler.setFormatter(console_formatter)
+
+    logger.addHandler(file_handler)
+    logger.addHandler(console_handler)
+
+    return logger

main.py

@@ -0,0 +1,434 @@
+"""
+Vector Data Ingestion Pipeline
+
+Fetches book content from URLs, extracts text, chunks, generates embeddings with Cohere,
+and stores in Qdrant Cloud.
+"""
+import argparse
+import sys
+import os
+import time
+from pathlib import Path
+from typing import List, Dict, Any
+
+import httpx
+from bs4 import BeautifulSoup
+import cohere
+from qdrant_client import QdrantClient
+from qdrant_client.models import Distance, VectorParams, PointStruct
+from dotenv import load_dotenv
+
+import config
+import utils
+from logging_config import setup_logging
+
+# Initialize logging
+logger = setup_logging()
+
+
+def fetch_url(url: str, client: httpx.Client, max_retries: int = 5) -> str:
+    """
+    Fetch HTML content from a URL with retry logic.
+    Returns HTML string or raises after max retries.
+    """
+    logger.info(f"Fetching {url}...")
+    for attempt in range(max_retries):
+        try:
+            response = client.get(url, timeout=30.0)
+            response.raise_for_status()
+            logger.debug(f"Successfully fetched {url} (status: {response.status_code})")
+            return response.text
+        except httpx.HTTPError as e:
+            if attempt == max_retries - 1:
+                logger.error(f"Failed to fetch {url} after {max_retries} attempts: {e}")
+                raise
+            delay = 2 ** attempt  # Simple backoff
+            time.sleep(delay)
+    return ""
+
+
+def extract_text(html: str, url: str) -> str:
+    """
+    Extract clean text from HTML.
+    Removes scripts, styles, nav, footer, etc.
+    Preserves paragraph structure.
+    """
+    soup = BeautifulSoup(html, 'html.parser')
+
+    # Remove non-content elements
+    for selector in ['script', 'style', 'nav', 'footer', 'header', 'aside']:
+        for element in soup.find_all(selector):
+            element.decompose()
+
+    # Get text with newlines between elements
+    text = soup.get_text(separator='\n', strip=True)
+
+    # Normalize whitespace
+    lines = [line.strip() for line in text.split('\n') if line.strip()]
+    clean_text = '\n'.join(lines)
+
+    return clean_text
+
+
+def chunk_text(text: str, chunk_size: int = 1000, overlap: int = 100) -> List[Dict[str, Any]]:
+    """
+    Split text into overlapping chunks.
+    Returns list of dicts with text and char_start/end.
+    """
+    if len(text) <= chunk_size:
+        return [{'text': text, 'char_start': 0, 'char_end': len(text)}]
+
+    chunks = []
+    start = 0
+    text_len = len(text)
+
+    while start < text_len:
+        end = min(start + chunk_size, text_len)
+
+        # If not first chunk, include overlap from previous chunk's tail
+        if start > 0:
+            start -= overlap
+            if start < 0:
+                start = 0
+
+        chunk_text = text[start:end]
+        chunks.append({
+            'text': chunk_text,
+            'char_start': start,
+            'char_end': end
+        })
+
+        # Move to next chunk (exclusive end becomes new start)
+        start = end
+
+        # If we're at the end, break
+        if end >= text_len:
+            break
+
+    return chunks
+
+
+def generate_embeddings(texts: List[str], cohere_client: cohere.ClientV2, batch_size: int = 96) -> List[List[float]]:
+    """
+    Generate embeddings for a list of texts using Cohere.
+    Batches requests to optimize throughput.
+    """
+    all_embeddings = []
+
+    for i in range(0, len(texts), batch_size):
+        batch = texts[i:i+batch_size]
+        try:
+            response = cohere_client.embed(
+                texts=batch,
+                model="embed-english-v3.0",
+                input_type="search_document"
+            )
+            # Cohere SDK v2 returns EmbedByTypeResponse with embeddings.float_ containing the vectors
+            embeddings_list = response.embeddings.float_
+
+            # Log the dimension from the first embedding
+            if embeddings_list and len(all_embeddings) == 0:
+                first_embed = embeddings_list[0]
+                if hasattr(first_embed, '__len__') and not isinstance(first_embed, str):
+                    dim = len(first_embed)
+                    logger.info(f"Cohere returned embeddings with dimension: {dim}")
+                else:
+                    logger.info(f"Cohere returned embedding type: {type(first_embed)}")
+
+            # embeddings_list is already a list of vectors (list of floats)
+            all_embeddings.extend(embeddings_list)
+
+            logger.debug(f"Generated embeddings for batch {i//batch_size + 1}/{(len(texts)-1)//batch_size + 1}")
+        except Exception as e:
+            logger.error(f"Failed to generate embeddings for batch starting at index {i}: {e}")
+            raise
+
+    if not all_embeddings:
+        raise ValueError("No embeddings generated")
+
+    # Validate dimension - should be 1024 for embed-english-v3.0
+    actual_dim = len(all_embeddings[0]) if hasattr(all_embeddings[0], '__len__') else len(all_embeddings[0])
+    if actual_dim != 1024:
+        logger.warning(f"Unexpected embedding dimension: {actual_dim} (expected 1024). Proceeding anyway.")
+
+    return all_embeddings
+
+
+def ensure_collection(client: QdrantClient, collection_name: str, vector_size: int = 1024):
+    """
+    Create collection if it doesn't exist.
+    """
+    try:
+        collections = client.get_collections().collections
+        collection_names = [c.name for c in collections]
+        if collection_name not in collection_names:
+            logger.info(f"Creating collection '{collection_name}'")
+            client.create_collection(
+                collection_name=collection_name,
+                vectors_config=VectorParams(size=vector_size, distance=Distance.COSINE)
+            )
+            logger.info(f"Collection '{collection_name}' created with vector size {vector_size}")
+        else:
+            logger.info(f"Collection '{collection_name}' already exists")
+    except Exception as e:
+        logger.error(f"Failed to ensure collection: {e}")
+        raise
+
+
+def upsert_chunks(
+    client: QdrantClient,
+    collection_name: str,
+    records: List[Dict[str, Any]],
+    deterministic_id_func,
+    track_changes: bool = False
+) -> Dict[str, int]:
+    """
+    Upsert chunk records to Qdrant.
+    Uses deterministic IDs for idempotency.
+    Returns statistics: {'new': X, 'updated': Y, 'total': Z}
+    """
+    points = []
+    for record in records:
+        point_id = deterministic_id_func(record['url'], record['chunk_index'])
+        # Verify ID is consistent (US2 - idempotency check)
+        verify_id = utils.verify_deterministic_id(record['url'], record['chunk_index'], point_id)
+        assert verify_id == point_id, "Deterministic ID verification failed"
+
+        point = PointStruct(
+            id=point_id,
+            vector=record['embedding'],
+            payload={
+                'url': record['url'],
+                'title': record.get('title', ''),
+                'section': record.get('section', ''),
+                'chunk_index': record['chunk_index'],
+                'text': record['text']
+            }
+        )
+        points.append(point)
+
+    # Batch upsert (Qdrant client handles batching internally if needed)
+    try:
+        # For now, we don't distinguish new vs updated without a pre-query
+        # But we can report total upserted
+        client.upsert(collection_name=collection_name, points=points)
+        logger.info(f"Upserted {len(points)} points to collection '{collection_name}'")
+        # In future, could check which points already existed by doing a scroll before upsert
+        return {'new': len(points), 'updated': 0, 'total': len(points)}
+    except Exception as e:
+        logger.error(f"Failed to upsert points: {e}")
+        raise
+
+
+def main():
+    """Main orchestration function."""
+    parser = argparse.ArgumentParser(description="Ingest book content into Qdrant")
+    parser.add_argument('--urls', nargs='+', help='List of URLs to ingest')
+    parser.add_argument('--urls-file', help='File containing one URL per line')
+    parser.add_argument('--chunk-size', type=int, default=1000, help='Chunk size in characters')
+    parser.add_argument('--overlap', type=int, default=100, help='Overlap between chunks in characters')
+    parser.add_argument('--batch-size', type=int, default=96, help='Embedding API batch size (max 96 for Cohere trial)')
+    parser.add_argument('--max-retries', type=int, default=5, help='Max retries for API calls')
+    parser.add_argument('--validate', action='store_true', help='Run validation only (no ingestion)')
+    parser.add_argument('--validate-sample', type=int, default=100, help='Number of sample points to check during validation')
+    args = parser.parse_args()
+
+    # Load and validate config
+    cfg = config.get_config()
+    config.validate_config(cfg)
+
+    # Initialize Qdrant client
+    qdrant_client = QdrantClient(
+        url=cfg['qdrant_url'],
+        api_key=cfg['qdrant_api_key']
+    )
+    ensure_collection(qdrant_client, cfg['qdrant_collection'])
+
+    # Validation-only mode
+    if args.validate:
+        run_validation(qdrant_client, cfg['qdrant_collection'], args.validate_sample)
+        sys.exit(0)
+
+    # Ingestion mode
+    cohere_client = cohere.ClientV2(api_key=cfg['cohere_api_key'])
+
+    # Get URLs
+    urls = []
+    if args.urls:
+        urls.extend(args.urls)
+    if args.urls_file:
+        with open(args.urls_file, 'r') as f:
+            urls.extend([line.strip() for line in f if line.strip()])
+
+    if not urls:
+        logger.error("No URLs provided. Use --urls or --urls-file")
+        sys.exit(1)
+
+    logger.info(f"Starting ingestion: {len(urls)} URLs")
+    logger.info(f"Chunk size: {args.chunk_size}, overlap: {args.overlap}")
+
+    total_chunks = 0
+    total_pages = 0
+    errors = []
+
+    # Process each URL
+    with httpx.Client() as http_client:
+        for url in urls:
+            try:
+                html = fetch_url(url, http_client, max_retries=args.max_retries)
+                text = extract_text(html, url)
+
+                if not text or len(text) < 10:
+                    logger.warning(f"Page {url} has insufficient text content, skipping")
+                    continue
+
+                chunks_data = chunk_text(text, chunk_size=args.chunk_size, overlap=args.overlap)
+                logger.info(f"Extracted {len(text)} characters from {url}, chunked into {len(chunks_data)} segments")
+
+                # Prepare texts for embedding
+                texts = [chunk['text'] for chunk in chunks_data]
+                embeddings = generate_embeddings(texts, cohere_client, batch_size=args.batch_size)
+
+                # Validate dimensions
+                if embeddings:
+                    actual_dim = len(embeddings[0]) if hasattr(embeddings[0], '__len__') else len(embeddings[0])
+                    if actual_dim != 1024:
+                        logger.warning(f"Embedding dimension {actual_dim} != 1024. Check Cohere model. Proceeding anyway.")
+
+                # Prepare records for Qdrant
+                records = []
+                for i, (chunk, embedding) in enumerate(zip(chunks_data, embeddings)):
+                    record = {
+                        'url': url,
+                        'title': url,  # Use full URL as title
+                        'section': '',  # TODO: extract from headings
+                        'chunk_index': i,
+                        'text': chunk['text'],
+                        'embedding': embedding
+                    }
+                    records.append(record)
+
+                # Upsert to Qdrant
+                stats = upsert_chunks(qdrant_client, cfg['qdrant_collection'], records, utils.deterministic_id)
+                total_chunks += stats['total']
+                total_pages += 1
+                logger.info(f"Successfully processed {url}")
+
+            except Exception as e:
+                logger.error(f"Failed to process {url}: {e}")
+                errors.append(url)
+                continue
+
+    # Summary
+    logger.info("=" * 50)
+    logger.info(f"Ingestion complete!")
+    logger.info(f"Total pages processed: {total_pages}")
+    logger.info(f"Total chunks stored: {total_chunks}")
+    if errors:
+        logger.warning(f"Failed URLs ({len(errors)}): {', '.join(errors)}")
+    logger.info("=" * 50)
+
+    # Verify collection
+    try:
+        info = qdrant_client.get_collection(cfg['qdrant_collection'])
+        logger.info(f"Qdrant collection '{cfg['qdrant_collection']}' now has {info.points_count} points")
+    except Exception as e:
+        logger.error(f"Could not verify collection: {e}")
+
+    sys.exit(0 if not errors else 1)
+
+
+def run_validation(client: QdrantClient, collection_name: str, sample_size: int = 100):
+    """
+    Run validation checks on the Qdrant collection.
+    Checks: dimensions, metadata completeness, sampling.
+    """
+    logger.info("=" * 50)
+    logger.info("Running validation mode...")
+    logger.info("=" * 50)
+
+    try:
+        # Get collection info
+        info = client.get_collection(collection_name)
+        total_points = info.points_count
+        vector_size = info.config.params.vectors.size
+
+        logger.info(f"Collection: {collection_name}")
+        logger.info(f"Total points: {total_points}")
+        logger.info(f"Vector size: {vector_size}")
+
+        # Validate dimension
+        if vector_size != 1024:
+            logger.error(f"❌ Invalid vector size: {vector_size} (expected 1024)")
+        else:
+            logger.info("✅ Vector dimension correct (1024)")
+
+        # Sample points to check metadata and text
+        if total_points > 0:
+            sample_count = min(sample_size, total_points)
+            logger.info(f"Sampling {sample_count} points for validation...")
+
+            # Scroll through some points
+            records = client.scroll(
+                collection_name=collection_name,
+                limit=sample_count,
+                with_payload=True,
+                with_vectors=False
+            )[0]
+
+            url_ok = 0
+            title_section_ok = 0
+            text_nonempty = 0
+            dimension_checks = 0
+
+            for record in records:
+                payload = record.payload or {}
+
+                # Check URL presence
+                if payload.get('url'):
+                    url_ok += 1
+
+                # Check title or section presence
+                if payload.get('title') or payload.get('section'):
+                    title_section_ok += 1
+
+                # Check text non-empty
+                if payload.get('text') and len(payload['text']) >= 10:
+                    text_nonempty += 1
+
+            # Calculate percentages
+            url_pct = (url_ok / sample_count) * 100
+            title_section_pct = (title_section_ok / sample_count) * 100
+            text_pct = (text_nonempty / sample_count) * 100
+
+            logger.info(f"Metadata completeness:")
+            logger.info(f"  URL present: {url_ok}/{sample_count} ({url_pct:.1f}%)")
+            logger.info(f"  Title/Section present: {title_section_ok}/{sample_count} ({title_section_pct:.1f}%)")
+            logger.info(f"  Text non-empty (≥10 chars): {text_nonempty}/{sample_count} ({text_pct:.1f}%)")
+
+            # Validation thresholds
+            if url_pct >= 99:
+                logger.info("✅ URL completeness excellent (≥99%)")
+            else:
+                logger.warning(f"⚠️ URL completeness below 99%: {url_pct:.1f}%")
+
+            if title_section_pct >= 95:
+                logger.info("✅ Title/Section completeness good (≥95%)")
+            else:
+                logger.warning(f"⚠️ Title/Section completeness below 95%: {title_section_pct:.1f}%")
+
+            if text_pct >= 98:
+                logger.info("✅ Text quality excellent (≥98%)")
+            else:
+                logger.warning(f"⚠️ Text quality below 98%: {text_pct:.1f}%")
+
+        logger.info("=" * 50)
+        logger.info("Validation complete!")
+
+    except Exception as e:
+        logger.error(f"Validation failed: {e}")
+        raise
+
+
+if __name__ == "__main__":
+    main()

pyproject.toml

@@ -0,0 +1,12 @@
+[project]
+name = "vector-data-ingest"
+version = "0.1.0"
+description = "Vector data ingestion pipeline for book RAG system"
+requires-python = ">=3.11"
+dependencies = [
+    "httpx>=0.27.0",
+    "beautifulsoup4>=4.12.0",
+    "cohere>=5.0.0",
+    "qdrant-client>=1.7.0",
+    "python-dotenv>=1.0.0",
+]

requirements.txt

@@ -0,0 +1,16 @@
+# Core dependencies
+httpx>=0.27.0
+beautifulsoup4>=4.12.0
+cohere>=5.0.0
+qdrant-client>=1.7.0
+python-dotenv>=1.0.0
+fastapi>=0.104.0
+uvicorn[standard]>=0.24.0
+openai>=1.0.0
+pydantic>=2.0.0
+openai-agents>=0.8.0
+
+# Additional dependencies for production
+lxml>=4.9.0
+aiofiles>=23.0.0
+python-multipart>=0.0.6

retrieve.log

@@ -0,0 +1,577 @@
+2026-02-17 02:36:58 - INFO - retrieve.py:224 - === Retrieval Pipeline Started ===
+2026-02-17 02:36:58 - INFO - retrieve.py:228 - Loading config from .env
+2026-02-17 02:36:58 - INFO - retrieve.py:233 - Initializing Cohere and Qdrant clients
+2026-02-17 02:37:00 - DEBUG - _trace.py:47 - connect_tcp.started host='d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io' port=6333 local_address=None timeout=5.0 socket_options=None
+2026-02-17 02:37:00 - DEBUG - _trace.py:47 - connect_tcp.complete return_value=<httpcore._backends.sync.SyncStream object at 0x7082a6032cc0>
+2026-02-17 02:37:00 - DEBUG - _trace.py:47 - start_tls.started ssl_context=<ssl.SSLContext object at 0x7082a6013850> server_hostname='d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io' timeout=5.0
+2026-02-17 02:37:01 - DEBUG - _trace.py:47 - start_tls.complete return_value=<httpcore._backends.sync.SyncStream object at 0x7082a6413c50>
+2026-02-17 02:37:01 - DEBUG - _trace.py:47 - send_request_headers.started request=<Request [b'GET']>
+2026-02-17 02:37:01 - DEBUG - _trace.py:47 - send_request_headers.complete
+2026-02-17 02:37:01 - DEBUG - _trace.py:47 - send_request_body.started request=<Request [b'GET']>
+2026-02-17 02:37:01 - DEBUG - _trace.py:47 - send_request_body.complete
+2026-02-17 02:37:01 - DEBUG - _trace.py:47 - receive_response_headers.started request=<Request [b'GET']>
+2026-02-17 02:37:01 - DEBUG - _trace.py:47 - receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Content-Encoding', b'gzip'), (b'Content-Type', b'application/json'), (b'Date', b'Mon, 16 Feb 2026 21:37:00 GMT'), (b'Vary', b'accept-encoding, Origin, Access-Control-Request-Method, Access-Control-Request-Headers'), (b'Transfer-Encoding', b'chunked')])
+2026-02-17 02:37:01 - INFO - _client.py:1025 - HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333 "HTTP/1.1 200 OK"
+2026-02-17 02:37:01 - DEBUG - _trace.py:47 - receive_response_body.started request=<Request [b'GET']>
+2026-02-17 02:37:01 - DEBUG - _trace.py:47 - receive_response_body.complete
+2026-02-17 02:37:01 - DEBUG - _trace.py:47 - response_closed.started
+2026-02-17 02:37:01 - DEBUG - _trace.py:47 - response_closed.complete
+2026-02-17 02:37:01 - DEBUG - _trace.py:47 - close.started
+2026-02-17 02:37:01 - DEBUG - _trace.py:47 - close.complete
+2026-02-17 02:37:01 - INFO - retrieve.py:238 - Checking collection 'book_embeddings'
+2026-02-17 02:37:01 - DEBUG - _trace.py:47 - connect_tcp.started host='d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io' port=6333 local_address=None timeout=5.0 socket_options=None
+2026-02-17 02:37:01 - DEBUG - _trace.py:47 - connect_tcp.complete return_value=<httpcore._backends.sync.SyncStream object at 0x7082a6033f50>
+2026-02-17 02:37:01 - DEBUG - _trace.py:47 - start_tls.started ssl_context=<ssl.SSLContext object at 0x7082a6013650> server_hostname='d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io' timeout=5.0
+2026-02-17 02:37:01 - DEBUG - _trace.py:47 - start_tls.complete return_value=<httpcore._backends.sync.SyncStream object at 0x7082a6033e60>
+2026-02-17 02:37:01 - DEBUG - _trace.py:47 - send_request_headers.started request=<Request [b'GET']>
+2026-02-17 02:37:01 - DEBUG - _trace.py:47 - send_request_headers.complete
+2026-02-17 02:37:01 - DEBUG - _trace.py:47 - send_request_body.started request=<Request [b'GET']>
+2026-02-17 02:37:01 - DEBUG - _trace.py:47 - send_request_body.complete
+2026-02-17 02:37:01 - DEBUG - _trace.py:47 - receive_response_headers.started request=<Request [b'GET']>
+2026-02-17 02:37:01 - DEBUG - _trace.py:47 - receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Content-Encoding', b'gzip'), (b'Content-Type', b'application/json'), (b'Date', b'Mon, 16 Feb 2026 21:37:00 GMT'), (b'Vary', b'accept-encoding, Origin, Access-Control-Request-Method, Access-Control-Request-Headers'), (b'Transfer-Encoding', b'chunked')])
+2026-02-17 02:37:01 - INFO - _client.py:1025 - HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333/collections/book_embeddings "HTTP/1.1 200 OK"
+2026-02-17 02:37:01 - DEBUG - _trace.py:47 - receive_response_body.started request=<Request [b'GET']>
+2026-02-17 02:37:01 - DEBUG - _trace.py:47 - receive_response_body.complete
+2026-02-17 02:37:01 - DEBUG - _trace.py:47 - response_closed.started
+2026-02-17 02:37:01 - DEBUG - _trace.py:47 - response_closed.complete
+2026-02-17 02:37:01 - INFO - retrieve.py:240 - Collection OK: vector_size=1024, points=277
+2026-02-17 02:37:01 - INFO - retrieve.py:121 - Embedding query: 'ROS 2...' (top_k=5)
+2026-02-17 02:37:01 - DEBUG - _trace.py:47 - connect_tcp.started host='api.cohere.com' port=443 local_address=None timeout=300 socket_options=None
+2026-02-17 02:37:01 - DEBUG - _trace.py:47 - connect_tcp.complete return_value=<httpcore._backends.sync.SyncStream object at 0x7082a5ef0e30>
+2026-02-17 02:37:01 - DEBUG - _trace.py:47 - start_tls.started ssl_context=<ssl.SSLContext object at 0x7082a6012f50> server_hostname='api.cohere.com' timeout=300
+2026-02-17 02:37:01 - DEBUG - _trace.py:47 - start_tls.complete return_value=<httpcore._backends.sync.SyncStream object at 0x7082a5ef0d40>
+2026-02-17 02:37:01 - DEBUG - _trace.py:47 - send_request_headers.started request=<Request [b'POST']>
+2026-02-17 02:37:01 - DEBUG - _trace.py:47 - send_request_headers.complete
+2026-02-17 02:37:01 - DEBUG - _trace.py:47 - send_request_body.started request=<Request [b'POST']>
+2026-02-17 02:37:01 - DEBUG - _trace.py:47 - send_request_body.complete
+2026-02-17 02:37:01 - DEBUG - _trace.py:47 - receive_response_headers.started request=<Request [b'POST']>
+2026-02-17 02:37:02 - DEBUG - _trace.py:47 - receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'access-control-expose-headers', b'X-Debug-Trace-ID'), (b'cache-control', b'no-cache, no-store, no-transform, must-revalidate, private, max-age=0'), (b'content-encoding', b'gzip'), (b'content-type', b'application/json'), (b'expires', b'Thu, 01 Jan 1970 00:00:00 GMT'), (b'num_chars', b'5'), (b'num_tokens', b'3'), (b'pragma', b'no-cache'), (b'vary', b'Origin,Accept-Encoding'), (b'x-accel-expires', b'0'), (b'x-debug-trace-id', b'3aff3ef949954c94990bb515bf13453d'), (b'x-endpoint-monthly-call-limit', b'1000'), (b'x-trial-endpoint-call-limit', b'100'), (b'x-trial-endpoint-call-remaining', b'99'), (b'date', b'Mon, 16 Feb 2026 21:37:01 GMT'), (b'x-envoy-upstream-service-time', b'44'), (b'server', b'envoy'), (b'Via', b'1.1 google'), (b'Alt-Svc', b'h3=":443"; ma=2592000,h3-29=":443"; ma=2592000'), (b'Transfer-Encoding', b'chunked')])
+2026-02-17 02:37:02 - INFO - _client.py:1025 - HTTP Request: POST https://api.cohere.com/v2/embed "HTTP/1.1 200 OK"
+2026-02-17 02:37:02 - DEBUG - _trace.py:47 - receive_response_body.started request=<Request [b'POST']>
+2026-02-17 02:37:02 - DEBUG - _trace.py:47 - receive_response_body.complete
+2026-02-17 02:37:02 - DEBUG - _trace.py:47 - response_closed.started
+2026-02-17 02:37:02 - DEBUG - _trace.py:47 - response_closed.complete
+2026-02-17 02:37:02 - DEBUG - retrieve.py:133 - Generated embedding in 0.45s, dimension: 1024
+2026-02-17 02:37:05 - ERROR - retrieve.py:156 - Search failed: 'QdrantClient' object has no attribute 'search'
+2026-02-17 02:37:05 - ERROR - retrieve.py:298 - API error: Qdrant search failed: 'QdrantClient' object has no attribute 'search'
+2026-02-17 02:37:05 - DEBUG - _trace.py:47 - close.started
+2026-02-17 02:37:05 - DEBUG - _trace.py:47 - close.complete
+2026-02-17 02:39:25 - INFO - retrieve.py:225 - === Retrieval Pipeline Started ===
+2026-02-17 02:39:25 - INFO - retrieve.py:229 - Loading config from .env
+2026-02-17 02:39:25 - INFO - retrieve.py:234 - Initializing Cohere and Qdrant clients
+2026-02-17 02:39:25 - DEBUG - _trace.py:47 - connect_tcp.started host='d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io' port=6333 local_address=None timeout=5.0 socket_options=None
+2026-02-17 02:39:25 - DEBUG - _trace.py:47 - connect_tcp.complete return_value=<httpcore._backends.sync.SyncStream object at 0x7a573c568380>
+2026-02-17 02:39:25 - DEBUG - _trace.py:47 - start_tls.started ssl_context=<ssl.SSLContext object at 0x7a573c40f8d0> server_hostname='d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io' timeout=5.0
+2026-02-17 02:39:26 - DEBUG - _trace.py:47 - start_tls.complete return_value=<httpcore._backends.sync.SyncStream object at 0x7a573c8138f0>
+2026-02-17 02:39:26 - DEBUG - _trace.py:47 - send_request_headers.started request=<Request [b'GET']>
+2026-02-17 02:39:26 - DEBUG - _trace.py:47 - send_request_headers.complete
+2026-02-17 02:39:26 - DEBUG - _trace.py:47 - send_request_body.started request=<Request [b'GET']>
+2026-02-17 02:39:26 - DEBUG - _trace.py:47 - send_request_body.complete
+2026-02-17 02:39:26 - DEBUG - _trace.py:47 - receive_response_headers.started request=<Request [b'GET']>
+2026-02-17 02:39:26 - DEBUG - _trace.py:47 - receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Content-Encoding', b'gzip'), (b'Content-Type', b'application/json'), (b'Date', b'Mon, 16 Feb 2026 21:39:25 GMT'), (b'Vary', b'accept-encoding, Origin, Access-Control-Request-Method, Access-Control-Request-Headers'), (b'Transfer-Encoding', b'chunked')])
+2026-02-17 02:39:26 - INFO - _client.py:1025 - HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333 "HTTP/1.1 200 OK"
+2026-02-17 02:39:26 - DEBUG - _trace.py:47 - receive_response_body.started request=<Request [b'GET']>
+2026-02-17 02:39:26 - DEBUG - _trace.py:47 - receive_response_body.complete
+2026-02-17 02:39:26 - DEBUG - _trace.py:47 - response_closed.started
+2026-02-17 02:39:26 - DEBUG - _trace.py:47 - response_closed.complete
+2026-02-17 02:39:26 - DEBUG - _trace.py:47 - close.started
+2026-02-17 02:39:26 - DEBUG - _trace.py:47 - close.complete
+2026-02-17 02:39:26 - INFO - retrieve.py:239 - Checking collection 'book_embeddings'
+2026-02-17 02:39:26 - DEBUG - _trace.py:47 - connect_tcp.started host='d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io' port=6333 local_address=None timeout=5.0 socket_options=None
+2026-02-17 02:39:26 - DEBUG - _trace.py:47 - connect_tcp.complete return_value=<httpcore._backends.sync.SyncStream object at 0x7a573c433d40>
+2026-02-17 02:39:26 - DEBUG - _trace.py:47 - start_tls.started ssl_context=<ssl.SSLContext object at 0x7a573c40f6d0> server_hostname='d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io' timeout=5.0
+2026-02-17 02:39:26 - DEBUG - _trace.py:47 - start_tls.complete return_value=<httpcore._backends.sync.SyncStream object at 0x7a573c433c50>
+2026-02-17 02:39:26 - DEBUG - _trace.py:47 - send_request_headers.started request=<Request [b'GET']>
+2026-02-17 02:39:26 - DEBUG - _trace.py:47 - send_request_headers.complete
+2026-02-17 02:39:26 - DEBUG - _trace.py:47 - send_request_body.started request=<Request [b'GET']>
+2026-02-17 02:39:26 - DEBUG - _trace.py:47 - send_request_body.complete
+2026-02-17 02:39:26 - DEBUG - _trace.py:47 - receive_response_headers.started request=<Request [b'GET']>
+2026-02-17 02:39:26 - DEBUG - _trace.py:47 - receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Content-Encoding', b'gzip'), (b'Content-Type', b'application/json'), (b'Date', b'Mon, 16 Feb 2026 21:39:25 GMT'), (b'Vary', b'accept-encoding, Origin, Access-Control-Request-Method, Access-Control-Request-Headers'), (b'Transfer-Encoding', b'chunked')])
+2026-02-17 02:39:26 - INFO - _client.py:1025 - HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333/collections/book_embeddings "HTTP/1.1 200 OK"
+2026-02-17 02:39:26 - DEBUG - _trace.py:47 - receive_response_body.started request=<Request [b'GET']>
+2026-02-17 02:39:26 - DEBUG - _trace.py:47 - receive_response_body.complete
+2026-02-17 02:39:26 - DEBUG - _trace.py:47 - response_closed.started
+2026-02-17 02:39:26 - DEBUG - _trace.py:47 - response_closed.complete
+2026-02-17 02:39:26 - INFO - retrieve.py:241 - Collection OK: vector_size=1024, points=277
+2026-02-17 02:39:26 - INFO - retrieve.py:121 - Embedding query: 'ROS 2...' (top_k=5)
+2026-02-17 02:39:26 - DEBUG - _trace.py:47 - connect_tcp.started host='api.cohere.com' port=443 local_address=None timeout=300 socket_options=None
+2026-02-17 02:39:26 - DEBUG - _trace.py:47 - connect_tcp.complete return_value=<httpcore._backends.sync.SyncStream object at 0x7a573c2f0c50>
+2026-02-17 02:39:26 - DEBUG - _trace.py:47 - start_tls.started ssl_context=<ssl.SSLContext object at 0x7a573c40efd0> server_hostname='api.cohere.com' timeout=300
+2026-02-17 02:39:26 - DEBUG - _trace.py:47 - start_tls.complete return_value=<httpcore._backends.sync.SyncStream object at 0x7a573c2f0b60>
+2026-02-17 02:39:26 - DEBUG - _trace.py:47 - send_request_headers.started request=<Request [b'POST']>
+2026-02-17 02:39:26 - DEBUG - _trace.py:47 - send_request_headers.complete
+2026-02-17 02:39:26 - DEBUG - _trace.py:47 - send_request_body.started request=<Request [b'POST']>
+2026-02-17 02:39:26 - DEBUG - _trace.py:47 - send_request_body.complete
+2026-02-17 02:39:26 - DEBUG - _trace.py:47 - receive_response_headers.started request=<Request [b'POST']>
+2026-02-17 02:39:27 - DEBUG - _trace.py:47 - receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'access-control-expose-headers', b'X-Debug-Trace-ID'), (b'cache-control', b'no-cache, no-store, no-transform, must-revalidate, private, max-age=0'), (b'content-encoding', b'gzip'), (b'content-type', b'application/json'), (b'expires', b'Thu, 01 Jan 1970 00:00:00 GMT'), (b'num_chars', b'5'), (b'num_tokens', b'3'), (b'pragma', b'no-cache'), (b'vary', b'Origin,Accept-Encoding'), (b'x-accel-expires', b'0'), (b'x-debug-trace-id', b'9cfafd18a8c68f9fc1f1e0d937c5508a'), (b'x-endpoint-monthly-call-limit', b'1000'), (b'x-trial-endpoint-call-limit', b'100'), (b'x-trial-endpoint-call-remaining', b'99'), (b'date', b'Mon, 16 Feb 2026 21:39:26 GMT'), (b'x-envoy-upstream-service-time', b'48'), (b'server', b'envoy'), (b'Via', b'1.1 google'), (b'Alt-Svc', b'h3=":443"; ma=2592000,h3-29=":443"; ma=2592000'), (b'Transfer-Encoding', b'chunked')])
+2026-02-17 02:39:27 - INFO - _client.py:1025 - HTTP Request: POST https://api.cohere.com/v2/embed "HTTP/1.1 200 OK"
+2026-02-17 02:39:27 - DEBUG - _trace.py:47 - receive_response_body.started request=<Request [b'POST']>
+2026-02-17 02:39:27 - DEBUG - _trace.py:47 - receive_response_body.complete
+2026-02-17 02:39:27 - DEBUG - _trace.py:47 - response_closed.started
+2026-02-17 02:39:27 - DEBUG - _trace.py:47 - response_closed.complete
+2026-02-17 02:39:27 - DEBUG - retrieve.py:133 - Generated embedding in 0.46s, dimension: 1024
+2026-02-17 02:39:27 - DEBUG - _trace.py:47 - send_request_headers.started request=<Request [b'POST']>
+2026-02-17 02:39:27 - DEBUG - _trace.py:47 - send_request_headers.complete
+2026-02-17 02:39:27 - DEBUG - _trace.py:47 - send_request_body.started request=<Request [b'POST']>
+2026-02-17 02:39:27 - DEBUG - _trace.py:47 - send_request_body.complete
+2026-02-17 02:39:27 - DEBUG - _trace.py:47 - receive_response_headers.started request=<Request [b'POST']>
+2026-02-17 02:39:27 - DEBUG - _trace.py:47 - receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Content-Encoding', b'gzip'), (b'Content-Type', b'application/json'), (b'Date', b'Mon, 16 Feb 2026 21:39:26 GMT'), (b'Vary', b'accept-encoding, Origin, Access-Control-Request-Method, Access-Control-Request-Headers'), (b'Transfer-Encoding', b'chunked')])
+2026-02-17 02:39:27 - INFO - _client.py:1025 - HTTP Request: POST https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333/collections/book_embeddings/points/query "HTTP/1.1 200 OK"
+2026-02-17 02:39:27 - DEBUG - _trace.py:47 - receive_response_body.started request=<Request [b'POST']>
+2026-02-17 02:39:27 - DEBUG - _trace.py:47 - receive_response_body.complete
+2026-02-17 02:39:27 - DEBUG - _trace.py:47 - response_closed.started
+2026-02-17 02:39:27 - DEBUG - _trace.py:47 - response_closed.complete
+2026-02-17 02:39:27 - INFO - retrieve.py:155 - Search completed in 0.31s, returned 5 results
+2026-02-17 02:39:27 - INFO - retrieve.py:170 - Total query time: 0.78s
+2026-02-17 02:39:27 - INFO - retrieve.py:279 - === Retrieval Pipeline Completed Successfully ===
+2026-02-17 02:39:27 - DEBUG - _trace.py:47 - close.started
+2026-02-17 02:39:27 - DEBUG - _trace.py:47 - close.complete
+2026-02-17 02:41:01 - INFO - retrieve.py:299 - === Retrieval Pipeline Started ===
+2026-02-17 02:41:01 - INFO - retrieve.py:303 - Loading config from .env
+2026-02-17 02:41:01 - INFO - retrieve.py:308 - Initializing Cohere and Qdrant clients
+2026-02-17 02:41:02 - DEBUG - _trace.py:47 - connect_tcp.started host='d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io' port=6333 local_address=None timeout=5.0 socket_options=None
+2026-02-17 02:41:03 - DEBUG - _trace.py:47 - connect_tcp.complete return_value=<httpcore._backends.sync.SyncStream object at 0x72e5e492f170>
+2026-02-17 02:41:03 - DEBUG - _trace.py:47 - start_tls.started ssl_context=<ssl.SSLContext object at 0x72e5e4823e50> server_hostname='d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io' timeout=5.0
+2026-02-17 02:41:03 - DEBUG - _trace.py:47 - start_tls.complete return_value=<httpcore._backends.sync.SyncStream object at 0x72e5e492fce0>
+2026-02-17 02:41:03 - DEBUG - _trace.py:47 - send_request_headers.started request=<Request [b'GET']>
+2026-02-17 02:41:03 - DEBUG - _trace.py:47 - send_request_headers.complete
+2026-02-17 02:41:04 - DEBUG - _trace.py:47 - send_request_body.started request=<Request [b'GET']>
+2026-02-17 02:41:04 - DEBUG - _trace.py:47 - send_request_body.complete
+2026-02-17 02:41:04 - DEBUG - _trace.py:47 - receive_response_headers.started request=<Request [b'GET']>
+2026-02-17 02:41:04 - DEBUG - _trace.py:47 - receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Content-Encoding', b'gzip'), (b'Content-Type', b'application/json'), (b'Date', b'Mon, 16 Feb 2026 21:41:02 GMT'), (b'Vary', b'accept-encoding, Origin, Access-Control-Request-Method, Access-Control-Request-Headers'), (b'Transfer-Encoding', b'chunked')])
+2026-02-17 02:41:04 - INFO - _client.py:1025 - HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333 "HTTP/1.1 200 OK"
+2026-02-17 02:41:04 - DEBUG - _trace.py:47 - receive_response_body.started request=<Request [b'GET']>
+2026-02-17 02:41:04 - DEBUG - _trace.py:47 - receive_response_body.complete
+2026-02-17 02:41:04 - DEBUG - _trace.py:47 - response_closed.started
+2026-02-17 02:41:04 - DEBUG - _trace.py:47 - response_closed.complete
+2026-02-17 02:41:04 - DEBUG - _trace.py:47 - close.started
+2026-02-17 02:41:04 - DEBUG - _trace.py:47 - close.complete
+2026-02-17 02:41:04 - INFO - retrieve.py:313 - Checking collection 'book_embeddings'
+2026-02-17 02:41:04 - DEBUG - _trace.py:47 - connect_tcp.started host='d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io' port=6333 local_address=None timeout=5.0 socket_options=None
+2026-02-17 02:41:04 - DEBUG - _trace.py:47 - connect_tcp.complete return_value=<httpcore._backends.sync.SyncStream object at 0x72e5e485dee0>
+2026-02-17 02:41:04 - DEBUG - _trace.py:47 - start_tls.started ssl_context=<ssl.SSLContext object at 0x72e5e4823c50> server_hostname='d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io' timeout=5.0
+2026-02-17 02:41:04 - DEBUG - _trace.py:47 - start_tls.complete return_value=<httpcore._backends.sync.SyncStream object at 0x72e5e485dc10>
+2026-02-17 02:41:04 - DEBUG - _trace.py:47 - send_request_headers.started request=<Request [b'GET']>
+2026-02-17 02:41:04 - DEBUG - _trace.py:47 - send_request_headers.complete
+2026-02-17 02:41:04 - DEBUG - _trace.py:47 - send_request_body.started request=<Request [b'GET']>
+2026-02-17 02:41:04 - DEBUG - _trace.py:47 - send_request_body.complete
+2026-02-17 02:41:04 - DEBUG - _trace.py:47 - receive_response_headers.started request=<Request [b'GET']>
+2026-02-17 02:41:05 - DEBUG - _trace.py:47 - receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Content-Encoding', b'gzip'), (b'Content-Type', b'application/json'), (b'Date', b'Mon, 16 Feb 2026 21:41:03 GMT'), (b'Vary', b'accept-encoding, Origin, Access-Control-Request-Method, Access-Control-Request-Headers'), (b'Transfer-Encoding', b'chunked')])
+2026-02-17 02:41:05 - INFO - _client.py:1025 - HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333/collections/book_embeddings "HTTP/1.1 200 OK"
+2026-02-17 02:41:05 - DEBUG - _trace.py:47 - receive_response_body.started request=<Request [b'GET']>
+2026-02-17 02:41:05 - DEBUG - _trace.py:47 - receive_response_body.complete
+2026-02-17 02:41:05 - DEBUG - _trace.py:47 - response_closed.started
+2026-02-17 02:41:05 - DEBUG - _trace.py:47 - response_closed.complete
+2026-02-17 02:41:05 - INFO - retrieve.py:315 - Collection OK: vector_size=1024, points=277
+2026-02-17 02:41:05 - INFO - retrieve.py:190 - Embedding query: 'ROS 2...' (top_k=5)
+2026-02-17 02:41:05 - DEBUG - _trace.py:47 - connect_tcp.started host='api.cohere.com' port=443 local_address=None timeout=300 socket_options=None
+2026-02-17 02:41:05 - DEBUG - _trace.py:47 - connect_tcp.complete return_value=<httpcore._backends.sync.SyncStream object at 0x72e5e485ede0>
+2026-02-17 02:41:05 - DEBUG - _trace.py:47 - start_tls.started ssl_context=<ssl.SSLContext object at 0x72e5e4823550> server_hostname='api.cohere.com' timeout=300
+2026-02-17 02:41:05 - DEBUG - _trace.py:47 - start_tls.complete return_value=<httpcore._backends.sync.SyncStream object at 0x72e5e485ecf0>
+2026-02-17 02:41:05 - DEBUG - _trace.py:47 - send_request_headers.started request=<Request [b'POST']>
+2026-02-17 02:41:05 - DEBUG - _trace.py:47 - send_request_headers.complete
+2026-02-17 02:41:05 - DEBUG - _trace.py:47 - send_request_body.started request=<Request [b'POST']>
+2026-02-17 02:41:05 - DEBUG - _trace.py:47 - send_request_body.complete
+2026-02-17 02:41:05 - DEBUG - _trace.py:47 - receive_response_headers.started request=<Request [b'POST']>
+2026-02-17 02:41:05 - DEBUG - _trace.py:47 - receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'access-control-expose-headers', b'X-Debug-Trace-ID'), (b'cache-control', b'no-cache, no-store, no-transform, must-revalidate, private, max-age=0'), (b'content-encoding', b'gzip'), (b'content-type', b'application/json'), (b'expires', b'Thu, 01 Jan 1970 00:00:00 GMT'), (b'num_chars', b'5'), (b'num_tokens', b'3'), (b'pragma', b'no-cache'), (b'vary', b'Origin,Accept-Encoding'), (b'x-accel-expires', b'0'), (b'x-debug-trace-id', b'7eab84de888d9426719d5f02fbc98608'), (b'x-endpoint-monthly-call-limit', b'1000'), (b'x-trial-endpoint-call-limit', b'100'), (b'x-trial-endpoint-call-remaining', b'99'), (b'date', b'Mon, 16 Feb 2026 21:41:04 GMT'), (b'x-envoy-upstream-service-time', b'49'), (b'server', b'envoy'), (b'Via', b'1.1 google'), (b'Alt-Svc', b'h3=":443"; ma=2592000,h3-29=":443"; ma=2592000'), (b'Transfer-Encoding', b'chunked')])
+2026-02-17 02:41:05 - INFO - _client.py:1025 - HTTP Request: POST https://api.cohere.com/v2/embed "HTTP/1.1 200 OK"
+2026-02-17 02:41:05 - DEBUG - _trace.py:47 - receive_response_body.started request=<Request [b'POST']>
+2026-02-17 02:41:05 - DEBUG - _trace.py:47 - receive_response_body.complete
+2026-02-17 02:41:05 - DEBUG - _trace.py:47 - response_closed.started
+2026-02-17 02:41:05 - DEBUG - _trace.py:47 - response_closed.complete
+2026-02-17 02:41:05 - DEBUG - retrieve.py:202 - Generated embedding in 0.44s, dimension: 1024
+2026-02-17 02:41:05 - DEBUG - _trace.py:47 - send_request_headers.started request=<Request [b'POST']>
+2026-02-17 02:41:05 - DEBUG - _trace.py:47 - send_request_headers.complete
+2026-02-17 02:41:05 - DEBUG - _trace.py:47 - send_request_body.started request=<Request [b'POST']>
+2026-02-17 02:41:05 - DEBUG - _trace.py:47 - send_request_body.complete
+2026-02-17 02:41:05 - DEBUG - _trace.py:47 - receive_response_headers.started request=<Request [b'POST']>
+2026-02-17 02:41:05 - DEBUG - _trace.py:47 - receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Content-Encoding', b'gzip'), (b'Content-Type', b'application/json'), (b'Date', b'Mon, 16 Feb 2026 21:41:04 GMT'), (b'Vary', b'accept-encoding, Origin, Access-Control-Request-Method, Access-Control-Request-Headers'), (b'Transfer-Encoding', b'chunked')])
+2026-02-17 02:41:05 - INFO - _client.py:1025 - HTTP Request: POST https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333/collections/book_embeddings/points/query "HTTP/1.1 200 OK"
+2026-02-17 02:41:05 - DEBUG - _trace.py:47 - receive_response_body.started request=<Request [b'POST']>
+2026-02-17 02:41:05 - DEBUG - _trace.py:47 - receive_response_body.complete
+2026-02-17 02:41:05 - DEBUG - _trace.py:47 - response_closed.started
+2026-02-17 02:41:05 - DEBUG - _trace.py:47 - response_closed.complete
+2026-02-17 02:41:05 - INFO - retrieve.py:224 - Search completed in 0.30s, returned 5 results
+2026-02-17 02:41:05 - INFO - retrieve.py:239 - Total query time: 0.75s
+2026-02-17 02:41:05 - DEBUG - retrieve.py:127 - Metadata completeness: 5/5 = 100.0%
+2026-02-17 02:41:05 - DEBUG - retrieve.py:157 - Chunk sequencing invalid for https://humanoid-ai-robotics-book-1.vercel.app/docs/module-1/intro-to-ros2: expected [0, 1, 2, 3], got [0, 2, 4, 5]
+2026-02-17 02:41:05 - INFO - retrieve.py:340 - Metadata completeness: 100.0%
+2026-02-17 02:41:05 - INFO - retrieve.py:341 - Chunk sequencing: INVALID
+2026-02-17 02:41:05 - INFO - retrieve.py:342 - Validation result: FAIL
+2026-02-17 02:41:05 - INFO - retrieve.py:369 - === Retrieval Pipeline Completed Successfully ===
+2026-02-17 02:41:05 - DEBUG - _trace.py:47 - close.started
+2026-02-17 02:41:05 - DEBUG - _trace.py:47 - close.complete
+2026-02-17 02:42:10 - INFO - retrieve.py:309 - === Retrieval Pipeline Started ===
+2026-02-17 02:42:10 - INFO - retrieve.py:313 - Loading config from .env
+2026-02-17 02:42:10 - INFO - retrieve.py:318 - Initializing Cohere and Qdrant clients
+2026-02-17 02:42:10 - DEBUG - _trace.py:47 - connect_tcp.started host='d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io' port=6333 local_address=None timeout=5.0 socket_options=None
+2026-02-17 02:42:11 - DEBUG - _trace.py:47 - connect_tcp.complete return_value=<httpcore._backends.sync.SyncStream object at 0x79f99627efc0>
+2026-02-17 02:42:11 - DEBUG - _trace.py:47 - start_tls.started ssl_context=<ssl.SSLContext object at 0x79f995c178d0> server_hostname='d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io' timeout=5.0
+2026-02-17 02:42:11 - DEBUG - _trace.py:47 - start_tls.complete return_value=<httpcore._backends.sync.SyncStream object at 0x79f995ee2090>
+2026-02-17 02:42:11 - DEBUG - _trace.py:47 - send_request_headers.started request=<Request [b'GET']>
+2026-02-17 02:42:11 - DEBUG - _trace.py:47 - send_request_headers.complete
+2026-02-17 02:42:11 - DEBUG - _trace.py:47 - send_request_body.started request=<Request [b'GET']>
+2026-02-17 02:42:11 - DEBUG - _trace.py:47 - send_request_body.complete
+2026-02-17 02:42:11 - DEBUG - _trace.py:47 - receive_response_headers.started request=<Request [b'GET']>
+2026-02-17 02:42:11 - DEBUG - _trace.py:47 - receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Content-Encoding', b'gzip'), (b'Content-Type', b'application/json'), (b'Date', b'Mon, 16 Feb 2026 21:42:09 GMT'), (b'Vary', b'accept-encoding, Origin, Access-Control-Request-Method, Access-Control-Request-Headers'), (b'Transfer-Encoding', b'chunked')])
+2026-02-17 02:42:11 - INFO - _client.py:1025 - HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333 "HTTP/1.1 200 OK"
+2026-02-17 02:42:11 - DEBUG - _trace.py:47 - receive_response_body.started request=<Request [b'GET']>
+2026-02-17 02:42:11 - DEBUG - _trace.py:47 - receive_response_body.complete
+2026-02-17 02:42:11 - DEBUG - _trace.py:47 - response_closed.started
+2026-02-17 02:42:11 - DEBUG - _trace.py:47 - response_closed.complete
+2026-02-17 02:42:11 - DEBUG - _trace.py:47 - close.started
+2026-02-17 02:42:11 - DEBUG - _trace.py:47 - close.complete
+2026-02-17 02:42:11 - INFO - retrieve.py:323 - Checking collection 'book_embeddings'
+2026-02-17 02:42:11 - DEBUG - _trace.py:47 - connect_tcp.started host='d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io' port=6333 local_address=None timeout=5.0 socket_options=None
+2026-02-17 02:42:11 - DEBUG - _trace.py:47 - connect_tcp.complete return_value=<httpcore._backends.sync.SyncStream object at 0x79f995c41ee0>
+2026-02-17 02:42:11 - DEBUG - _trace.py:47 - start_tls.started ssl_context=<ssl.SSLContext object at 0x79f995c176d0> server_hostname='d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io' timeout=5.0
+2026-02-17 02:42:12 - DEBUG - _trace.py:47 - start_tls.complete return_value=<httpcore._backends.sync.SyncStream object at 0x79f995c41e20>
+2026-02-17 02:42:12 - DEBUG - _trace.py:47 - send_request_headers.started request=<Request [b'GET']>
+2026-02-17 02:42:12 - DEBUG - _trace.py:47 - send_request_headers.complete
+2026-02-17 02:42:12 - DEBUG - _trace.py:47 - send_request_body.started request=<Request [b'GET']>
+2026-02-17 02:42:12 - DEBUG - _trace.py:47 - send_request_body.complete
+2026-02-17 02:42:12 - DEBUG - _trace.py:47 - receive_response_headers.started request=<Request [b'GET']>
+2026-02-17 02:42:12 - DEBUG - _trace.py:47 - receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Content-Encoding', b'gzip'), (b'Content-Type', b'application/json'), (b'Date', b'Mon, 16 Feb 2026 21:42:09 GMT'), (b'Vary', b'accept-encoding, Origin, Access-Control-Request-Method, Access-Control-Request-Headers'), (b'Transfer-Encoding', b'chunked')])
+2026-02-17 02:42:12 - INFO - _client.py:1025 - HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333/collections/book_embeddings "HTTP/1.1 200 OK"
+2026-02-17 02:42:12 - DEBUG - _trace.py:47 - receive_response_body.started request=<Request [b'GET']>
+2026-02-17 02:42:12 - DEBUG - _trace.py:47 - receive_response_body.complete
+2026-02-17 02:42:12 - DEBUG - _trace.py:47 - response_closed.started
+2026-02-17 02:42:12 - DEBUG - _trace.py:47 - response_closed.complete
+2026-02-17 02:42:12 - INFO - retrieve.py:325 - Collection OK: vector_size=1024, points=277
+2026-02-17 02:42:12 - INFO - retrieve.py:200 - Embedding query: 'ROS 2...' (top_k=5)
+2026-02-17 02:42:12 - DEBUG - _trace.py:47 - connect_tcp.started host='api.cohere.com' port=443 local_address=None timeout=300 socket_options=None
+2026-02-17 02:42:12 - DEBUG - _trace.py:47 - connect_tcp.complete return_value=<httpcore._backends.sync.SyncStream object at 0x79f995c42d50>
+2026-02-17 02:42:12 - DEBUG - _trace.py:47 - start_tls.started ssl_context=<ssl.SSLContext object at 0x79f995c16fd0> server_hostname='api.cohere.com' timeout=300
+2026-02-17 02:42:12 - DEBUG - _trace.py:47 - start_tls.complete return_value=<httpcore._backends.sync.SyncStream object at 0x79f995c42c60>
+2026-02-17 02:42:12 - DEBUG - _trace.py:47 - send_request_headers.started request=<Request [b'POST']>
+2026-02-17 02:42:12 - DEBUG - _trace.py:47 - send_request_headers.complete
+2026-02-17 02:42:12 - DEBUG - _trace.py:47 - send_request_body.started request=<Request [b'POST']>
+2026-02-17 02:42:12 - DEBUG - _trace.py:47 - send_request_body.complete
+2026-02-17 02:42:12 - DEBUG - _trace.py:47 - receive_response_headers.started request=<Request [b'POST']>
+2026-02-17 02:42:12 - DEBUG - _trace.py:47 - receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'access-control-expose-headers', b'X-Debug-Trace-ID'), (b'cache-control', b'no-cache, no-store, no-transform, must-revalidate, private, max-age=0'), (b'content-encoding', b'gzip'), (b'content-type', b'application/json'), (b'expires', b'Thu, 01 Jan 1970 00:00:00 GMT'), (b'num_chars', b'5'), (b'num_tokens', b'3'), (b'pragma', b'no-cache'), (b'vary', b'Origin,Accept-Encoding'), (b'x-accel-expires', b'0'), (b'x-debug-trace-id', b'd004675c4a251ca57146bf895e986d52'), (b'x-endpoint-monthly-call-limit', b'1000'), (b'x-trial-endpoint-call-limit', b'100'), (b'x-trial-endpoint-call-remaining', b'99'), (b'date', b'Mon, 16 Feb 2026 21:42:10 GMT'), (b'x-envoy-upstream-service-time', b'41'), (b'server', b'envoy'), (b'Via', b'1.1 google'), (b'Alt-Svc', b'h3=":443"; ma=2592000,h3-29=":443"; ma=2592000'), (b'Transfer-Encoding', b'chunked')])
+2026-02-17 02:42:12 - INFO - _client.py:1025 - HTTP Request: POST https://api.cohere.com/v2/embed "HTTP/1.1 200 OK"
+2026-02-17 02:42:12 - DEBUG - _trace.py:47 - receive_response_body.started request=<Request [b'POST']>
+2026-02-17 02:42:12 - DEBUG - _trace.py:47 - receive_response_body.complete
+2026-02-17 02:42:12 - DEBUG - _trace.py:47 - response_closed.started
+2026-02-17 02:42:12 - DEBUG - _trace.py:47 - response_closed.complete
+2026-02-17 02:42:12 - DEBUG - retrieve.py:212 - Generated embedding in 0.42s, dimension: 1024
+2026-02-17 02:42:12 - DEBUG - _trace.py:47 - send_request_headers.started request=<Request [b'POST']>
+2026-02-17 02:42:12 - DEBUG - _trace.py:47 - send_request_headers.complete
+2026-02-17 02:42:12 - DEBUG - _trace.py:47 - send_request_body.started request=<Request [b'POST']>
+2026-02-17 02:42:12 - DEBUG - _trace.py:47 - send_request_body.complete
+2026-02-17 02:42:12 - DEBUG - _trace.py:47 - receive_response_headers.started request=<Request [b'POST']>
+2026-02-17 02:42:12 - DEBUG - _trace.py:47 - receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Content-Encoding', b'gzip'), (b'Content-Type', b'application/json'), (b'Date', b'Mon, 16 Feb 2026 21:42:10 GMT'), (b'Vary', b'accept-encoding, Origin, Access-Control-Request-Method, Access-Control-Request-Headers'), (b'Transfer-Encoding', b'chunked')])
+2026-02-17 02:42:12 - INFO - _client.py:1025 - HTTP Request: POST https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333/collections/book_embeddings/points/query "HTTP/1.1 200 OK"
+2026-02-17 02:42:12 - DEBUG - _trace.py:47 - receive_response_body.started request=<Request [b'POST']>
+2026-02-17 02:42:12 - DEBUG - _trace.py:47 - receive_response_body.complete
+2026-02-17 02:42:12 - DEBUG - _trace.py:47 - response_closed.started
+2026-02-17 02:42:12 - DEBUG - _trace.py:47 - response_closed.complete
+2026-02-17 02:42:12 - INFO - retrieve.py:234 - Search completed in 0.31s, returned 5 results
+2026-02-17 02:42:12 - INFO - retrieve.py:249 - Total query time: 0.74s
+2026-02-17 02:42:12 - DEBUG - retrieve.py:127 - Metadata completeness: 5/5 = 100.0%
+2026-02-17 02:42:12 - DEBUG - retrieve.py:170 - Chunk indexing valid for 2 URLs
+2026-02-17 02:42:12 - INFO - retrieve.py:350 - Metadata completeness: 100.0%
+2026-02-17 02:42:12 - INFO - retrieve.py:351 - Chunk sequencing: VALID
+2026-02-17 02:42:12 - INFO - retrieve.py:352 - Validation result: PASS
+2026-02-17 02:42:12 - INFO - retrieve.py:379 - === Retrieval Pipeline Completed Successfully ===
+2026-02-17 02:42:12 - DEBUG - _trace.py:47 - close.started
+2026-02-17 02:42:12 - DEBUG - _trace.py:47 - close.complete
+2026-02-17 02:45:35 - INFO - retrieve.py:309 - === Retrieval Pipeline Started ===
+2026-02-17 02:45:35 - INFO - retrieve.py:313 - Loading config from .env
+2026-02-17 02:45:35 - INFO - retrieve.py:318 - Initializing Cohere and Qdrant clients
+2026-02-17 02:45:36 - DEBUG - _trace.py:47 - connect_tcp.started host='d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io' port=6333 local_address=None timeout=5.0 socket_options=None
+2026-02-17 02:45:36 - DEBUG - _trace.py:47 - connect_tcp.complete return_value=<httpcore._backends.sync.SyncStream object at 0x7b345d46efc0>
+2026-02-17 02:45:36 - DEBUG - _trace.py:47 - start_tls.started ssl_context=<ssl.SSLContext object at 0x7b345d2438d0> server_hostname='d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io' timeout=5.0
+2026-02-17 02:45:36 - DEBUG - _trace.py:47 - start_tls.complete return_value=<httpcore._backends.sync.SyncStream object at 0x7b345d632570>
+2026-02-17 02:45:36 - DEBUG - _trace.py:47 - send_request_headers.started request=<Request [b'GET']>
+2026-02-17 02:45:36 - DEBUG - _trace.py:47 - send_request_headers.complete
+2026-02-17 02:45:36 - DEBUG - _trace.py:47 - send_request_body.started request=<Request [b'GET']>
+2026-02-17 02:45:36 - DEBUG - _trace.py:47 - send_request_body.complete
+2026-02-17 02:45:36 - DEBUG - _trace.py:47 - receive_response_headers.started request=<Request [b'GET']>
+2026-02-17 02:45:36 - DEBUG - _trace.py:47 - receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Content-Encoding', b'gzip'), (b'Content-Type', b'application/json'), (b'Date', b'Mon, 16 Feb 2026 21:45:34 GMT'), (b'Vary', b'accept-encoding, Origin, Access-Control-Request-Method, Access-Control-Request-Headers'), (b'Transfer-Encoding', b'chunked')])
+2026-02-17 02:45:36 - INFO - _client.py:1025 - HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333 "HTTP/1.1 200 OK"
+2026-02-17 02:45:36 - DEBUG - _trace.py:47 - receive_response_body.started request=<Request [b'GET']>
+2026-02-17 02:45:36 - DEBUG - _trace.py:47 - receive_response_body.complete
+2026-02-17 02:45:36 - DEBUG - _trace.py:47 - response_closed.started
+2026-02-17 02:45:36 - DEBUG - _trace.py:47 - response_closed.complete
+2026-02-17 02:45:36 - DEBUG - _trace.py:47 - close.started
+2026-02-17 02:45:36 - DEBUG - _trace.py:47 - close.complete
+2026-02-17 02:45:36 - INFO - retrieve.py:323 - Checking collection 'book_embeddings'
+2026-02-17 02:45:36 - DEBUG - _trace.py:47 - connect_tcp.started host='d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io' port=6333 local_address=None timeout=5.0 socket_options=None
+2026-02-17 02:45:36 - DEBUG - _trace.py:47 - connect_tcp.complete return_value=<httpcore._backends.sync.SyncStream object at 0x7b345d26e000>
+2026-02-17 02:45:36 - DEBUG - _trace.py:47 - start_tls.started ssl_context=<ssl.SSLContext object at 0x7b345d2436d0> server_hostname='d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io' timeout=5.0
+2026-02-17 02:45:36 - DEBUG - _trace.py:47 - start_tls.complete return_value=<httpcore._backends.sync.SyncStream object at 0x7b345d26dee0>
+2026-02-17 02:45:36 - DEBUG - _trace.py:47 - send_request_headers.started request=<Request [b'GET']>
+2026-02-17 02:45:36 - DEBUG - _trace.py:47 - send_request_headers.complete
+2026-02-17 02:45:36 - DEBUG - _trace.py:47 - send_request_body.started request=<Request [b'GET']>
+2026-02-17 02:45:36 - DEBUG - _trace.py:47 - send_request_body.complete
+2026-02-17 02:45:36 - DEBUG - _trace.py:47 - receive_response_headers.started request=<Request [b'GET']>
+2026-02-17 02:45:37 - DEBUG - _trace.py:47 - receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Content-Encoding', b'gzip'), (b'Content-Type', b'application/json'), (b'Date', b'Mon, 16 Feb 2026 21:45:35 GMT'), (b'Vary', b'accept-encoding, Origin, Access-Control-Request-Method, Access-Control-Request-Headers'), (b'Transfer-Encoding', b'chunked')])
+2026-02-17 02:45:37 - INFO - _client.py:1025 - HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333/collections/book_embeddings "HTTP/1.1 200 OK"
+2026-02-17 02:45:37 - DEBUG - _trace.py:47 - receive_response_body.started request=<Request [b'GET']>
+2026-02-17 02:45:37 - DEBUG - _trace.py:47 - receive_response_body.complete
+2026-02-17 02:45:37 - DEBUG - _trace.py:47 - response_closed.started
+2026-02-17 02:45:37 - DEBUG - _trace.py:47 - response_closed.complete
+2026-02-17 02:45:37 - INFO - retrieve.py:325 - Collection OK: vector_size=1024, points=277
+2026-02-17 02:45:37 - DEBUG - _trace.py:47 - close.started
+2026-02-17 02:45:37 - DEBUG - _trace.py:47 - close.complete
+2026-02-17 02:45:53 - INFO - retrieve.py:309 - === Retrieval Pipeline Started ===
+2026-02-17 02:45:53 - INFO - retrieve.py:313 - Loading config from .env
+2026-02-17 02:45:53 - INFO - retrieve.py:318 - Initializing Cohere and Qdrant clients
+2026-02-17 02:45:54 - DEBUG - _trace.py:47 - connect_tcp.started host='d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io' port=6333 local_address=None timeout=5.0 socket_options=None
+2026-02-17 02:45:54 - DEBUG - _trace.py:47 - connect_tcp.complete return_value=<httpcore._backends.sync.SyncStream object at 0x77effa835ee0>
+2026-02-17 02:45:54 - DEBUG - _trace.py:47 - start_tls.started ssl_context=<ssl.SSLContext object at 0x77effa73b8d0> server_hostname='d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io' timeout=5.0
+2026-02-17 02:45:54 - DEBUG - _trace.py:47 - start_tls.complete return_value=<httpcore._backends.sync.SyncStream object at 0x77effab2a5d0>
+2026-02-17 02:45:54 - DEBUG - _trace.py:47 - send_request_headers.started request=<Request [b'GET']>
+2026-02-17 02:45:54 - DEBUG - _trace.py:47 - send_request_headers.complete
+2026-02-17 02:45:54 - DEBUG - _trace.py:47 - send_request_body.started request=<Request [b'GET']>
+2026-02-17 02:45:54 - DEBUG - _trace.py:47 - send_request_body.complete
+2026-02-17 02:45:54 - DEBUG - _trace.py:47 - receive_response_headers.started request=<Request [b'GET']>
+2026-02-17 02:45:54 - DEBUG - _trace.py:47 - receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Content-Encoding', b'gzip'), (b'Content-Type', b'application/json'), (b'Date', b'Mon, 16 Feb 2026 21:45:53 GMT'), (b'Vary', b'accept-encoding, Origin, Access-Control-Request-Method, Access-Control-Request-Headers'), (b'Transfer-Encoding', b'chunked')])
+2026-02-17 02:45:54 - INFO - _client.py:1025 - HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333 "HTTP/1.1 200 OK"
+2026-02-17 02:45:54 - DEBUG - _trace.py:47 - receive_response_body.started request=<Request [b'GET']>
+2026-02-17 02:45:54 - DEBUG - _trace.py:47 - receive_response_body.complete
+2026-02-17 02:45:54 - DEBUG - _trace.py:47 - response_closed.started
+2026-02-17 02:45:54 - DEBUG - _trace.py:47 - response_closed.complete
+2026-02-17 02:45:54 - DEBUG - _trace.py:47 - close.started
+2026-02-17 02:45:54 - DEBUG - _trace.py:47 - close.complete
+2026-02-17 02:45:54 - INFO - retrieve.py:323 - Checking collection 'book_embeddings'
+2026-02-17 02:45:54 - DEBUG - _trace.py:47 - connect_tcp.started host='d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io' port=6333 local_address=None timeout=5.0 socket_options=None
+2026-02-17 02:45:54 - DEBUG - _trace.py:47 - connect_tcp.complete return_value=<httpcore._backends.sync.SyncStream object at 0x77effa765fa0>
+2026-02-17 02:45:54 - DEBUG - _trace.py:47 - start_tls.started ssl_context=<ssl.SSLContext object at 0x77effa73b6d0> server_hostname='d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io' timeout=5.0
+2026-02-17 02:45:55 - DEBUG - _trace.py:47 - start_tls.complete return_value=<httpcore._backends.sync.SyncStream object at 0x77effa765ee0>
+2026-02-17 02:45:55 - DEBUG - _trace.py:47 - send_request_headers.started request=<Request [b'GET']>
+2026-02-17 02:45:55 - DEBUG - _trace.py:47 - send_request_headers.complete
+2026-02-17 02:45:55 - DEBUG - _trace.py:47 - send_request_body.started request=<Request [b'GET']>
+2026-02-17 02:45:55 - DEBUG - _trace.py:47 - send_request_body.complete
+2026-02-17 02:45:55 - DEBUG - _trace.py:47 - receive_response_headers.started request=<Request [b'GET']>
+2026-02-17 02:45:55 - DEBUG - _trace.py:47 - receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Content-Encoding', b'gzip'), (b'Content-Type', b'application/json'), (b'Date', b'Mon, 16 Feb 2026 21:45:54 GMT'), (b'Vary', b'accept-encoding, Origin, Access-Control-Request-Method, Access-Control-Request-Headers'), (b'Transfer-Encoding', b'chunked')])
+2026-02-17 02:45:55 - INFO - _client.py:1025 - HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333/collections/book_embeddings "HTTP/1.1 200 OK"
+2026-02-17 02:45:55 - DEBUG - _trace.py:47 - receive_response_body.started request=<Request [b'GET']>
+2026-02-17 02:45:55 - DEBUG - _trace.py:47 - receive_response_body.complete
+2026-02-17 02:45:55 - DEBUG - _trace.py:47 - response_closed.started
+2026-02-17 02:45:55 - DEBUG - _trace.py:47 - response_closed.complete
+2026-02-17 02:45:55 - INFO - retrieve.py:325 - Collection OK: vector_size=1024, points=277
+2026-02-17 02:45:55 - ERROR - retrieve.py:383 - Validation error: Query text must be non-empty
+2026-02-17 02:45:55 - DEBUG - _trace.py:47 - close.started
+2026-02-17 02:45:55 - DEBUG - _trace.py:47 - close.complete
+2026-02-17 02:46:17 - INFO - retrieve.py:309 - === Retrieval Pipeline Started ===
+2026-02-17 02:46:17 - INFO - retrieve.py:313 - Loading config from .env
+2026-02-17 02:46:17 - INFO - retrieve.py:318 - Initializing Cohere and Qdrant clients
+2026-02-17 02:46:17 - DEBUG - _trace.py:47 - connect_tcp.started host='d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io' port=6333 local_address=None timeout=5.0 socket_options=None
+2026-02-17 02:46:17 - DEBUG - _trace.py:47 - connect_tcp.complete return_value=<httpcore._backends.sync.SyncStream object at 0x7523b47f0380>
+2026-02-17 02:46:17 - DEBUG - _trace.py:47 - start_tls.started ssl_context=<ssl.SSLContext object at 0x7523b45938d0> server_hostname='d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io' timeout=5.0
+2026-02-17 02:46:17 - DEBUG - _trace.py:47 - start_tls.complete return_value=<httpcore._backends.sync.SyncStream object at 0x7523b45bd790>
+2026-02-17 02:46:17 - DEBUG - _trace.py:47 - send_request_headers.started request=<Request [b'GET']>
+2026-02-17 02:46:17 - DEBUG - _trace.py:47 - send_request_headers.complete
+2026-02-17 02:46:17 - DEBUG - _trace.py:47 - send_request_body.started request=<Request [b'GET']>
+2026-02-17 02:46:17 - DEBUG - _trace.py:47 - send_request_body.complete
+2026-02-17 02:46:17 - DEBUG - _trace.py:47 - receive_response_headers.started request=<Request [b'GET']>
+2026-02-17 02:46:18 - DEBUG - _trace.py:47 - receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Content-Encoding', b'gzip'), (b'Content-Type', b'application/json'), (b'Date', b'Mon, 16 Feb 2026 21:46:15 GMT'), (b'Vary', b'accept-encoding, Origin, Access-Control-Request-Method, Access-Control-Request-Headers'), (b'Transfer-Encoding', b'chunked')])
+2026-02-17 02:46:18 - INFO - _client.py:1025 - HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333 "HTTP/1.1 200 OK"
+2026-02-17 02:46:18 - DEBUG - _trace.py:47 - receive_response_body.started request=<Request [b'GET']>
+2026-02-17 02:46:18 - DEBUG - _trace.py:47 - receive_response_body.complete
+2026-02-17 02:46:18 - DEBUG - _trace.py:47 - response_closed.started
+2026-02-17 02:46:18 - DEBUG - _trace.py:47 - response_closed.complete
+2026-02-17 02:46:18 - DEBUG - _trace.py:47 - close.started
+2026-02-17 02:46:18 - DEBUG - _trace.py:47 - close.complete
+2026-02-17 02:46:18 - INFO - retrieve.py:323 - Checking collection 'book_embeddings'
+2026-02-17 02:46:18 - DEBUG - _trace.py:47 - connect_tcp.started host='d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io' port=6333 local_address=None timeout=5.0 socket_options=None
+2026-02-17 02:46:18 - DEBUG - _trace.py:47 - connect_tcp.complete return_value=<httpcore._backends.sync.SyncStream object at 0x7523b45bdf40>
+2026-02-17 02:46:18 - DEBUG - _trace.py:47 - start_tls.started ssl_context=<ssl.SSLContext object at 0x7523b45936d0> server_hostname='d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io' timeout=5.0
+2026-02-17 02:46:18 - DEBUG - _trace.py:47 - start_tls.complete return_value=<httpcore._backends.sync.SyncStream object at 0x7523b45bde80>
+2026-02-17 02:46:18 - DEBUG - _trace.py:47 - send_request_headers.started request=<Request [b'GET']>
+2026-02-17 02:46:18 - DEBUG - _trace.py:47 - send_request_headers.complete
+2026-02-17 02:46:18 - DEBUG - _trace.py:47 - send_request_body.started request=<Request [b'GET']>
+2026-02-17 02:46:18 - DEBUG - _trace.py:47 - send_request_body.complete
+2026-02-17 02:46:18 - DEBUG - _trace.py:47 - receive_response_headers.started request=<Request [b'GET']>
+2026-02-17 02:46:18 - DEBUG - _trace.py:47 - receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Content-Encoding', b'gzip'), (b'Content-Type', b'application/json'), (b'Date', b'Mon, 16 Feb 2026 21:46:15 GMT'), (b'Vary', b'accept-encoding, Origin, Access-Control-Request-Method, Access-Control-Request-Headers'), (b'Transfer-Encoding', b'chunked')])
+2026-02-17 02:46:18 - INFO - _client.py:1025 - HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333/collections/book_embeddings "HTTP/1.1 200 OK"
+2026-02-17 02:46:18 - DEBUG - _trace.py:47 - receive_response_body.started request=<Request [b'GET']>
+2026-02-17 02:46:18 - DEBUG - _trace.py:47 - receive_response_body.complete
+2026-02-17 02:46:18 - DEBUG - _trace.py:47 - response_closed.started
+2026-02-17 02:46:18 - DEBUG - _trace.py:47 - response_closed.complete
+2026-02-17 02:46:18 - INFO - retrieve.py:325 - Collection OK: vector_size=1024, points=277
+2026-02-17 02:46:18 - ERROR - retrieve.py:383 - Validation error: top_k must be between 1 and 100
+2026-02-17 02:46:18 - DEBUG - _trace.py:47 - close.started
+2026-02-17 02:46:18 - DEBUG - _trace.py:47 - close.complete
+2026-02-17 02:46:34 - INFO - retrieve.py:309 - === Retrieval Pipeline Started ===
+2026-02-17 02:46:34 - INFO - retrieve.py:313 - Loading config from .env
+2026-02-17 02:46:34 - INFO - retrieve.py:318 - Initializing Cohere and Qdrant clients
+2026-02-17 02:46:35 - DEBUG - _trace.py:47 - connect_tcp.started host='d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io' port=6333 local_address=None timeout=5.0 socket_options=None
+2026-02-17 02:46:35 - DEBUG - _trace.py:47 - connect_tcp.complete return_value=<httpcore._backends.sync.SyncStream object at 0x7dc43c5a6240>
+2026-02-17 02:46:35 - DEBUG - _trace.py:47 - start_tls.started ssl_context=<ssl.SSLContext object at 0x7dc43c22be50> server_hostname='d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io' timeout=5.0
+2026-02-17 02:46:35 - DEBUG - _trace.py:47 - start_tls.complete return_value=<httpcore._backends.sync.SyncStream object at 0x7dc43c2659a0>
+2026-02-17 02:46:35 - DEBUG - _trace.py:47 - send_request_headers.started request=<Request [b'GET']>
+2026-02-17 02:46:35 - DEBUG - _trace.py:47 - send_request_headers.complete
+2026-02-17 02:46:35 - DEBUG - _trace.py:47 - send_request_body.started request=<Request [b'GET']>
+2026-02-17 02:46:35 - DEBUG - _trace.py:47 - send_request_body.complete
+2026-02-17 02:46:35 - DEBUG - _trace.py:47 - receive_response_headers.started request=<Request [b'GET']>
+2026-02-17 02:46:35 - DEBUG - _trace.py:47 - receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Content-Encoding', b'gzip'), (b'Content-Type', b'application/json'), (b'Date', b'Mon, 16 Feb 2026 21:46:33 GMT'), (b'Vary', b'accept-encoding, Origin, Access-Control-Request-Method, Access-Control-Request-Headers'), (b'Transfer-Encoding', b'chunked')])
+2026-02-17 02:46:35 - INFO - _client.py:1025 - HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333 "HTTP/1.1 200 OK"
+2026-02-17 02:46:35 - DEBUG - _trace.py:47 - receive_response_body.started request=<Request [b'GET']>
+2026-02-17 02:46:35 - DEBUG - _trace.py:47 - receive_response_body.complete
+2026-02-17 02:46:35 - DEBUG - _trace.py:47 - response_closed.started
+2026-02-17 02:46:35 - DEBUG - _trace.py:47 - response_closed.complete
+2026-02-17 02:46:35 - DEBUG - _trace.py:47 - close.started
+2026-02-17 02:46:35 - DEBUG - _trace.py:47 - close.complete
+2026-02-17 02:46:35 - INFO - retrieve.py:323 - Checking collection 'book_embeddings'
+2026-02-17 02:46:35 - DEBUG - _trace.py:47 - connect_tcp.started host='d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io' port=6333 local_address=None timeout=5.0 socket_options=None
+2026-02-17 02:46:35 - DEBUG - _trace.py:47 - connect_tcp.complete return_value=<httpcore._backends.sync.SyncStream object at 0x7dc43c2661b0>
+2026-02-17 02:46:35 - DEBUG - _trace.py:47 - start_tls.started ssl_context=<ssl.SSLContext object at 0x7dc43c22bc50> server_hostname='d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io' timeout=5.0
+2026-02-17 02:46:35 - DEBUG - _trace.py:47 - start_tls.complete return_value=<httpcore._backends.sync.SyncStream object at 0x7dc43c266090>
+2026-02-17 02:46:35 - DEBUG - _trace.py:47 - send_request_headers.started request=<Request [b'GET']>
+2026-02-17 02:46:35 - DEBUG - _trace.py:47 - send_request_headers.complete
+2026-02-17 02:46:35 - DEBUG - _trace.py:47 - send_request_body.started request=<Request [b'GET']>
+2026-02-17 02:46:35 - DEBUG - _trace.py:47 - send_request_body.complete
+2026-02-17 02:46:35 - DEBUG - _trace.py:47 - receive_response_headers.started request=<Request [b'GET']>
+2026-02-17 02:46:36 - DEBUG - _trace.py:47 - receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Content-Encoding', b'gzip'), (b'Content-Type', b'application/json'), (b'Date', b'Mon, 16 Feb 2026 21:46:34 GMT'), (b'Vary', b'accept-encoding, Origin, Access-Control-Request-Method, Access-Control-Request-Headers'), (b'Transfer-Encoding', b'chunked')])
+2026-02-17 02:46:36 - INFO - _client.py:1025 - HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333/collections/book_embeddings "HTTP/1.1 200 OK"
+2026-02-17 02:46:36 - DEBUG - _trace.py:47 - receive_response_body.started request=<Request [b'GET']>
+2026-02-17 02:46:36 - DEBUG - _trace.py:47 - receive_response_body.complete
+2026-02-17 02:46:36 - DEBUG - _trace.py:47 - response_closed.started
+2026-02-17 02:46:36 - DEBUG - _trace.py:47 - response_closed.complete
+2026-02-17 02:46:36 - INFO - retrieve.py:325 - Collection OK: vector_size=1024, points=277
+2026-02-17 02:46:36 - ERROR - retrieve.py:383 - Validation error: top_k must be between 1 and 100
+2026-02-17 02:46:36 - DEBUG - _trace.py:47 - close.started
+2026-02-17 02:46:36 - DEBUG - _trace.py:47 - close.complete
+2026-02-17 02:46:56 - INFO - retrieve.py:309 - === Retrieval Pipeline Started ===
+2026-02-17 02:46:56 - INFO - retrieve.py:313 - Loading config from .env
+2026-02-17 02:46:56 - INFO - retrieve.py:318 - Initializing Cohere and Qdrant clients
+2026-02-17 02:46:57 - DEBUG - _trace.py:47 - connect_tcp.started host='d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io' port=6333 local_address=None timeout=5.0 socket_options=None
+2026-02-17 02:46:57 - DEBUG - _trace.py:47 - connect_tcp.complete return_value=<httpcore._backends.sync.SyncStream object at 0x7608387a20c0>
+2026-02-17 02:46:57 - DEBUG - _trace.py:47 - start_tls.started ssl_context=<ssl.SSLContext object at 0x7608384335d0> server_hostname='d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io' timeout=5.0
+2026-02-17 02:46:57 - DEBUG - _trace.py:47 - start_tls.complete return_value=<httpcore._backends.sync.SyncStream object at 0x76083845da00>
+2026-02-17 02:46:57 - DEBUG - _trace.py:47 - send_request_headers.started request=<Request [b'GET']>
+2026-02-17 02:46:57 - DEBUG - _trace.py:47 - send_request_headers.complete
+2026-02-17 02:46:57 - DEBUG - _trace.py:47 - send_request_body.started request=<Request [b'GET']>
+2026-02-17 02:46:57 - DEBUG - _trace.py:47 - send_request_body.complete
+2026-02-17 02:46:57 - DEBUG - _trace.py:47 - receive_response_headers.started request=<Request [b'GET']>
+2026-02-17 02:46:57 - DEBUG - _trace.py:47 - receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Content-Encoding', b'gzip'), (b'Content-Type', b'application/json'), (b'Date', b'Mon, 16 Feb 2026 21:46:56 GMT'), (b'Vary', b'accept-encoding, Origin, Access-Control-Request-Method, Access-Control-Request-Headers'), (b'Transfer-Encoding', b'chunked')])
+2026-02-17 02:46:57 - INFO - _client.py:1025 - HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333 "HTTP/1.1 200 OK"
+2026-02-17 02:46:57 - DEBUG - _trace.py:47 - receive_response_body.started request=<Request [b'GET']>
+2026-02-17 02:46:57 - DEBUG - _trace.py:47 - receive_response_body.complete
+2026-02-17 02:46:57 - DEBUG - _trace.py:47 - response_closed.started
+2026-02-17 02:46:57 - DEBUG - _trace.py:47 - response_closed.complete
+2026-02-17 02:46:57 - DEBUG - _trace.py:47 - close.started
+2026-02-17 02:46:57 - DEBUG - _trace.py:47 - close.complete
+2026-02-17 02:46:57 - INFO - retrieve.py:323 - Checking collection 'book_embeddings'
+2026-02-17 02:46:57 - DEBUG - _trace.py:47 - connect_tcp.started host='d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io' port=6333 local_address=None timeout=5.0 socket_options=None
+2026-02-17 02:46:57 - DEBUG - _trace.py:47 - connect_tcp.complete return_value=<httpcore._backends.sync.SyncStream object at 0x76083845e2a0>
+2026-02-17 02:46:58 - DEBUG - _trace.py:47 - start_tls.started ssl_context=<ssl.SSLContext object at 0x7608384333d0> server_hostname='d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io' timeout=5.0
+2026-02-17 02:46:58 - DEBUG - _trace.py:47 - start_tls.complete return_value=<httpcore._backends.sync.SyncStream object at 0x76083845e180>
+2026-02-17 02:46:58 - DEBUG - _trace.py:47 - send_request_headers.started request=<Request [b'GET']>
+2026-02-17 02:46:58 - DEBUG - _trace.py:47 - send_request_headers.complete
+2026-02-17 02:46:58 - DEBUG - _trace.py:47 - send_request_body.started request=<Request [b'GET']>
+2026-02-17 02:46:58 - DEBUG - _trace.py:47 - send_request_body.complete
+2026-02-17 02:46:58 - DEBUG - _trace.py:47 - receive_response_headers.started request=<Request [b'GET']>
+2026-02-17 02:46:58 - DEBUG - _trace.py:47 - receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Content-Encoding', b'gzip'), (b'Content-Type', b'application/json'), (b'Date', b'Mon, 16 Feb 2026 21:46:57 GMT'), (b'Vary', b'accept-encoding, Origin, Access-Control-Request-Method, Access-Control-Request-Headers'), (b'Transfer-Encoding', b'chunked')])
+2026-02-17 02:46:58 - INFO - _client.py:1025 - HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333/collections/book_embeddings "HTTP/1.1 200 OK"
+2026-02-17 02:46:58 - DEBUG - _trace.py:47 - receive_response_body.started request=<Request [b'GET']>
+2026-02-17 02:46:58 - DEBUG - _trace.py:47 - receive_response_body.complete
+2026-02-17 02:46:58 - DEBUG - _trace.py:47 - response_closed.started
+2026-02-17 02:46:58 - DEBUG - _trace.py:47 - response_closed.complete
+2026-02-17 02:46:58 - INFO - retrieve.py:325 - Collection OK: vector_size=1024, points=277
+2026-02-17 02:46:58 - INFO - retrieve.py:200 - Embedding query: 'asdfghjkl1234567890xyz_nonexistent_query_should_return_zero_results...' (top_k=5)
+2026-02-17 02:46:58 - DEBUG - _trace.py:47 - connect_tcp.started host='api.cohere.com' port=443 local_address=None timeout=300 socket_options=None
+2026-02-17 02:46:58 - DEBUG - _trace.py:47 - connect_tcp.complete return_value=<httpcore._backends.sync.SyncStream object at 0x76083845f230>
+2026-02-17 02:46:58 - DEBUG - _trace.py:47 - start_tls.started ssl_context=<ssl.SSLContext object at 0x760838432cd0> server_hostname='api.cohere.com' timeout=300
+2026-02-17 02:46:58 - DEBUG - _trace.py:47 - start_tls.complete return_value=<httpcore._backends.sync.SyncStream object at 0x76083845f140>
+2026-02-17 02:46:58 - DEBUG - _trace.py:47 - send_request_headers.started request=<Request [b'POST']>
+2026-02-17 02:46:58 - DEBUG - _trace.py:47 - send_request_headers.complete
+2026-02-17 02:46:58 - DEBUG - _trace.py:47 - send_request_body.started request=<Request [b'POST']>
+2026-02-17 02:46:58 - DEBUG - _trace.py:47 - send_request_body.complete
+2026-02-17 02:46:58 - DEBUG - _trace.py:47 - receive_response_headers.started request=<Request [b'POST']>
+2026-02-17 02:46:58 - DEBUG - _trace.py:47 - receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'access-control-expose-headers', b'X-Debug-Trace-ID'), (b'cache-control', b'no-cache, no-store, no-transform, must-revalidate, private, max-age=0'), (b'content-encoding', b'gzip'), (b'content-type', b'application/json'), (b'expires', b'Thu, 01 Jan 1970 00:00:00 GMT'), (b'num_chars', b'67'), (b'num_tokens', b'28'), (b'pragma', b'no-cache'), (b'vary', b'Origin,Accept-Encoding'), (b'x-accel-expires', b'0'), (b'x-debug-trace-id', b'fd1019b199609124264ce2207ddc1acf'), (b'x-endpoint-monthly-call-limit', b'1000'), (b'x-trial-endpoint-call-limit', b'100'), (b'x-trial-endpoint-call-remaining', b'99'), (b'date', b'Mon, 16 Feb 2026 21:46:57 GMT'), (b'x-envoy-upstream-service-time', b'56'), (b'server', b'envoy'), (b'Via', b'1.1 google'), (b'Alt-Svc', b'h3=":443"; ma=2592000,h3-29=":443"; ma=2592000'), (b'Transfer-Encoding', b'chunked')])
+2026-02-17 02:46:58 - INFO - _client.py:1025 - HTTP Request: POST https://api.cohere.com/v2/embed "HTTP/1.1 200 OK"
+2026-02-17 02:46:58 - DEBUG - _trace.py:47 - receive_response_body.started request=<Request [b'POST']>
+2026-02-17 02:46:58 - DEBUG - _trace.py:47 - receive_response_body.complete
+2026-02-17 02:46:58 - DEBUG - _trace.py:47 - response_closed.started
+2026-02-17 02:46:58 - DEBUG - _trace.py:47 - response_closed.complete
+2026-02-17 02:46:58 - DEBUG - retrieve.py:212 - Generated embedding in 0.46s, dimension: 1024
+2026-02-17 02:46:58 - DEBUG - _trace.py:47 - send_request_headers.started request=<Request [b'POST']>
+2026-02-17 02:46:58 - DEBUG - _trace.py:47 - send_request_headers.complete
+2026-02-17 02:46:58 - DEBUG - _trace.py:47 - send_request_body.started request=<Request [b'POST']>
+2026-02-17 02:46:58 - DEBUG - _trace.py:47 - send_request_body.complete
+2026-02-17 02:46:58 - DEBUG - _trace.py:47 - receive_response_headers.started request=<Request [b'POST']>
+2026-02-17 02:46:59 - DEBUG - _trace.py:47 - receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Content-Encoding', b'gzip'), (b'Content-Type', b'application/json'), (b'Date', b'Mon, 16 Feb 2026 21:46:57 GMT'), (b'Vary', b'accept-encoding, Origin, Access-Control-Request-Method, Access-Control-Request-Headers'), (b'Transfer-Encoding', b'chunked')])
+2026-02-17 02:46:59 - INFO - _client.py:1025 - HTTP Request: POST https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333/collections/book_embeddings/points/query "HTTP/1.1 200 OK"
+2026-02-17 02:46:59 - DEBUG - _trace.py:47 - receive_response_body.started request=<Request [b'POST']>
+2026-02-17 02:46:59 - DEBUG - _trace.py:47 - receive_response_body.complete
+2026-02-17 02:46:59 - DEBUG - _trace.py:47 - response_closed.started
+2026-02-17 02:46:59 - DEBUG - _trace.py:47 - response_closed.complete
+2026-02-17 02:46:59 - INFO - retrieve.py:234 - Search completed in 0.30s, returned 5 results
+2026-02-17 02:46:59 - INFO - retrieve.py:249 - Total query time: 0.76s
+2026-02-17 02:46:59 - INFO - retrieve.py:379 - === Retrieval Pipeline Completed Successfully ===
+2026-02-17 02:46:59 - DEBUG - _trace.py:47 - close.started
+2026-02-17 02:46:59 - DEBUG - _trace.py:47 - close.complete
+2026-02-17 03:15:14 - INFO - retrieve.py:309 - === Retrieval Pipeline Started ===
+2026-02-17 03:15:14 - INFO - retrieve.py:313 - Loading config from .env
+2026-02-17 03:15:14 - INFO - retrieve.py:318 - Initializing Cohere and Qdrant clients
+2026-02-17 03:15:15 - DEBUG - _trace.py:47 - connect_tcp.started host='d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io' port=6333 local_address=None timeout=5.0 socket_options=None
+2026-02-17 03:15:15 - DEBUG - _trace.py:47 - connect_tcp.complete return_value=<httpcore._backends.sync.SyncStream object at 0x7960c15697f0>
+2026-02-17 03:15:15 - DEBUG - _trace.py:47 - start_tls.started ssl_context=<ssl.SSLContext object at 0x7960c1543850> server_hostname='d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io' timeout=5.0
+2026-02-17 03:15:15 - DEBUG - _trace.py:47 - start_tls.complete return_value=<httpcore._backends.sync.SyncStream object at 0x7960c1569730>
+2026-02-17 03:15:15 - DEBUG - _trace.py:47 - send_request_headers.started request=<Request [b'GET']>
+2026-02-17 03:15:15 - DEBUG - _trace.py:47 - send_request_headers.complete
+2026-02-17 03:15:15 - DEBUG - _trace.py:47 - send_request_body.started request=<Request [b'GET']>
+2026-02-17 03:15:15 - DEBUG - _trace.py:47 - send_request_body.complete
+2026-02-17 03:15:15 - DEBUG - _trace.py:47 - receive_response_headers.started request=<Request [b'GET']>
+2026-02-17 03:15:16 - DEBUG - _trace.py:47 - receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Content-Encoding', b'gzip'), (b'Content-Type', b'application/json'), (b'Date', b'Mon, 16 Feb 2026 22:15:14 GMT'), (b'Vary', b'accept-encoding, Origin, Access-Control-Request-Method, Access-Control-Request-Headers'), (b'Transfer-Encoding', b'chunked')])
+2026-02-17 03:15:16 - INFO - _client.py:1025 - HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333 "HTTP/1.1 200 OK"
+2026-02-17 03:15:16 - DEBUG - _trace.py:47 - receive_response_body.started request=<Request [b'GET']>
+2026-02-17 03:15:16 - DEBUG - _trace.py:47 - receive_response_body.complete
+2026-02-17 03:15:16 - DEBUG - _trace.py:47 - response_closed.started
+2026-02-17 03:15:16 - DEBUG - _trace.py:47 - response_closed.complete
+2026-02-17 03:15:16 - DEBUG - _trace.py:47 - close.started
+2026-02-17 03:15:16 - DEBUG - _trace.py:47 - close.complete
+2026-02-17 03:15:16 - INFO - retrieve.py:323 - Checking collection 'book_embeddings'
+2026-02-17 03:15:16 - DEBUG - _trace.py:47 - connect_tcp.started host='d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io' port=6333 local_address=None timeout=5.0 socket_options=None
+2026-02-17 03:15:16 - DEBUG - _trace.py:47 - connect_tcp.complete return_value=<httpcore._backends.sync.SyncStream object at 0x7960c1569df0>
+2026-02-17 03:15:16 - DEBUG - _trace.py:47 - start_tls.started ssl_context=<ssl.SSLContext object at 0x7960c1543650> server_hostname='d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io' timeout=5.0
+2026-02-17 03:15:16 - DEBUG - _trace.py:47 - start_tls.complete return_value=<httpcore._backends.sync.SyncStream object at 0x7960c1569b80>
+2026-02-17 03:15:16 - DEBUG - _trace.py:47 - send_request_headers.started request=<Request [b'GET']>
+2026-02-17 03:15:16 - DEBUG - _trace.py:47 - send_request_headers.complete
+2026-02-17 03:15:16 - DEBUG - _trace.py:47 - send_request_body.started request=<Request [b'GET']>
+2026-02-17 03:15:16 - DEBUG - _trace.py:47 - send_request_body.complete
+2026-02-17 03:15:16 - DEBUG - _trace.py:47 - receive_response_headers.started request=<Request [b'GET']>
+2026-02-17 03:15:16 - DEBUG - _trace.py:47 - receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Content-Encoding', b'gzip'), (b'Content-Type', b'application/json'), (b'Date', b'Mon, 16 Feb 2026 22:15:15 GMT'), (b'Vary', b'accept-encoding, Origin, Access-Control-Request-Method, Access-Control-Request-Headers'), (b'Transfer-Encoding', b'chunked')])
+2026-02-17 03:15:16 - INFO - _client.py:1025 - HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333/collections/book_embeddings "HTTP/1.1 200 OK"
+2026-02-17 03:15:16 - DEBUG - _trace.py:47 - receive_response_body.started request=<Request [b'GET']>
+2026-02-17 03:15:16 - DEBUG - _trace.py:47 - receive_response_body.complete
+2026-02-17 03:15:16 - DEBUG - _trace.py:47 - response_closed.started
+2026-02-17 03:15:16 - DEBUG - _trace.py:47 - response_closed.complete
+2026-02-17 03:15:16 - INFO - retrieve.py:325 - Collection OK: vector_size=1024, points=277
+2026-02-17 03:15:16 - INFO - retrieve.py:200 - Embedding query: 'ROS 2...' (top_k=5)
+2026-02-17 03:15:16 - DEBUG - _trace.py:47 - connect_tcp.started host='api.cohere.com' port=443 local_address=None timeout=300 socket_options=None
+2026-02-17 03:15:16 - DEBUG - _trace.py:47 - connect_tcp.complete return_value=<httpcore._backends.sync.SyncStream object at 0x7960c156ad20>
+2026-02-17 03:15:16 - DEBUG - _trace.py:47 - start_tls.started ssl_context=<ssl.SSLContext object at 0x7960c1542f50> server_hostname='api.cohere.com' timeout=300
+2026-02-17 03:15:16 - DEBUG - _trace.py:47 - start_tls.complete return_value=<httpcore._backends.sync.SyncStream object at 0x7960c156ac30>
+2026-02-17 03:15:16 - DEBUG - _trace.py:47 - send_request_headers.started request=<Request [b'POST']>
+2026-02-17 03:15:16 - DEBUG - _trace.py:47 - send_request_headers.complete
+2026-02-17 03:15:16 - DEBUG - _trace.py:47 - send_request_body.started request=<Request [b'POST']>
+2026-02-17 03:15:16 - DEBUG - _trace.py:47 - send_request_body.complete
+2026-02-17 03:15:16 - DEBUG - _trace.py:47 - receive_response_headers.started request=<Request [b'POST']>
+2026-02-17 03:15:16 - DEBUG - _trace.py:47 - receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'access-control-expose-headers', b'X-Debug-Trace-ID'), (b'cache-control', b'no-cache, no-store, no-transform, must-revalidate, private, max-age=0'), (b'content-encoding', b'gzip'), (b'content-type', b'application/json'), (b'expires', b'Thu, 01 Jan 1970 00:00:00 GMT'), (b'num_chars', b'5'), (b'num_tokens', b'3'), (b'pragma', b'no-cache'), (b'vary', b'Origin,Accept-Encoding'), (b'x-accel-expires', b'0'), (b'x-debug-trace-id', b'4df4171dafe6d9e613e8c5e32b7ae73b'), (b'x-endpoint-monthly-call-limit', b'1000'), (b'x-trial-endpoint-call-limit', b'100'), (b'x-trial-endpoint-call-remaining', b'99'), (b'date', b'Mon, 16 Feb 2026 22:15:15 GMT'), (b'x-envoy-upstream-service-time', b'42'), (b'server', b'envoy'), (b'Via', b'1.1 google'), (b'Alt-Svc', b'h3=":443"; ma=2592000,h3-29=":443"; ma=2592000'), (b'Transfer-Encoding', b'chunked')])
+2026-02-17 03:15:16 - INFO - _client.py:1025 - HTTP Request: POST https://api.cohere.com/v2/embed "HTTP/1.1 200 OK"
+2026-02-17 03:15:16 - DEBUG - _trace.py:47 - receive_response_body.started request=<Request [b'POST']>
+2026-02-17 03:15:16 - DEBUG - _trace.py:47 - receive_response_body.complete
+2026-02-17 03:15:16 - DEBUG - _trace.py:47 - response_closed.started
+2026-02-17 03:15:16 - DEBUG - _trace.py:47 - response_closed.complete
+2026-02-17 03:15:16 - DEBUG - retrieve.py:212 - Generated embedding in 0.49s, dimension: 1024
+2026-02-17 03:15:16 - DEBUG - _trace.py:47 - send_request_headers.started request=<Request [b'POST']>
+2026-02-17 03:15:16 - DEBUG - _trace.py:47 - send_request_headers.complete
+2026-02-17 03:15:16 - DEBUG - _trace.py:47 - send_request_body.started request=<Request [b'POST']>
+2026-02-17 03:15:16 - DEBUG - _trace.py:47 - send_request_body.complete
+2026-02-17 03:15:16 - DEBUG - _trace.py:47 - receive_response_headers.started request=<Request [b'POST']>
+2026-02-17 03:15:17 - DEBUG - _trace.py:47 - receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Content-Encoding', b'gzip'), (b'Content-Type', b'application/json'), (b'Date', b'Mon, 16 Feb 2026 22:15:15 GMT'), (b'Vary', b'accept-encoding, Origin, Access-Control-Request-Method, Access-Control-Request-Headers'), (b'Transfer-Encoding', b'chunked')])
+2026-02-17 03:15:17 - INFO - _client.py:1025 - HTTP Request: POST https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333/collections/book_embeddings/points/query "HTTP/1.1 200 OK"
+2026-02-17 03:15:17 - DEBUG - _trace.py:47 - receive_response_body.started request=<Request [b'POST']>
+2026-02-17 03:15:17 - DEBUG - _trace.py:47 - receive_response_body.complete
+2026-02-17 03:15:17 - DEBUG - _trace.py:47 - response_closed.started
+2026-02-17 03:15:17 - DEBUG - _trace.py:47 - response_closed.complete
+2026-02-17 03:15:17 - INFO - retrieve.py:234 - Search completed in 0.31s, returned 5 results
+2026-02-17 03:15:17 - INFO - retrieve.py:249 - Total query time: 0.80s
+2026-02-17 03:15:17 - INFO - retrieve.py:379 - === Retrieval Pipeline Completed Successfully ===
+2026-02-17 03:15:17 - DEBUG - _trace.py:47 - close.started
+2026-02-17 03:15:17 - DEBUG - _trace.py:47 - close.complete

retrieve.py

@@ -0,0 +1,408 @@
+"""
+Retrieval pipeline for RAG validation.
+
+This module provides functions to:
+- Convert search queries to embeddings using Cohere
+- Perform similarity search against Qdrant collection
+- Format and return results with metadata
+"""
+import argparse
+import json
+import sys
+import time
+import logging
+from pathlib import Path
+from typing import List, Dict, Any
+
+# Add parent directory to path for imports
+sys.path.insert(0, str(Path(__file__).parent))
+
+import cohere
+from qdrant_client import QdrantClient
+
+# Importfrom existing modules
+import config
+import utils
+from logging_config import setup_logging
+
+# Initialize logger
+logger = logging.getLogger(__name__)
+
+# Custom exceptions
+class ConfigurationError(Exception):
+    """Raised when required configuration is missing."""
+    pass
+
+class CollectionNotFoundError(Exception):
+    """Raised when Qdrant collection doesn't exist."""
+    pass
+
+class DimensionMismatchError(Exception):
+    """Raised when embedding dimension doesn't match collection."""
+    pass
+
+class APIError(Exception):
+    """Raised when Cohere or Qdrant API call fails after retries."""
+    pass
+
+def validate_config(cfg: dict) -> None:
+    """Validate that all required config values are present."""
+    required = ["cohere_api_key", "qdrant_url", "qdrant_api_key"]
+    missing = [key for key in required if not cfg.get(key)]
+    if missing:
+        raise ConfigurationError(f"Missing required environment variables: {', '.join(missing)}")
+
+def init_clients(cfg: dict):
+    """Initialize Cohere and Qdrant clients."""
+    cohere_client = cohere.ClientV2(api_key=cfg["cohere_api_key"])
+    qdrant_client = QdrantClient(url=cfg["qdrant_url"], api_key=cfg["qdrant_api_key"])
+    return cohere_client, qdrant_client
+
+def check_collection(qdrant_client: QdrantClient, collection_name: str) -> Dict[str, Any]:
+    """Verify collection exists and has correct vector size."""
+    try:
+        info = qdrant_client.get_collection(collection_name)
+    except Exception as e:
+        if "not found" in str(e).lower():
+            raise CollectionNotFoundError(f"Collection '{collection_name}' does not exist")
+        raise
+
+    vector_size = info.config.params.vectors.size
+    if vector_size != 1024:
+        raise DimensionMismatchError(f"Expected vector size 1024 but got {vector_size}")
+
+    return {
+        "exists": True,
+        "vector_size": vector_size,
+        "points_count": info.points_count
+    }
+
+def embed_query(text: str, cohere_client: cohere.ClientV2) -> List[float]:
+    """Generate embedding for a search query using Cohere."""
+    try:
+        response = cohere_client.embed(
+            texts=[text],
+            model="embed-english-v3.0",
+            input_type="search_query"
+        )
+        # Extract embedding from response.embeddings.float_
+        embedding = response.embeddings.float_[0]
+        return embedding
+    except Exception as e:
+        logger.error(f"Failed to generate embedding: {e}")
+        raise APIError(f"Cohere embedding failed: {e}")
+
+def validate_metadata_completeness(results: List[Dict[str, Any]]) -> float:
+    """
+    Check metadata completeness in search results.
+
+    Returns:
+        Percentage (0-100) of results with complete metadata:
+        - url present and non-empty
+        - text present with length ≥ 10
+        - at least one of title or section non-empty
+    """
+    if not results:
+        return 0.0
+
+    complete = 0
+    total = len(results)
+
+    for result in results:
+        payload = result.get('payload', {})
+        url = payload.get('url', '')
+        text = payload.get('text', '')
+        title = payload.get('title', '')
+        section = payload.get('section', '')
+
+        # Check completeness criteria
+        url_ok = bool(url and url.strip())
+        text_ok = len(text or '') >= 10
+        title_section_ok = bool((title and title.strip()) or (section and section.strip()))
+
+        if url_ok and text_ok and title_section_ok:
+            complete += 1
+
+    percentage = (complete / total) * 100
+    logger.debug(f"Metadata completeness: {complete}/{total} = {percentage:.1f}%")
+    return percentage
+
+
+def validate_chunk_sequencing(results: List[Dict[str, Any]]) -> bool:
+    """
+    Verify that chunk_index values are properly assigned: integers >= 0 and unique per URL.
+
+    Note: Since search may return only a subset of chunks for a URL, we cannot
+    verify full sequential continuity (0,1,2,3...). Instead we check:
+    - All chunk_index values are integers >= 0
+    - No duplicate chunk_index for the same URL in the result set
+
+    Args:
+        results: List of search results
+
+    Returns:
+        True if chunk indices are valid, False otherwise
+    """
+    # Group by URL
+    url_chunks = {}
+    for result in results:
+        payload = result.get('payload', {})
+        url = payload.get('url', '')
+        chunk_idx = payload.get('chunk_index')
+
+        if url not in url_chunks:
+            url_chunks[url] = []
+        url_chunks[url].append(chunk_idx)
+
+    # Check each URL's chunks are valid
+    for url, indices in url_chunks.items():
+        # All indices must be integers >= 0
+        for idx in indices:
+            if not isinstance(idx, int) or idx < 0:
+                logger.debug(f"Invalid chunk_index for {url}: {idx} (must be non-negative integer)")
+                return False
+
+        # Check for duplicates (within this URL's results)
+        if len(set(indices)) != len(indices):
+            logger.debug(f"Duplicate chunk_index for {url}: {indices}")
+            return False
+
+    logger.debug(f"Chunk indexing valid for {len(url_chunks)} URLs")
+    return True
+
+
+def search(
+    query_text: str,
+    cohere_client: cohere.ClientV2,
+    qdrant_client: QdrantClient,
+    collection_name: str,
+    top_k: int = 5
+) -> List[Dict[str, Any]]:
+    """
+    Convert query to embedding and retrieve top-K relevant chunks.
+
+    Args:
+        query_text: User's search query (non-empty, ≤1000 chars)
+        top_k: Number of results to return (1-100)
+
+    Returns:
+        List of search results with id, score, and payload
+    """
+    # Validate inputs
+    if not query_text or not query_text.strip():
+        raise ValueError("Query text must be non-empty")
+    query_text = query_text.strip()
+    if len(query_text) > 1000:
+        raise ValueError("Query text must be ≤ 1000 characters")
+    if top_k < 1 or top_k > 100:
+        raise ValueError("top_k must be between 1 and 100")
+
+    logger.info(f"Embedding query: '{query_text[:100]}...' (top_k={top_k})")
+    start_time = time.time()
+
+    # Generate query embedding with retry
+    try:
+        embedding = utils.retry_with_backoff(
+            lambda: embed_query(query_text, cohere_client),
+            max_retries=3,
+            base_delay=1.0,
+            max_delay=10.0
+        )
+        embed_time = time.time() - start_time
+        logger.debug(f"Generated embedding in {embed_time:.2f}s, dimension: {len(embedding)}")
+    except Exception as e:
+        logger.error(f"Failed to embed query: {e}")
+        raise
+
+    # Search Qdrant with retry
+    try:
+        search_start = time.time()
+        response = utils.retry_with_backoff(
+            lambda: qdrant_client.query_points(
+                collection_name=collection_name,
+                query=embedding,
+                limit=top_k,
+                with_payload=True,
+                with_vectors=False
+            ),
+            max_retries=3,
+            base_delay=1.0,
+            max_delay=10.0
+        )
+        results = response.points
+        search_time = time.time() - search_start
+        logger.info(f"Search completed in {search_time:.2f}s, returned {len(results)} results")
+    except Exception as e:
+        logger.error(f"Search failed: {e}")
+        raise APIError(f"Qdrant search failed: {e}")
+
+    # Format results
+    formatted = []
+    for result in results:
+        formatted.append({
+            "id": str(result.id),
+            "score": float(result.score),
+            "payload": result.payload
+        })
+
+    total_time = time.time() - start_time
+    logger.info(f"Total query time: {total_time:.2f}s")
+
+    return formatted
+
+def format_results(
+    results: List[Dict[str, Any]],
+    query: str,
+    latency_ms: int
+) -> Dict[str, Any]:
+    """Format search results into JSON output structure."""
+    output = {
+        "query": query,
+        "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        "results": results,
+        "metadata": {
+            "total_results": len(results),
+            "collection": None,  # Will be filled by main
+            "latency_ms": latency_ms
+        }
+    }
+    return output
+
+def main() -> int:
+    """CLI entrypoint for retrieval."""
+    parser = argparse.ArgumentParser(
+        description="Retrieve relevant chunks from Qdrant using Cohere embeddings"
+    )
+    parser.add_argument(
+        "--query",
+        type=str,
+        help="Search query text"
+    )
+    parser.add_argument(
+        "--top-k",
+        type=int,
+        default=5,
+        help="Number of results to return (default: 5)"
+    )
+    parser.add_argument(
+        "--output",
+        type=str,
+        help="Output file path (default: stdout)"
+    )
+    parser.add_argument(
+        "--config",
+        type=str,
+        default=".env",
+        help="Path to .env config file (default: .env)"
+    )
+    parser.add_argument(
+        "--validate-metadata",
+        action="store_true",
+        help="Run metadata validation on search results (requires --query)"
+    )
+
+    args = parser.parse_args()
+
+    # Setup logging
+    log_file = "retrieve.log"
+    setup_logging(log_file=log_file, console_level="INFO")
+    logger.info("=== Retrieval Pipeline Started ===")
+
+    try:
+        # Load config
+        logger.info(f"Loading config from {args.config}")
+        cfg = config.get_config()
+        validate_config(cfg)
+
+        # Initialize clients
+        logger.info("Initializing Cohere and Qdrant clients")
+        cohere_client, qdrant_client = init_clients(cfg)
+
+        # Check collection
+        collection_name = cfg["qdrant_collection"]
+        logger.info(f"Checking collection '{collection_name}'")
+        coll_info = check_collection(qdrant_client, collection_name)
+        logger.info(f"Collection OK: vector_size={coll_info['vector_size']}, points={coll_info['points_count']}")
+
+        # Validate query argument
+        if not args.query:
+            parser.error("--query is required")
+
+        # Perform search
+        results = search(
+            query_text=args.query,
+            cohere_client=cohere_client,
+            qdrant_client=qdrant_client,
+            collection_name=collection_name,
+            top_k=args.top_k
+        )
+
+        # Perform metadata validation if requested
+        metadata_validation = None
+        if args.validate_metadata:
+            completeness = validate_metadata_completeness(results)
+            sequencing = validate_chunk_sequencing(results)
+            metadata_validation = {
+                "completeness_pct": round(completeness, 2),
+                "sequencing_valid": sequencing,
+                "pass": completeness >= 98.0 and sequencing
+            }
+            logger.info(f"Metadata completeness: {completeness:.1f}%")
+            logger.info(f"Chunk sequencing: {'VALID' if sequencing else 'INVALID'}")
+            logger.info(f"Validation result: {'PASS' if metadata_validation['pass'] else 'FAIL'}")
+
+        # Format output
+        output = {
+            "query": args.query,
+            "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+            "results": results,
+            "metadata": {
+                "total_results": len(results),
+                "collection": collection_name,
+                "vector_size": coll_info['vector_size'],
+                "points_count": coll_info['points_count']
+            }
+        }
+
+        if metadata_validation:
+            output["metadata_validation"] = metadata_validation
+
+        # Output JSON
+        json_output = json.dumps(output, indent=2)
+        if args.output:
+            with open(args.output, 'w') as f:
+                f.write(json_output)
+            logger.info(f"Results written to {args.output}")
+        else:
+            print(json_output)
+
+        logger.info("=== Retrieval Pipeline Completed Successfully ===")
+        return 0
+
+    except ValueError as ve:
+        logger.error(f"Validation error: {ve}")
+        print(f"ERROR: {ve}", file=sys.stderr)
+        return 2
+    except ConfigurationError as ce:
+        logger.error(f"Configuration error: {ce}")
+        print(f"ERROR: {ce}", file=sys.stderr)
+        return 1
+    except CollectionNotFoundError as cnfe:
+        logger.error(f"Collection error: {cnfe}")
+        print(f"ERROR: {cnfe}", file=sys.stderr)
+        return 1
+    except DimensionMismatchError as dme:
+        logger.error(f"Dimension error: {dme}")
+        print(f"ERROR: {dme}", file=sys.stderr)
+        return 1
+    except APIError as api_err:
+        logger.error(f"API error: {api_err}")
+        print(f"ERROR: {api_err}", file=sys.stderr)
+        return 1
+    except Exception as e:
+        logger.exception(f"Unexpected error: {e}")
+        print(f"ERROR: Unexpected error: {e}", file=sys.stderr)
+        return 1
+
+if __name__ == "__main__":
+    sys.exit(main())

sitemap_urls.txt

@@ -0,0 +1,40 @@
+https://humanoid-ai-robotics-book-1.vercel.app/blog
+https://humanoid-ai-robotics-book-1.vercel.app/blog/archive
+https://humanoid-ai-robotics-book-1.vercel.app/blog/authors
+https://humanoid-ai-robotics-book-1.vercel.app/blog/authors/all-sebastien-lorber-articles
+https://humanoid-ai-robotics-book-1.vercel.app/blog/authors/yangshun
+https://humanoid-ai-robotics-book-1.vercel.app/blog/first-blog-post
+https://humanoid-ai-robotics-book-1.vercel.app/blog/long-blog-post
+https://humanoid-ai-robotics-book-1.vercel.app/blog/mdx-blog-post
+https://humanoid-ai-robotics-book-1.vercel.app/blog/tags
+https://humanoid-ai-robotics-book-1.vercel.app/blog/tags/docusaurus
+https://humanoid-ai-robotics-book-1.vercel.app/blog/tags/facebook
+https://humanoid-ai-robotics-book-1.vercel.app/blog/tags/hello
+https://humanoid-ai-robotics-book-1.vercel.app/blog/tags/hola
+https://humanoid-ai-robotics-book-1.vercel.app/blog/welcome
+https://humanoid-ai-robotics-book-1.vercel.app/markdown-page
+https://humanoid-ai-robotics-book-1.vercel.app/docs/category/tutorial---basics
+https://humanoid-ai-robotics-book-1.vercel.app/docs/category/tutorial---extras
+https://humanoid-ai-robotics-book-1.vercel.app/docs/intro
+https://humanoid-ai-robotics-book-1.vercel.app/docs/module-1/intro-to-ros2
+https://humanoid-ai-robotics-book-1.vercel.app/docs/module-1/ros2-communication-model
+https://humanoid-ai-robotics-book-1.vercel.app/docs/module-1/urdf-humanoids
+https://humanoid-ai-robotics-book-1.vercel.app/docs/module-2/gazebo-physics-simulation
+https://humanoid-ai-robotics-book-1.vercel.app/docs/module-2/intro-to-digital-twins
+https://humanoid-ai-robotics-book-1.vercel.app/docs/module-2/unity-interaction-sensors
+https://humanoid-ai-robotics-book-1.vercel.app/docs/module-3/intro-to-ai-robot-brain
+https://humanoid-ai-robotics-book-1.vercel.app/docs/module-3/navigation-intelligence
+https://humanoid-ai-robotics-book-1.vercel.app/docs/module-3/perception-simulation-isaac-sim
+https://humanoid-ai-robotics-book-1.vercel.app/docs/module-4/cognitive-planning-llms
+https://humanoid-ai-robotics-book-1.vercel.app/docs/module-4/documentation-standards
+https://humanoid-ai-robotics-book-1.vercel.app/docs/module-4/intro-to-vla
+https://humanoid-ai-robotics-book-1.vercel.app/docs/module-4/voice-to-action-whisper
+https://humanoid-ai-robotics-book-1.vercel.app/docs/tutorial-basics/congratulations
+https://humanoid-ai-robotics-book-1.vercel.app/docs/tutorial-basics/create-a-blog-post
+https://humanoid-ai-robotics-book-1.vercel.app/docs/tutorial-basics/create-a-document
+https://humanoid-ai-robotics-book-1.vercel.app/docs/tutorial-basics/create-a-page
+https://humanoid-ai-robotics-book-1.vercel.app/docs/tutorial-basics/deploy-your-site
+https://humanoid-ai-robotics-book-1.vercel.app/docs/tutorial-basics/markdown-features
+https://humanoid-ai-robotics-book-1.vercel.app/docs/tutorial-extras/manage-docs-versions
+https://humanoid-ai-robotics-book-1.vercel.app/docs/tutorial-extras/translate-your-site
+https://humanoid-ai-robotics-book-1.vercel.app/

test_local.py

@@ -0,0 +1,118 @@
+"""
+Local test that reads sample HTML file and processes it through the pipeline.
+This tests the full flow without relying on external URLs.
+"""
+import sys
+import os
+from pathlib import Path
+
+# Add backend to path
+sys.path.insert(0, str(Path(__file__).parent))
+
+import config
+import utils
+from logging_config import setup_logging
+from main import (
+    extract_text, chunk_text, generate_embeddings, ensure_collection,
+    upsert_chunks, run_validation
+)
+import cohere
+from qdrant_client import QdrantClient
+
+logger = setup_logging()
+
+def test_with_fixture():
+    """Test pipeline using the sample_page.html fixture."""
+    import cohere as cohere_module
+
+    # Load config
+    cfg = config.get_config()
+    config.validate_config(cfg)
+
+    # Read fixture HTML
+    fixture_path = Path(__file__).parent / 'tests' / 'fixtures' / 'sample_page.html'
+    with open(fixture_path, 'r') as f:
+        html = f.read()
+
+    url = "https://example.com/sample-page"
+
+    # Extract text
+    text = extract_text(html, url)
+    logger.info(f"Extracted {len(text)} characters from fixture")
+    print(f"Text preview: {text[:200]}...")
+
+    # Chunk text
+    chunks_data = chunk_text(text, chunk_size=500, overlap=50)
+    logger.info(f"Chunked into {len(chunks_data)} segments")
+    print(f"First chunk: {chunks_data[0]['text'][:100]}...")
+
+    # Generate embeddings - test with Cohere directly to see response structure
+    cohere_client = cohere.ClientV2(api_key=cfg['cohere_api_key'])
+    texts = [chunk['text'] for chunk in chunks_data]
+
+    # Simple test: get one embedding to understand structure
+    test_response = cohere_client.embed(
+        texts=[texts[0]],
+        model="embed-english-v3.0",
+        input_type="search_document"
+    )
+    print(f"\nCohere response type: {type(test_response)}")
+    print(f"Embeddings attribute: {type(test_response.embeddings)}")
+    print(f"Embeddings.float_ attribute: {type(test_response.embeddings.float_)}")
+
+    # Access embeddings directly via .float_ attribute
+    embeddings_float = test_response.embeddings.float_
+    if embeddings_float:
+        first_emb = embeddings_float[0]
+        print(f"First embedding type: {type(first_emb)}")
+        print(f"First embedding dimension: {len(first_emb)}")
+        print(f"First embedding sample (first 5 values): {first_emb[:5]}")
+
+    # Now generate all using the proper extraction method
+    embeddings = generate_embeddings(texts, cohere_client, batch_size=96)
+    logger.info(f"Generated {len(embeddings)} embeddings")
+    print(f"Embedding dimension: {len(embeddings[0])}")
+
+    # Initialize Qdrant and ensure collection
+    qdrant_client = QdrantClient(
+        url=cfg['qdrant_url'],
+        api_key=cfg['qdrant_api_key']
+    )
+    collection = cfg['qdrant_collection']
+    ensure_collection(qdrant_client, collection)
+
+    # Prepare records
+    records = []
+    for i, (chunk, embedding) in enumerate(zip(chunks_data, embeddings)):
+        record = {
+            'url': url,
+            'title': url,
+            'section': '',
+            'chunk_index': i,
+            'text': chunk['text'],
+            'embedding': embedding
+        }
+        records.append(record)
+
+    # Upsert
+    stats = upsert_chunks(qdrant_client, collection, records, utils.deterministic_id)
+    logger.info(f"Upsert stats: {stats}")
+    print(f"Upserted {stats['total']} points to Qdrant")
+
+    # Verify
+    info = qdrant_client.get_collection(collection)
+    print(f"Collection now has {info.points_count} points")
+
+    # Run validation
+    print("\n--- Running Validation ---")
+    run_validation(qdrant_client, collection, sample_size=min(10, info.points_count))
+
+    print("\n✅ Local test complete!")
+    return True
+
+if __name__ == "__main__":
+    try:
+        test_with_fixture()
+    except Exception as e:
+        logger.error(f"Test failed: {e}", exc_info=True)
+        sys.exit(1)

test_run.log

@@ -0,0 +1,32 @@
+INFO: HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333 "HTTP/1.1 200 OK"
+INFO: HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333/collections "HTTP/1.1 200 OK"
+INFO: Collection 'book_embeddings' already exists
+INFO: Starting ingestion: 3 URLs
+INFO: Chunk size: 500, overlap: 50
+INFO: Fetching https://bookshelf.math.buffalo.edu/bookshelf-free-books-direction.html...
+ERROR: Failed to fetch https://bookshelf.math.buffalo.edu/bookshelf-free-books-direction.html after 5 attempts: [Errno -2] Name or service not known
+ERROR: Failed to process https://bookshelf.math.buffalo.edu/bookshelf-free-books-direction.html: [Errno -2] Name or service not known
+INFO: Fetching https://en.wikipedia.org/wiki/Book...
+INFO: HTTP Request: GET https://en.wikipedia.org/wiki/Book "HTTP/1.1 403 Forbidden"
+INFO: HTTP Request: GET https://en.wikipedia.org/wiki/Book "HTTP/1.1 403 Forbidden"
+INFO: HTTP Request: GET https://en.wikipedia.org/wiki/Book "HTTP/1.1 403 Forbidden"
+INFO: HTTP Request: GET https://en.wikipedia.org/wiki/Book "HTTP/1.1 403 Forbidden"
+INFO: HTTP Request: GET https://en.wikipedia.org/wiki/Book "HTTP/1.1 403 Forbidden"
+ERROR: Failed to fetch https://en.wikipedia.org/wiki/Book after 5 attempts: Client error '403 Forbidden' for url 'https://en.wikipedia.org/wiki/Book'
+For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/403
+ERROR: Failed to process https://en.wikipedia.org/wiki/Book: Client error '403 Forbidden' for url 'https://en.wikipedia.org/wiki/Book'
+For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/403
+INFO: Fetching https://www.gutenberg.org/files/1342/1342-h/1342-h.htm...
+INFO: HTTP Request: GET https://www.gutenberg.org/files/1342/1342-h/1342-h.htm "HTTP/1.1 200 OK"
+INFO: Extracted 717156 characters from https://www.gutenberg.org/files/1342/1342-h/1342-h.htm, chunked into 1435 segments
+INFO: HTTP Request: POST https://api.cohere.com/v2/embed "HTTP/1.1 400 Bad Request"
+ERROR: Failed to generate embeddings for batch starting at index 0: headers: {'access-control-expose-headers': 'X-Debug-Trace-ID', 'cache-control': 'no-cache, no-store, no-transform, must-revalidate, private, max-age=0', 'content-encoding': 'gzip', 'content-type': 'application/json', 'expires': 'Thu, 01 Jan 1970 00:00:00 GMT', 'pragma': 'no-cache', 'vary': 'Origin,Accept-Encoding', 'x-accel-expires': '0', 'x-debug-trace-id': '1da9e24a767597a154eaa71df36dcb52', 'x-endpoint-monthly-call-limit': '1000', 'x-trial-endpoint-call-limit': '100', 'x-trial-endpoint-call-remaining': '99', 'date': 'Mon, 16 Feb 2026 20:38:33 GMT', 'x-envoy-upstream-service-time': '5', 'server': 'envoy', 'via': '1.1 google', 'alt-svc': 'h3=":443"; ma=2592000,h3-29=":443"; ma=2592000', 'transfer-encoding': 'chunked'}, status_code: 400, body: {'id': 'cd20a5e4-1efc-44b5-a9f6-43d7a24004ce', 'message': 'invalid request: total number of texts must be at most 96 - received 100'}
+ERROR: Failed to process https://www.gutenberg.org/files/1342/1342-h/1342-h.htm: headers: {'access-control-expose-headers': 'X-Debug-Trace-ID', 'cache-control': 'no-cache, no-store, no-transform, must-revalidate, private, max-age=0', 'content-encoding': 'gzip', 'content-type': 'application/json', 'expires': 'Thu, 01 Jan 1970 00:00:00 GMT', 'pragma': 'no-cache', 'vary': 'Origin,Accept-Encoding', 'x-accel-expires': '0', 'x-debug-trace-id': '1da9e24a767597a154eaa71df36dcb52', 'x-endpoint-monthly-call-limit': '1000', 'x-trial-endpoint-call-limit': '100', 'x-trial-endpoint-call-remaining': '99', 'date': 'Mon, 16 Feb 2026 20:38:33 GMT', 'x-envoy-upstream-service-time': '5', 'server': 'envoy', 'via': '1.1 google', 'alt-svc': 'h3=":443"; ma=2592000,h3-29=":443"; ma=2592000', 'transfer-encoding': 'chunked'}, status_code: 400, body: {'id': 'cd20a5e4-1efc-44b5-a9f6-43d7a24004ce', 'message': 'invalid request: total number of texts must be at most 96 - received 100'}
+INFO: ==================================================
+INFO: Ingestion complete!
+INFO: Total pages processed: 0
+INFO: Total chunks stored: 0
+WARNING: Failed URLs (3): https://bookshelf.math.buffalo.edu/bookshelf-free-books-direction.html, https://en.wikipedia.org/wiki/Book, https://www.gutenberg.org/files/1342/1342-h/1342-h.htm
+INFO: ==================================================
+INFO: HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333/collections/book_embeddings "HTTP/1.1 200 OK"
+INFO: Qdrant collection 'book_embeddings' now has 0 points

test_simple.log

@@ -0,0 +1,18 @@
+INFO: HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333 "HTTP/1.1 200 OK"
+INFO: HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333/collections "HTTP/1.1 200 OK"
+INFO: Collection 'book_embeddings' already exists
+INFO: Starting ingestion: 1 URLs
+INFO: Chunk size: 200, overlap: 20
+INFO: Fetching http://example.com...
+INFO: HTTP Request: GET http://example.com "HTTP/1.1 200 OK"
+INFO: Extracted 142 characters from http://example.com, chunked into 1 segments
+INFO: HTTP Request: POST https://api.cohere.com/v2/embed "HTTP/1.1 200 OK"
+ERROR: Failed to process http://example.com: Unexpected embedding dimension: 2
+INFO: ==================================================
+INFO: Ingestion complete!
+INFO: Total pages processed: 0
+INFO: Total chunks stored: 0
+WARNING: Failed URLs (1): http://example.com
+INFO: ==================================================
+INFO: HTTP Request: GET https://d27e55fe-c28d-4275-81d7-7eff807c14a5.europe-west3-0.gcp.cloud.qdrant.io:6333/collections/book_embeddings "HTTP/1.1 200 OK"
+INFO: Qdrant collection 'book_embeddings' now has 0 points

test_urls.txt

@@ -0,0 +1,3 @@
+https://bookshelf.math.buffalo.edu/bookshelf-free-books-direction.html
+https://en.wikipedia.org/wiki/Book
+https://www.gutenberg.org/files/1342/1342-h/1342-h.htm

tests/fixtures/sample_page.html

@@ -0,0 +1,30 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Sample Book Page - Introduction</title>
+</head>
+<body>
+    <nav>Navigation menu here</nav>
+    <header>Book Title: Example Book</header>
+
+    <h1>Chapter 1: Introduction</h1>
+
+    <p>This is the first paragraph of the introduction. It provides an overview of the topic.</p>
+
+    <p>The second paragraph discusses the background and context. It contains several sentences that should be chunked appropriately when the text is processed.</p>
+
+    <h2>1.1 Background</h2>
+
+    <p>This is a subsection with its own content. It provides more detailed information about the background.</p>
+
+    <p>Another paragraph in the background section. This helps test the chunking algorithm with multiple paragraphs and sections.</p>
+
+    <h2>1.2 Objectives</h2>
+
+    <p>This subsection covers the objectives. It should be extracted with its section identifier.</p>
+
+    <p>Final paragraph of this sample page. It ensures we have enough text to create multiple chunks when using a chunk size of 1000 characters.</p>
+
+    <footer>Footer content like copyright info should be removed.</footer>
+</body>
+</html>

utils.py

@@ -0,0 +1,54 @@
+"""
+Utility functions for the ingestion pipeline.
+"""
+import hashlib
+import time
+import random
+from typing import Callable, Any
+
+def deterministic_id(url: str, chunk_index: int) -> str:
+    """
+    Generate deterministic point ID for Qdrant.
+    Uses SHA256 hash of "url:chunk_index" and formats as a valid UUID.
+    Qdrant accepts UUID v4 format or unsigned integers.
+    """
+    import uuid
+    key = f"{url}:{chunk_index}"
+    hash_bytes = hashlib.sha256(key.encode()).digest()
+    # Convert to UUID v4 format (using random variant) but deterministic from hash
+    # Use first 16 bytes of SHA256 to create a UUID
+    uuid_bytes = hash_bytes[:16]
+    # Set version to 4 (random) and variant to RFC 4122
+    uuid_bytes = uuid_bytes[:6] + bytes([(uuid_bytes[6] & 0x0f) | 0x40]) + bytes([uuid_bytes[7] & 0x3f | 0x80]) + uuid_bytes[8:]
+    return str(uuid.UUID(bytes=uuid_bytes))
+
+
+def verify_deterministic_id(url: str, chunk_index: int, expected_id: str = None) -> str:
+    """
+    Verify deterministic ID generation and detect collisions.
+    Returns the generated ID. If expected_id is provided and differs, logs warning.
+    """
+    generated_id = deterministic_id(url, chunk_index)
+    if expected_id and generated_id != expected_id:
+        logger.warning(f"ID collision detected for {url}:{chunk_index}. Expected {expected_id}, got {generated_id}")
+    return generated_id
+
+def retry_with_backoff(
+    func: Callable,
+    max_retries: int = 5,
+    base_delay: float = 1.0,
+    max_delay: float = 30.0,
+    jitter: float = 0.1
+) -> Any:
+    """
+    Retry a function with exponential backoff and jitter.
+    Suitable for API calls (Cohere, Qdrant, HTTP).
+    """
+    for attempt in range(max_retries):
+        try:
+            return func()
+        except Exception as e:
+            if attempt == max_retries - 1:
+                raise
+            delay = min(base_delay * (2 ** attempt) + random.uniform(-jitter, jitter), max_delay)
+            time.sleep(delay)

validate.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+"""
+Simple validation script to test the ingestion pipeline components.
+"""
+import sys
+import os
+sys.path.insert(0, '.')
+
+def test_imports():
+    """Test all imports work."""
+    import config
+    import utils
+    import logging_config
+    print("✓ All imports successful")
+
+def test_deterministic_id():
+    """Test deterministic ID generation."""
+    from utils import deterministic_id
+    id1 = deterministic_id("https://example.com/page", 0)
+    id2 = deterministic_id("https://example.com/page", 0)
+    id3 = deterministic_id("https://example.com/page", 1)
+    assert id1 == id2, "Same input should produce same ID"
+    assert id1 != id3, "Different chunk_index should produce different ID"
+    assert len(id1) == 64, "SHA256 hex should be 64 characters"
+    print("✓ Deterministic ID generation works")
+
+def test_chunking():
+    """Test text chunking logic."""
+    from main import chunk_text
+    text = "A" * 2500  # 2500 characters
+    chunks = chunk_text(text, chunk_size=1000, overlap=100)
+    assert len(chunks) > 1, "Long text should produce multiple chunks"
+    assert all('text' in c and 'char_start' in c and 'char_end' in c for c in chunks), "Chunks have required fields"
+    # Check overlap
+    for i in range(1, len(chunks)):
+        gap = chunks[i]['char_start'] - chunks[i-1]['char_end']
+        assert gap <= 100, f"Overlap should be <=100, got {gap}"
+    print(f"✓ Chunking works: {len(chunks)} chunks from 2500 chars")
+
+def test_config():
+    """Test config loading."""
+    import config as cfg_module
+    try:
+        cfg = cfg_module.get_config()
+        cfg_module.validate_config(cfg)
+    except ValueError as e:
+        if "Missing required environment variables" in str(e):
+            print("✓ Config validation works (expected: missing env vars when .env not set)")
+        else:
+            raise
+
+if __name__ == "__main__":
+    print("Running validation tests...")
+    test_imports()
+    test_deterministic_id()
+    test_chunking()
+    test_config()
+    print("\n✓ All validation tests passed!")
+    print("\nTo test full pipeline, set COHERE_API_KEY and QDRANT credentials in .env and run:")
+    print("  python backend/main.py --urls https://example.com")