Merge pull request #5 from huggingface/explore-tool

agent/config.py

@@ -8,7 +8,6 @@ from fastmcp.mcp_config import (
     RemoteMCPServer,
     StdioMCPServer,
 )
-from litellm import Tool
 from pydantic import BaseModel
 
 # These two are the canonical server config types for MCP servers.
@@ -19,7 +18,6 @@ class Config(BaseModel):
     """Configuration manager"""
 
     model_name: str
-    tools: list[Tool] = []
     mcpServers: dict[str, MCPServerConfig] = {}
 
 

agent/config_claude_mcp.json

@@ -1,11 +0,0 @@
-{
-    "mcpServers": {
-        "huggingface": {
-            "type": "http",
-            "url": "https://huggingface.co/mcp",
-            "headers": {
-                "Authorization": "Bearer ${HF_TOKEN}"
-            }
-        }
-    }
-}

agent/config_mcp_example copy.json

@@ -1,21 +0,0 @@
-{
-  "model_name": "anthropic/claude-sonnet-4-5-20250929",
-  "tools": [],
-  "system_prompt_path": "",
-  "mcpServers": {
-    "hf-mcp-server": {
-      "transport": "http",
-      "url": "https://huggingface.co/mcp?login",
-      "headers": {
-        "Authorization": "Bearer ${HF_TOKEN}"
-      }
-    },
-    "playwright": {
-      "transport": "stdio",
-      "command": "npx",
-      "args": [
-        "@playwright/mcp@latest"
-      ]
-    }
-  }
-}

agent/context_manager/manager.py

@@ -19,17 +19,24 @@ class ContextManager:
         compact_size: float = 0.1,
         untouched_messages: int = 5,
         tool_specs: list[dict[str, Any]] | None = None,
+        prompt_file_suffix: str = "system_prompt.yaml",
     ):
-        self.system_prompt = self._load_system_prompt(tool_specs or [])
+        self.system_prompt = self._load_system_prompt(
+            tool_specs or [], prompt_file_suffix="system_prompt.yaml"
+        )
         self.max_context = max_context
         self.compact_size = int(max_context * compact_size)
         self.context_length = len(self.system_prompt) // 4
         self.untouched_messages = untouched_messages
         self.items: list[Message] = [Message(role="system", content=self.system_prompt)]
 
-    def _load_system_prompt(self, tool_specs: list[dict[str, Any]]):
+    def _load_system_prompt(
+        self,
+        tool_specs: list[dict[str, Any]],
+        prompt_file_suffix: str = "system_prompt.yaml",
+    ):
         """Load and render the system prompt from YAML file with Jinja2"""
-        prompt_file = Path(__file__).parent.parent / "prompts" / "system_prompt.yaml"
+        prompt_file = Path(__file__).parent.parent / "prompts" / f"{prompt_file_suffix}"
 
         with open(prompt_file, "r") as f:
             prompt_data = yaml.safe_load(f)

agent/core/session.py

@@ -36,10 +36,11 @@ class Session:
         event_queue: asyncio.Queue,
         config: Config | None = None,
         tool_router=None,
+        context_manager: ContextManager | None = None,
     ):
         self.tool_router = tool_router
         tool_specs = tool_router.get_tool_specs_for_llm() if tool_router else []
-        self.context_manager = ContextManager(
+        self.context_manager = context_manager or ContextManager(
             max_context=get_max_tokens(config.model_name),
             compact_size=0.1,
             untouched_messages=5,
@@ -49,7 +50,6 @@ class Session:
         self.session_id = str(uuid.uuid4())
         self.config = config or Config(
             model_name="anthropic/claude-sonnet-4-5-20250929",
-            tools=[],
         )
         self.is_running = True
         self.current_task: asyncio.Task | None = None

agent/core/tools.py

@@ -14,6 +14,7 @@ from mcp.types import EmbeddedResource, ImageContent, TextContent
 
 from agent.config import MCPServerConfig
 from agent.tools.jobs_tool import HF_JOBS_TOOL_SPEC, hf_jobs_handler
+from agent.tools.search_docs_tool import SEARCH_DOCS_TOOL_SPEC, search_docs_handler
 from agent.tools.plan_tool import PLAN_TOOL_SPEC, plan_tool_handler
 
 # Suppress aiohttp deprecation warning
@@ -21,7 +22,7 @@ warnings.filterwarnings(
     "ignore", category=DeprecationWarning, module="aiohttp.connector"
 )
 
-NOT_ALLOWED_TOOL_NAMES = ["hf_jobs"]
+NOT_ALLOWED_TOOL_NAMES = ["hf_jobs", "hf_doc_search", "hf_doc_fetch"]
 
 
 def convert_mcp_content_to_string(content: list) -> str:
@@ -187,7 +188,9 @@ class ToolRouter:
 
 def create_builtin_tools() -> list[ToolSpec]:
     """Create built-in tool specifications"""
-    print(f"Creating built-in tools: {HF_JOBS_TOOL_SPEC['name']}, {PLAN_TOOL_SPEC['name']}")
+    print(
+        f"Creating built-in tools: {HF_JOBS_TOOL_SPEC['name']}, {SEARCH_DOCS_TOOL_SPEC['name']}, {PLAN_TOOL_SPEC['name']}"
+    )
     return [
         ToolSpec(
             name=HF_JOBS_TOOL_SPEC["name"],
@@ -196,9 +199,15 @@ def create_builtin_tools() -> list[ToolSpec]:
             handler=hf_jobs_handler,
         ),
         ToolSpec(
-            name=PLAN_TOOL_SPEC["name"],
+            name=SEARCH_DOCS_TOOL_SPEC["name"],
+            description=SEARCH_DOCS_TOOL_SPEC["description"],
+            parameters=SEARCH_DOCS_TOOL_SPEC["parameters"],
+            handler=search_docs_handler,
+        ),
+      ToolSpec(
+            ame=PLAN_TOOL_SPEC["name"],
             description=PLAN_TOOL_SPEC["description"],
             parameters=PLAN_TOOL_SPEC["parameters"],
             handler=plan_tool_handler,
-        ),
+        )
     ]

agent/main.py

@@ -222,7 +222,7 @@ async def main():
     ready_event = asyncio.Event()
 
     # Start agent loop in background
-    config_path = Path(__file__).parent / "config_mcp_example.json"
+    config_path = Path(__file__).parent.parent / "configs" / "main_agent_config.json"
     config = load_config(config_path)
 
     # Create tool router

agent/prompts/search_docs_system_prompt.yaml

@@ -0,0 +1,38 @@
+search_docs_system_prompt: |
+  You are a specialized documentation search agent. Your task is to comprehensively search and synthesize information from Hugging Face documentation.
+
+  # Search Strategy
+
+  You must search thoroughly before synthesizing results. Follow this approach:
+
+  1. **Query Analysis**: Identify the core concepts and intent of the query
+  2. **Initial Search**: Start with a broad search capturing the main topic
+  3. **Iterative Refinement**: Run multiple searches to go deeper into topics. You will see parsed HTML pages, also look into links on the html pages for best information - first-pass results often miss key details
+  4. **You must get to the end truth**: You must get to the bottom of the truth for this search query. You CAN NOT say that somebody should look up documentation. You must look it up yourself and give the best answer you can.
+
+  ## Query Formulation Best Practices
+
+  - Add relevant synonyms and related technical terms
+  - Remove filler words, focus on searchable concepts
+  - Break complex questions into focused sub-queries
+  - Include domain-specific terminology when applicable
+  - Try both specific terms and general related terms
+
+  # Response Guidelines
+
+  After gathering results, synthesize them following these principles:
+
+  1. **Analyze Relevance**: Evaluate which results directly answer the query
+  2. **Synthesize**: Combine information from multiple sources when applicable
+  3. **Prioritize**: Present information in order of relevance
+  4. **Cite Sources**: Reference which documents you're drawing from especially include relevant code samples and links to the code samples.
+  5. **Acknowledge Gaps**: If documents don't fully answer the query, explicitly state this
+  6. **Handle Conflicts**: If sources contradict, note this and explain your reasoning
+  7. **Be Concise**: Provide a clear, direct answer without unnecessary elaboration
+
+  # Constraints
+
+  - Only provide information found in the documentation
+  - Do not make assumptions beyond what the sources state
+  - If information is not found, say so clearly rather than guessing
+  - Focus on answering the query directly

agent/tools/__init__.py

@@ -3,6 +3,14 @@ Hugging Face tools for the agent
 """
 
 from agent.tools.jobs_tool import HF_JOBS_TOOL_SPEC, HfJobsTool, hf_jobs_handler
+from agent.tools.search_docs_tool import SEARCH_DOCS_TOOL_SPEC, search_docs_handler
 from agent.tools.types import ToolResult
 
-__all__ = ["ToolResult", "HF_JOBS_TOOL_SPEC", "hf_jobs_handler", HfJobsTool]
+__all__ = [
+    "ToolResult",
+    "HF_JOBS_TOOL_SPEC",
+    "hf_jobs_handler",
+    "HfJobsTool",
+    "SEARCH_DOCS_TOOL_SPEC",
+    "search_docs_handler",
+]

agent/tools/_search_agent_tools.py

@@ -0,0 +1,747 @@
+"""
+Tools available to the search sub-agent
+These tools are used by the search sub-agent spawned by search_docs_tool
+"""
+
+import asyncio
+import os
+import time
+from typing import Any
+
+import httpx
+from bs4 import BeautifulSoup
+
+# Cache for OpenAPI spec to avoid repeated fetches
+_openapi_spec_cache: dict[str, Any] | None = None
+
+
+async def _fetch_html_page(hf_token: str, endpoint: str) -> str:
+    """Fetch the HTML page for a given endpoint"""
+    base_url = "https://huggingface.co/docs"
+    url = f"{base_url}/{endpoint}"
+    headers = {"Authorization": f"Bearer {hf_token}"}
+
+    fetch_start = time.perf_counter()
+    async with httpx.AsyncClient(timeout=30.0, follow_redirects=True) as client:
+        response = await client.get(url, headers=headers)
+        response.raise_for_status()
+
+    fetch_time = time.perf_counter() - fetch_start
+    print(f"[DEBUG] _fetch_html_page: Fetched in {fetch_time:.2f}s")
+
+    return response.text
+
+
+def _parse_sidebar_navigation(html_content: str) -> list[dict[str, str]]:
+    """Parse the sidebar navigation and extract all links"""
+    parse_start = time.perf_counter()
+
+    soup = BeautifulSoup(html_content, "html.parser")
+    sidebar = soup.find("nav", class_=lambda x: x and "flex-auto" in x)
+
+    if not sidebar:
+        raise ValueError("Could not find navigation sidebar")
+
+    links = sidebar.find_all("a", href=True)
+    nav_data = []
+
+    for link in links:
+        title = link.get_text(strip=True)
+        href = link["href"]
+
+        # Make URL absolute
+        page_url = f"https://huggingface.co{href}" if href.startswith("/") else href
+        nav_data.append({"title": title, "url": page_url})
+
+    parse_time = time.perf_counter() - parse_start
+    print(
+        f"[DEBUG] _parse_sidebar_navigation: Parsed in {parse_time:.2f}s, found {len(nav_data)} links"
+    )
+
+    return nav_data
+
+
+async def _fetch_single_glimpse(
+    client: httpx.AsyncClient, hf_token: str, item: dict[str, str]
+) -> dict[str, str]:
+    """Fetch a glimpse (first 300 chars) for a single page"""
+    md_url = f"{item['url']}.md"
+    headers = {"Authorization": f"Bearer {hf_token}"}
+
+    try:
+        response = await client.get(md_url, headers=headers)
+        response.raise_for_status()
+
+        content = response.text
+        glimpse = content[:300].strip()
+        if len(content) > 300:
+            glimpse += "..."
+
+        return {
+            "title": item["title"],
+            "url": item["url"],
+            "md_url": md_url,
+            "glimpse": glimpse,
+        }
+    except Exception as e:
+        return {
+            "title": item["title"],
+            "url": item["url"],
+            "md_url": md_url,
+            "glimpse": f"[Could not fetch glimpse: {str(e)[:50]}]",
+        }
+
+
+async def _fetch_all_glimpses(
+    hf_token: str, nav_data: list[dict[str, str]]
+) -> list[dict[str, str]]:
+    """Fetch glimpses for all pages in parallel"""
+    glimpse_start = time.perf_counter()
+
+    async with httpx.AsyncClient(timeout=30.0, follow_redirects=True) as client:
+        result_items = await asyncio.gather(
+            *[_fetch_single_glimpse(client, hf_token, item) for item in nav_data]
+        )
+
+    glimpse_time = time.perf_counter() - glimpse_start
+    print(
+        f"[DEBUG] _fetch_all_glimpses: Fetched {len(result_items)} glimpses in {glimpse_time:.2f}s"
+    )
+
+    return list(result_items)
+
+
+def _format_exploration_results(
+    endpoint: str, result_items: list[dict[str, str]]
+) -> str:
+    """Format the exploration results as a readable string"""
+    base_url = "https://huggingface.co/docs"
+    url = f"{base_url}/{endpoint}"
+    result = f"Documentation structure for: {url}\n\n"
+    result += f"Found {len(result_items)} pages:\n\n"
+
+    for i, item in enumerate(result_items, 1):
+        result += f"{i}. **{item['title']}**\n"
+        result += f"   URL: {item['url']}\n"
+        result += f"   Glimpse: {item['glimpse']}\n\n"
+
+    return result
+
+
+async def explore_hf_docs(hf_token: str, endpoint: str) -> str:
+    """Main function to explore documentation structure"""
+    start_time = time.perf_counter()
+    print(f"[DEBUG] explore_hf_docs: Starting for endpoint '{endpoint}'")
+
+    # Fetch HTML page
+    html_content = await _fetch_html_page(hf_token, endpoint)
+
+    # Parse navigation
+    nav_data = _parse_sidebar_navigation(html_content)
+
+    if not nav_data:
+        raise ValueError(f"No navigation links found for endpoint '{endpoint}'")
+
+    # Fetch all glimpses in parallel
+    result_items = await _fetch_all_glimpses(hf_token, nav_data)
+
+    # Format results
+    result = _format_exploration_results(endpoint, result_items)
+
+    total_time = time.perf_counter() - start_time
+    print(f"[DEBUG] explore_hf_docs: Total time {total_time:.2f}s")
+
+    return result
+
+
+async def explore_hf_docs_handler(arguments: dict[str, Any]) -> tuple[str, bool]:
+    """
+    Explore the documentation structure for a given endpoint by parsing the sidebar navigation
+
+    Args:
+        arguments: Dictionary with 'endpoint' parameter (e.g., 'trl', 'transformers', etc.)
+
+    Returns:
+        Tuple of (structured_navigation_with_glimpses, success)
+    """
+    endpoint = arguments.get("endpoint", "")
+
+    if not endpoint:
+        return "Error: No endpoint provided", False
+
+    # Get HF token from environment
+    hf_token = os.environ.get("HF_TOKEN")
+
+    if not hf_token:
+        return "Error: HF_TOKEN environment variable not set", False
+
+    endpoint = endpoint.lstrip("/")
+
+    try:
+        result = await explore_hf_docs(hf_token, endpoint)
+        return result, True
+
+    except httpx.HTTPStatusError as e:
+        return (
+            f"HTTP error: {e.response.status_code} - {e.response.text[:200]}",
+            False,
+        )
+    except httpx.RequestError as e:
+        return f"Request error: {str(e)}", False
+    except ValueError as e:
+        return f"Error: {str(e)}", False
+    except Exception as e:
+        return f"Unexpected error: {str(e)}", False
+
+
+async def _fetch_openapi_spec() -> dict[str, Any]:
+    """Fetch and cache the HuggingFace OpenAPI specification"""
+    global _openapi_spec_cache
+
+    if _openapi_spec_cache is not None:
+        print("[DEBUG] _fetch_openapi_spec: Using cached spec")
+        return _openapi_spec_cache
+
+    start_time = time.perf_counter()
+    print("[DEBUG] _fetch_openapi_spec: Fetching from API")
+
+    url = "https://huggingface.co/.well-known/openapi.json"
+
+    async with httpx.AsyncClient(timeout=30.0, follow_redirects=True) as client:
+        response = await client.get(url)
+        response.raise_for_status()
+
+    spec = response.json()
+    _openapi_spec_cache = spec
+
+    fetch_time = time.perf_counter() - start_time
+    print(f"[DEBUG] _fetch_openapi_spec: Fetched and cached in {fetch_time:.2f}s")
+
+    return spec
+
+
+def _extract_all_tags(spec: dict[str, Any]) -> list[str]:
+    """Extract all unique tags from the OpenAPI spec"""
+    tags = set()
+
+    # Get tags from the tags section
+    for tag_obj in spec.get("tags", []):
+        if "name" in tag_obj:
+            tags.add(tag_obj["name"])
+
+    # Also get tags from paths (in case some aren't in the tags section)
+    for path, path_item in spec.get("paths", {}).items():
+        for method, operation in path_item.items():
+            if method in ["get", "post", "put", "delete", "patch", "head", "options"]:
+                for tag in operation.get("tags", []):
+                    tags.add(tag)
+
+    return sorted(list(tags))
+
+
+def _search_openapi_by_tag(spec: dict[str, Any], tag: str) -> list[dict[str, Any]]:
+    """Search for API endpoints with a specific tag"""
+    results = []
+    paths = spec.get("paths", {})
+    servers = spec.get("servers", [])
+    base_url = (
+        servers[0].get("url", "https://huggingface.co")
+        if servers
+        else "https://huggingface.co"
+    )
+
+    for path, path_item in paths.items():
+        for method, operation in path_item.items():
+            if method not in [
+                "get",
+                "post",
+                "put",
+                "delete",
+                "patch",
+                "head",
+                "options",
+            ]:
+                continue
+
+            operation_tags = operation.get("tags", [])
+            if tag in operation_tags:
+                # Extract parameters
+                parameters = operation.get("parameters", [])
+                request_body = operation.get("requestBody", {})
+                responses = operation.get("responses", {})
+
+                results.append(
+                    {
+                        "path": path,
+                        "method": method.upper(),
+                        "operationId": operation.get("operationId", ""),
+                        "summary": operation.get("summary", ""),
+                        "description": operation.get("description", ""),
+                        "parameters": parameters,
+                        "request_body": request_body,
+                        "responses": responses,
+                        "base_url": base_url,
+                    }
+                )
+
+    return results
+
+
+def _generate_curl_example(endpoint: dict[str, Any]) -> str:
+    """Generate a curl command example for an endpoint"""
+    method = endpoint["method"]
+    path = endpoint["path"]
+    base_url = endpoint["base_url"]
+
+    # Build the full URL with example path parameters
+    full_path = path
+    for param in endpoint.get("parameters", []):
+        if param.get("in") == "path" and param.get("required"):
+            param_name = param["name"]
+            example = param.get(
+                "example", param.get("schema", {}).get("example", f"<{param_name}>")
+            )
+            full_path = full_path.replace(f"{{{param_name}}}", str(example))
+
+    curl = f"curl -X {method} \\\n  '{base_url}{full_path}'"
+
+    # Add query parameters if any
+    query_params = [p for p in endpoint.get("parameters", []) if p.get("in") == "query"]
+    if query_params and query_params[0].get("required"):
+        param = query_params[0]
+        example = param.get("example", param.get("schema", {}).get("example", "value"))
+        curl += f"?{param['name']}={example}"
+
+    # Add headers
+    curl += " \\\n  -H 'Authorization: Bearer $HF_TOKEN'"
+
+    # Add request body if applicable
+    if method in ["POST", "PUT", "PATCH"] and endpoint.get("request_body"):
+        content = endpoint["request_body"].get("content", {})
+        if "application/json" in content:
+            curl += " \\\n  -H 'Content-Type: application/json'"
+            schema = content["application/json"].get("schema", {})
+            example = schema.get("example", "{}")
+            if isinstance(example, dict):
+                import json
+
+                example = json.dumps(example, indent=2)
+            curl += f" \\\n  -d '{example}'"
+
+    return curl
+
+
+def _format_parameters(parameters: list[dict[str, Any]]) -> str:
+    """Format parameter information from OpenAPI spec"""
+    if not parameters:
+        return ""
+
+    # Group parameters by type
+    path_params = [p for p in parameters if p.get("in") == "path"]
+    query_params = [p for p in parameters if p.get("in") == "query"]
+    header_params = [p for p in parameters if p.get("in") == "header"]
+
+    output = []
+
+    if path_params:
+        output.append("**Path Parameters:**")
+        for param in path_params:
+            name = param.get("name", "")
+            required = " (required)" if param.get("required") else " (optional)"
+            description = param.get("description", "")
+            param_type = param.get("schema", {}).get("type", "string")
+            example = param.get("example") or param.get("schema", {}).get("example", "")
+
+            output.append(f"- `{name}` ({param_type}){required}: {description}")
+            if example:
+                output.append(f"  Example: `{example}`")
+
+    if query_params:
+        if output:
+            output.append("")
+        output.append("**Query Parameters:**")
+        for param in query_params:
+            name = param.get("name", "")
+            required = " (required)" if param.get("required") else " (optional)"
+            description = param.get("description", "")
+            param_type = param.get("schema", {}).get("type", "string")
+            example = param.get("example") or param.get("schema", {}).get("example", "")
+
+            output.append(f"- `{name}` ({param_type}){required}: {description}")
+            if example:
+                output.append(f"  Example: `{example}`")
+
+    if header_params:
+        if output:
+            output.append("")
+        output.append("**Header Parameters:**")
+        for param in header_params:
+            name = param.get("name", "")
+            required = " (required)" if param.get("required") else " (optional)"
+            description = param.get("description", "")
+
+            output.append(f"- `{name}`{required}: {description}")
+
+    return "\n".join(output)
+
+
+def _format_response_info(responses: dict[str, Any]) -> str:
+    """Format response information from OpenAPI spec"""
+    if not responses:
+        return "No response information available"
+
+    output = []
+    for status_code, response_obj in list(responses.items())[
+        :3
+    ]:  # Show first 3 status codes
+        desc = response_obj.get("description", "")
+        output.append(f"- **{status_code}**: {desc}")
+
+        content = response_obj.get("content", {})
+        if "application/json" in content:
+            schema = content["application/json"].get("schema", {})
+            if "type" in schema:
+                output.append(f"  Returns: {schema.get('type', 'object')}")
+
+    return "\n".join(output)
+
+
+def _format_openapi_results(results: list[dict[str, Any]], tag: str) -> str:
+    """Format OpenAPI search results as markdown with curl examples"""
+    if not results:
+        return f"No API endpoints found with tag '{tag}'"
+
+    output = f"# API Endpoints for tag: `{tag}`\n\n"
+    output += f"Found {len(results)} endpoint(s)\n\n"
+    output += "---\n\n"
+
+    for i, endpoint in enumerate(results, 1):
+        output += f"## {i}. {endpoint['method']} {endpoint['path']}\n\n"
+
+        if endpoint["summary"]:
+            output += f"**Summary:** {endpoint['summary']}\n\n"
+
+        if endpoint["description"]:
+            desc = endpoint["description"][:300]
+            if len(endpoint["description"]) > 300:
+                desc += "..."
+            output += f"**Description:** {desc}\n\n"
+
+        # Parameters
+        params_info = _format_parameters(endpoint.get("parameters", []))
+        if params_info:
+            output += params_info + "\n\n"
+
+        # Curl example
+        output += "**Usage:**\n```bash\n"
+        output += _generate_curl_example(endpoint)
+        output += "\n```\n\n"
+
+        # Response info
+        output += "**Returns:**\n"
+        output += _format_response_info(endpoint["responses"])
+        output += "\n\n"
+
+        output += "---\n\n"
+
+    return output
+
+
+async def search_openapi_handler(arguments: dict[str, Any]) -> tuple[str, bool]:
+    """
+    Search the HuggingFace OpenAPI specification by tag
+
+    Args:
+        arguments: Dictionary with 'tag' parameter
+
+    Returns:
+        Tuple of (search_results, success)
+    """
+    start_time = time.perf_counter()
+    tag = arguments.get("tag", "")
+    print(f"[DEBUG] search_openapi: Starting for tag '{tag}'")
+
+    if not tag:
+        return "Error: No tag provided", False
+
+    try:
+        # Fetch OpenAPI spec (cached after first fetch)
+        spec = await _fetch_openapi_spec()
+
+        # Search for endpoints with this tag
+        results = _search_openapi_by_tag(spec, tag)
+
+        # Format results
+        formatted = _format_openapi_results(results, tag)
+
+        total_time = time.perf_counter() - start_time
+        print(f"[DEBUG] search_openapi: Total time {total_time:.2f}s")
+
+        return formatted, True
+
+    except httpx.HTTPStatusError as e:
+        return f"HTTP error fetching OpenAPI spec: {e.response.status_code}", False
+    except httpx.RequestError as e:
+        return f"Request error: {str(e)}", False
+    except Exception as e:
+        return f"Error searching OpenAPI spec: {str(e)}", False
+
+
+async def hf_docs_fetch_handler(arguments: dict[str, Any]) -> tuple[str, bool]:
+    """
+    Fetch full documentation content from a specific HF docs page
+
+    Args:
+        arguments: Dictionary with 'url' parameter (full URL to the doc page)
+
+    Returns:
+        Tuple of (full_markdown_content, success)
+    """
+    start_time = time.perf_counter()
+    url = arguments.get("url", "")
+    print(f"[DEBUG] fetch_hf_docs: Starting for URL '{url}'")
+
+    if not url:
+        return "Error: No URL provided", False
+
+    # Get HF token from environment
+    hf_token = os.environ.get("HF_TOKEN")
+
+    if not hf_token:
+        return (
+            "Error: HF_TOKEN environment variable not set",
+            False,
+        )
+
+    # Add .md extension if not already present
+    if not url.endswith(".md"):
+        url = f"{url}.md"
+
+    try:
+        # Make request with auth
+        headers = {"Authorization": f"Bearer {hf_token}"}
+
+        fetch_start = time.perf_counter()
+        async with httpx.AsyncClient(timeout=30.0, follow_redirects=True) as client:
+            response = await client.get(url, headers=headers)
+            response.raise_for_status()
+
+        fetch_time = time.perf_counter() - fetch_start
+        content = response.text
+        content_size_kb = len(content) / 1024
+
+        print(
+            f"[DEBUG] fetch_hf_docs: Fetched {content_size_kb:.1f}KB in {fetch_time:.2f}s"
+        )
+
+        # Return the markdown content directly
+        result = f"Documentation from: {url}\n\n{content}"
+
+        total_time = time.perf_counter() - start_time
+        print(f"[DEBUG] fetch_hf_docs: Total time {total_time:.2f}s")
+
+        return result, True
+
+    except httpx.HTTPStatusError as e:
+        return (
+            f"HTTP error fetching {url}: {e.response.status_code} - {e.response.text[:200]}",
+            False,
+        )
+    except httpx.RequestError as e:
+        return f"Request error fetching {url}: {str(e)}", False
+    except Exception as e:
+        return f"Error fetching documentation: {str(e)}", False
+
+
+# Tool specifications for the search sub-agent
+
+EXPLORE_HF_DOCS_TOOL_SPEC = {
+    "name": "explore_hf_docs",
+    "description": (
+        "Explore the Hugging Face documentation at a glance. "
+        "Select an endpoint from the available options and get a list of all documentation pages "
+        "with their titles, URLs, and a 300-character glimpse of each page. "
+        "Use this to discover what documentation is available before fetching specific pages."
+    ),
+    "parameters": {
+        "type": "object",
+        "properties": {
+            "endpoint": {
+                "type": "string",
+                "enum": [
+                    "hub",
+                    "transformers",
+                    "diffusers",
+                    "datasets",
+                    "gradio",
+                    "trackio",
+                    "smolagents",
+                    "huggingface_hub",
+                    "huggingface.js",
+                    "transformers.js",
+                    "inference-providers",
+                    "inference-endpoints",
+                    "peft",
+                    "accelerate",
+                    "optimum",
+                    "optimum-habana",
+                    "optimum-neuron",
+                    "optimum-intel",
+                    "optimum-executorch",
+                    "optimum-tpu",
+                    "tokenizers",
+                    "llm-course",
+                    "robotics-course",
+                    "mcp-course",
+                    "smol-course",
+                    "agents-course",
+                    "deep-rl-course",
+                    "computer-vision-course",
+                    "evaluate",
+                    "tasks",
+                    "dataset-viewer",
+                    "trl",
+                    "simulate",
+                    "sagemaker",
+                    "timm",
+                    "safetensors",
+                    "tgi",
+                    "setfit",
+                    "audio-course",
+                    "lerobot",
+                    "autotrain",
+                    "tei",
+                    "bitsandbytes",
+                    "cookbook",
+                    "sentence_transformers",
+                    "ml-games-course",
+                    "diffusion-course",
+                    "ml-for-3d-course",
+                    "chat-ui",
+                    "leaderboards",
+                    "lighteval",
+                    "argilla",
+                    "distilabel",
+                    "microsoft-azure",
+                    "kernels",
+                    "google-cloud",
+                ],
+                "description": (
+                    "The documentation endpoint to explore. Each endpoint corresponds to a major section of the Hugging Face documentation:\n\n"
+                    "• hub — Find answers to questions about models/datasets/spaces, auth, versioning, metadata.\n"
+                    "• transformers — Core model library: architectures, configs, tokenizers, training & inference APIs.\n"
+                    "• diffusers — Diffusion pipelines, schedulers, fine-tuning, training, and deployment patterns.\n"
+                    "• datasets — Dataset loading, streaming, processing, Arrow format, Hub integration.\n"
+                    "• gradio — UI components and demos for interacting with ML models.\n"
+                    "• trackio — Experiment tracking, metrics logging, and run comparison.\n"
+                    "• smolagents — Lightweight agent abstractions and tool-using patterns.\n"
+                    "• huggingface_hub — Python client for Hub operations (auth, upload/download, repo management).\n"
+                    "• huggingface.js — JS/TS client for Hub APIs in browser and Node.\n"
+                    "• transformers.js — Run Transformer models in browser/Node via WebGPU/WASM.\n"
+                    "• inference-providers — Unified interface for third-party inference backends.\n"
+                    "• inference-endpoints — Managed, scalable model deployments on HF infrastructure.\n"
+                    "• peft — Parameter-efficient fine-tuning methods (LoRA, adapters, etc.).\n"
+                    "• accelerate — Hardware-agnostic, distributed and mixed-precision training orchestration.\n"
+                    "• optimum — Hardware-aware optimization and model export tooling.\n"
+                    "• optimum-habana — Training and inference on Habana Gaudi accelerators.\n"
+                    "• optimum-neuron — Optimization workflows for AWS Inferentia/Trainium.\n"
+                    "• optimum-intel — Intel CPU/GPU optimizations (OpenVINO, IPEX).\n"
+                    "• optimum-executorch — Exporting models to ExecuTorch for edge/mobile.\n"
+                    "• optimum-tpu — TPU-specific training and optimization paths.\n"
+                    "• tokenizers — Fast tokenizer internals, training, and low-level APIs.\n"
+                    "• llm-course — End-to-end LLM concepts, training, and deployment.\n"
+                    "• robotics-course — Learning-based robotics foundations.\n"
+                    "• mcp-course — Model Context Protocol concepts and usage.\n"
+                    "• smol-course — Small-model and efficiency-focused workflows.\n"
+                    "• agents-course — Tool-using, planning, and multi-step agent design.\n"
+                    "• deep-rl-course — Deep reinforcement learning foundations.\n"
+                    "• computer-vision-course — Vision models, datasets, and pipelines.\n"
+                    "• evaluate — Metrics, evaluation workflows, and training-loop integration.\n"
+                    "• tasks — Canonical task definitions and model categorization.\n"
+                    "• dataset-viewer — Dataset preview, streaming views, and viewer internals.\n"
+                    "• trl — RLHF, DPO, PPO, and SFT utilities for LLMs.\n"
+                    "• simulate — Experimental simulation tools and workflows.\n"
+                    "• sagemaker — Deploying Hugging Face models on AWS SageMaker.\n"
+                    "• timm — Image model zoo and utilities via HF integrations.\n"
+                    "• safetensors — Safe, fast tensor serialization format.\n"
+                    "• tgi — High-throughput text generation server for LLMs.\n"
+                    "• setfit — Few-shot text classification via sentence embeddings.\n"
+                    "• audio-course — Speech and audio models, datasets, and tasks.\n"
+                    "• lerobot — Robotics datasets, policies, and learning workflows.\n"
+                    "• autotrain — No/low-code model training on Hugging Face.\n"
+                    "• tei — Optimized inference server for embedding workloads.\n"
+                    "• bitsandbytes — Quantization and memory-efficient optimizers.\n"
+                    "• cookbook — Practical, task-oriented recipes across the ecosystem.\n"
+                    "• sentence_transformers — Embedding models, training recipes, similarity/search workflows.\n"
+                    "• ml-games-course — Game-based ML and reinforcement learning experiments.\n"
+                    "• diffusion-course — Diffusion model theory and hands-on practice.\n"
+                    "• ml-for-3d-course — 3D representations, models, and learning techniques.\n"
+                    "• chat-ui — Reference chat interfaces for LLM deployment.\n"
+                    "• leaderboards — Evaluation leaderboards and submission mechanics.\n"
+                    "• lighteval — Lightweight, reproducible LLM evaluation framework.\n"
+                    "• argilla — Data annotation, feedback, and human-in-the-loop workflows.\n"
+                    "• distilabel — Synthetic data generation and distillation pipelines.\n"
+                    "• microsoft-azure — Azure deployment and integration guides.\n"
+                    "• kernels — Lightweight execution environments and notebook-style workflows.\n"
+                    "• google-cloud — GCP deployment and serving workflows.\n"
+                ),
+            },
+        },
+        "required": ["endpoint"],
+    },
+}
+
+HF_DOCS_FETCH_TOOL_SPEC = {
+    "name": "fetch_hf_docs",
+    "description": (
+        "Fetch the full content of a specific HF documentation page. "
+        "Provide the full URL to the doc page (e.g., from explore_hf_docs results). "
+        "Returns the complete markdown content of that page. "
+        "Use explore_hf_docs first to discover available pages."
+    ),
+    "parameters": {
+        "type": "object",
+        "properties": {
+            "url": {
+                "type": "string",
+                "description": (
+                    "The full URL to the documentation page. "
+                    "Example: 'https://huggingface.co/docs/trl/dpo_trainer' "
+                    "The .md extension will be added automatically if not present."
+                ),
+            },
+        },
+        "required": ["url"],
+    },
+}
+
+
+async def _get_api_search_tool_spec() -> dict[str, Any]:
+    """
+    Dynamically generate the OpenAPI tool spec with tag enum populated at runtime
+    This must be called async to fetch the OpenAPI spec and extract tags
+    """
+    spec = await _fetch_openapi_spec()
+    tags = _extract_all_tags(spec)
+
+    return {
+        "name": "search_hf_api_endpoints",
+        "description": (
+            "Search the HuggingFace OpenAPI specification by tag to find related API endpoints. "
+            "Returns all endpoints with the specified tag including curl examples showing how to use them. "
+            "Each result includes the endpoint path, summary, usage example with curl, and response information."
+        ),
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "tag": {
+                    "type": "string",
+                    "enum": tags,
+                    "description": (
+                        "The API tag to search for. Each tag groups related API endpoints. "
+                    ),
+                },
+            },
+            "required": ["tag"],
+        },
+    }

agent/tools/search_docs_tool.py

@@ -0,0 +1,237 @@
+"""
+Search documentation tool that spawns a sub-agent
+The sub-agent has its own agent loop and set of specialized search tools
+"""
+
+import asyncio
+from typing import Any
+
+from litellm.utils import get_max_tokens
+
+from agent.core.session import Session
+
+
+async def create_search_tool_router(github_mcp_config: dict[str, Any] | None = None):
+    """
+    Create a ToolRouter instance for the search sub-agent
+    Async because OpenAPI tool needs to fetch and parse spec at initialization
+
+    Args:
+        github_mcp_config: Optional GitHub MCP server configuration
+    """
+    # Import at runtime to avoid circular dependency
+    from fastmcp import Client
+
+    from agent.core.tools import ToolRouter
+
+    # List of allowed GitHub MCP tools
+    ALLOWED_GITHUB_TOOLS = {
+        "list_pull_requests",
+        "list_issues",
+        "search_code",
+        "search_issues",
+        "search_repositories",
+        "search_users",
+        "get_pull_request_status",
+        "get_pull_request_reviews",
+        "get_pull_request",
+        "get_issue",
+        "get_file_contents",
+    }
+
+    class SearchDocsToolRouter(ToolRouter):
+        """Specialized ToolRouter for the search sub-agent"""
+
+        def __init__(self, github_mcp_config: dict[str, Any] | None = None):
+            self.tools: dict[str, Any] = {}
+            self.mcp_servers: dict[str, dict[str, Any]] = {}
+            self._mcp_initialized = False
+
+            # Initialize MCP client with GitHub server if provided
+            if github_mcp_config:
+                self.mcp_client = Client({"mcpServers": github_mcp_config})
+            else:
+                self.mcp_client = None
+
+        async def initialize_tools(self):
+            """Initialize tools asynchronously"""
+            tools = await make_search_agent_tools()
+            for tool in tools:
+                self.register_tool(tool)
+
+        async def register_mcp_tools(self) -> None:
+            """Register only allowed GitHub MCP tools"""
+            if self.mcp_client is None:
+                return
+
+            tools = await self.mcp_client.list_tools()
+            for tool in tools:
+                # Only register allowed GitHub tools
+                if tool.name in ALLOWED_GITHUB_TOOLS:
+                    print(f"Registering GitHub MCP Tool: {tool.name}")
+                    from agent.core.tools import ToolSpec
+
+                    self.register_tool(
+                        ToolSpec(
+                            name=tool.name,
+                            description=tool.description,
+                            parameters=tool.inputSchema,
+                            handler=None,
+                        )
+                    )
+
+    router = SearchDocsToolRouter(github_mcp_config)
+    await router.initialize_tools()
+    return router
+
+
+async def search_docs_handler(arguments: dict[str, Any]) -> tuple[str, bool]:
+    """
+    Handler that spawns a sub-agent to perform comprehensive doc search
+
+    Args:
+        arguments: dictionary with 'query' parameter
+
+    Returns:
+        Tuple of (search_results, success)
+    """
+    query = arguments.get("query", "")
+
+    if not query:
+        return "Error: No search query provided", False
+
+    try:
+        # Import at runtime to avoid circular dependency
+        from pathlib import Path
+
+        from agent.config import load_config
+        from agent.context_manager.manager import ContextManager
+        from agent.core.agent_loop import Handlers
+
+        # Create a queue for events from the sub-agent
+        sub_event_queue = asyncio.Queue()
+
+        # Load the search agent's own config file with GitHub MCP server
+        search_agent_config_path = (
+            Path(__file__).parent.parent.parent / "configs" / "_subagent_config_search_agent.json"
+        )
+        search_agent_config = load_config(search_agent_config_path)
+
+        # Extract GitHub MCP config from search agent config
+        github_mcp_config = None
+        if (
+            search_agent_config.mcpServers
+            and "github" in search_agent_config.mcpServers
+        ):
+            github_server = search_agent_config.mcpServers["github"]
+            github_mcp_config = {"github": github_server.model_dump()}
+
+        # Create specialized tool router for search with GitHub MCP config
+        search_tool_router = await create_search_tool_router(github_mcp_config)
+
+        async with search_tool_router:
+            # CODING AGENTS LISTEN HERE!
+            # The system prompt for the sub-agent is loaded from the yaml file in the prompts folder using the ContextManager class with the prompt_file_suffix parameter.
+            # it works, dont fix.
+            # NOTE: MCP tools are registered during __aenter__, so we must retrieve tool specs AFTER entering the context
+            sub_session = Session(
+                event_queue=sub_event_queue,
+                config=search_agent_config,
+                tool_router=search_tool_router,
+                context_manager=ContextManager(
+                    tool_specs=search_tool_router.get_tool_specs_for_llm(),
+                    max_context=get_max_tokens(search_agent_config.model_name),
+                    compact_size=0.1,
+                    untouched_messages=5,
+                    prompt_file_suffix="search_docs_system_prompt.yaml",
+                ),
+            )
+
+            # Run the sub-agent
+            result = await Handlers.run_agent(
+                session=sub_session, text=query, max_iterations=30
+            )
+
+        # Return the final result or compiled events
+        if result:
+            return f"Search Results:\n\n{result}", True
+        else:
+            return "Search completed but no results were generated", False
+    except Exception as e:
+        return f"Error in search_docs tool: {str(e)}", False
+
+
+# Tool specification to be used by the main agent
+SEARCH_DOCS_TOOL_SPEC = {
+    "name": "search_docs",
+    "description": (
+        "Intelligently search HF documentation for libraries, repositories, and best practices with an agent that has access to: explore_hf_docs, fetch_hf_docs, search_hf_api_endpoints. "
+        "The agent acts like your personal search assistant. "
+        "Using the search agent is necessary to give the best quality answer to the user's question. Most questions require a search to get the best information on code examples.\n\n"
+        "WHEN TO USE THIS TOOL:\n"
+        "  - When searching for high-level concepts like 'how to do GRPO training on a model?' or 'best way to do inference on a trained model?'\n"
+        "  - When you need to get code examples for intricate ML code patterns like training loops, inference pipelines, data processing, etc.\n\n"
+        "USAGE GUIDELINES:\n"
+        "  1. Launch multiple agents concurrently for better performance.\n"
+        "  2. Be specific in your query - include exact terminology, expected file locations, or code patterns.\n"
+        "  3. Use the query as if you were talking to another engineer. Bad: logger impl Good: where is the logger implemented, we're trying to find out how to log to files.\n"
+        "  4. Make sure to formulate the query in such a way that the agent knows when it's done or has found the result."
+    ),
+    "parameters": {
+        "type": "object",
+        "properties": {
+            "query": {
+                "type": "string",
+                "description": (
+                    "The search query describing to the agent what it should do. Be "
+                    "specific and include technical terms, file types, or expected "
+                    "code patterns to help the agent find relevant code. Formulate "
+                    "the query in a way that makes it clear to the agent when it "
+                    "has found the right thing."
+                ),
+            },
+        },
+        "required": ["query"],
+    },
+}
+
+
+async def make_search_agent_tools():
+    """
+    Create a list of tools for the search agent
+    Async because OpenAPI tool spec needs to be populated at runtime
+    """
+    # Import at runtime to avoid circular dependency
+    from agent.core.tools import ToolSpec
+    from agent.tools._search_agent_tools import (
+        EXPLORE_HF_DOCS_TOOL_SPEC,
+        HF_DOCS_FETCH_TOOL_SPEC,
+        _get_api_search_tool_spec,
+        explore_hf_docs_handler,
+        hf_docs_fetch_handler,
+        search_openapi_handler,
+    )
+
+    # Get the OpenAPI tool spec with dynamically populated tags
+    openapi_spec = await _get_api_search_tool_spec()
+
+    return [
+        ToolSpec(
+            name=EXPLORE_HF_DOCS_TOOL_SPEC["name"],
+            description=EXPLORE_HF_DOCS_TOOL_SPEC["description"],
+            parameters=EXPLORE_HF_DOCS_TOOL_SPEC["parameters"],
+            handler=explore_hf_docs_handler,
+        ),
+        ToolSpec(
+            name=HF_DOCS_FETCH_TOOL_SPEC["name"],
+            description=HF_DOCS_FETCH_TOOL_SPEC["description"],
+            parameters=HF_DOCS_FETCH_TOOL_SPEC["parameters"],
+            handler=hf_docs_fetch_handler,
+        ),
+        ToolSpec(
+            name=openapi_spec["name"],
+            description=openapi_spec["description"],
+            parameters=openapi_spec["parameters"],
+            handler=search_openapi_handler,
+        ),
+    ]

configs/_subagent_config_search_agent.json

@@ -0,0 +1,12 @@
+{
+  "model_name": "anthropic/claude-haiku-4-5",
+  "mcpServers": {
+    "github": {
+      "transport": "http",
+      "url": "https://api.githubcopilot.com/mcp/",
+      "headers": {
+        "Authorization": "Bearer ${GITHUB_TOKEN}"
+      }
+    }
+  }
+}

agent/config_mcp_example.json → configs/main_agent_config.json

@@ -1,7 +1,5 @@
 {
   "model_name": "anthropic/claude-sonnet-4-5-20250929",
-  "tools": [],
-  "system_prompt_path": "",
   "mcpServers": {
     "hf-mcp-server": {
       "transport": "http",

pyproject.toml

@@ -20,4 +20,5 @@ dependencies = [
     "transformers>=2.3.0",
     "torch>=2.9.1",
     "pytest>=9.0.2",
+    "trafilatura>=2.0.0",
 ]

run_search_agent.py

@@ -0,0 +1,142 @@
+"""
+Standalone test script for the search sub-agent
+Run with: uv run python test_search_agent.py
+"""
+
+import asyncio
+
+from litellm.utils import get_max_tokens
+
+from agent.config import Config
+from agent.context_manager.manager import ContextManager
+from agent.core.agent_loop import Handlers
+from agent.core.session import Session
+from agent.tools.search_docs_tool import create_search_tool_router
+
+
+async def test_search_agent(query: str):
+    """Test the search sub-agent with a query"""
+    print(f"Testing search agent with query: {query}\n")
+    print("=" * 60)
+
+    # Create event queue for the sub-agent
+    sub_event_queue = asyncio.Queue()
+
+    # Create search tool router
+    search_tool_router = await create_search_tool_router()
+
+    # Create config
+    sub_config = Config(
+        model_name="anthropic/claude-haiku-4-5",
+    )
+
+    # Create session with custom system prompt
+    sub_session = Session(
+        event_queue=sub_event_queue,
+        config=sub_config,
+        tool_router=search_tool_router,
+        context_manager=ContextManager(
+            tool_specs=search_tool_router.get_tool_specs_for_llm(),
+            max_context=get_max_tokens(sub_config.model_name),
+            compact_size=0.1,
+            untouched_messages=5,
+            prompt_file_suffix="search_docs_system_prompt.yaml",
+        ),
+    )
+
+    # Event listener to show what the sub-agent is doing
+    async def event_monitor():
+        while True:
+            try:
+                event = await asyncio.wait_for(sub_event_queue.get(), timeout=1.0)
+
+                if event.event_type == "assistant_message":
+                    content = event.data.get("content", "") if event.data else ""
+                    if content:
+                        print(f"\n🤖 Sub-agent: {content}\n")
+
+                elif event.event_type == "tool_call":
+                    tool_name = event.data.get("tool", "") if event.data else ""
+                    arguments = event.data.get("arguments", {}) if event.data else {}
+                    print(f"🔧 Tool call: {tool_name}")
+                    print(f"   Args: {arguments}")
+
+                elif event.event_type == "tool_output":
+                    output = event.data.get("output", "") if event.data else ""
+                    success = event.data.get("success", False) if event.data else False
+                    status = "✅" if success else "❌"
+
+                    print(f"{status} Tool output: {output}\n")
+
+                elif event.event_type == "turn_complete":
+                    print("✅ Sub-agent turn complete")
+                    break
+
+            except asyncio.TimeoutError:
+                # Check if agent is still running
+                continue
+            except Exception as e:
+                print(f"⚠️  Event error: {e}")
+                break
+
+    # Run the sub-agent and event monitor concurrently
+    async with search_tool_router:
+        monitor_task = asyncio.create_task(event_monitor())
+
+        result = await Handlers.run_agent(
+            session=sub_session, text=query, max_iterations=30
+        )
+
+        # Wait for event monitor to finish
+        await asyncio.wait_for(monitor_task, timeout=5.0)
+
+    print("\n" + "=" * 60)
+    print("FINAL RESULT:")
+    print("=" * 60)
+    if result:
+        print(result)
+    else:
+        print("No result returned")
+    print("=" * 60)
+
+
+async def main():
+    """Main test function"""
+    print("🧪 Search Sub-Agent Test\n")
+
+    # Example queries to test
+    test_queries = [
+        # "Explore the TRL documentation structure and find information about DPO trainer",
+        # "is there a way to get the logs from a served huggingface space",
+        # "How do I train GLM4.7 with a GRPO training loop with trl with llm judge as a reward model for training on hle?"
+        "can i stream logs through the api for a served huggingface space",
+    ]
+
+    for i, query in enumerate(test_queries, 1):
+        print(f"\n{'=' * 60}")
+        print(f"TEST {i}/{len(test_queries)}")
+        print(f"{'=' * 60}\n")
+
+        try:
+            await test_search_agent(query)
+        except Exception as e:
+            print(f"\n❌ Test failed: {e}")
+            import traceback
+
+            traceback.print_exc()
+
+        if i < len(test_queries):
+            print("\n\nPress Enter to continue to next test...")
+            input()
+
+
+if __name__ == "__main__":
+    try:
+        asyncio.run(main())
+    except KeyboardInterrupt:
+        print("\n\n⚠️  Test interrupted")
+    except Exception as e:
+        print(f"\n❌ Error: {e}")
+        import traceback
+
+        traceback.print_exc()

uv.lock

@@ -230,6 +230,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/f8/aa/5082412d1ee302e9e7d80b6949bc4d2a8fa1149aaab610c5fc24709605d6/authlib-1.6.5-py2.py3-none-any.whl", hash = "sha256:3e0e0507807f842b02175507bdee8957a1d5707fd4afb17c32fb43fee90b6e3a", size = 243608, upload-time = "2025-10-02T13:36:07.637Z" },
 ]
 
+[[package]]
+name = "babel"
+version = "2.17.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/7d/6b/d52e42361e1aa00709585ecc30b3f9684b3ab62530771402248b1b1d6240/babel-2.17.0.tar.gz", hash = "sha256:0c54cffb19f690cdcc52a3b50bcbf71e07a808d1c80d549f2459b9d2cf0afb9d", size = 9951852, upload-time = "2025-02-01T15:17:41.026Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/b7/b8/3fe70c75fe32afc4bb507f75563d39bc5642255d1d94f1f23604725780bf/babel-2.17.0-py3-none-any.whl", hash = "sha256:4d0b53093fdfb4b21c92b5213dba5a1b23885afa8383709427046b21c366e5f2", size = 10182537, upload-time = "2025-02-01T15:17:37.39Z" },
+]
+
 [[package]]
 name = "beartype"
 version = "0.22.6"
@@ -433,6 +442,20 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" },
 ]
 
+[[package]]
+name = "courlan"
+version = "1.3.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "babel" },
+    { name = "tld" },
+    { name = "urllib3" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/6f/54/6d6ceeff4bed42e7a10d6064d35ee43a810e7b3e8beb4abeae8cff4713ae/courlan-1.3.2.tar.gz", hash = "sha256:0b66f4db3a9c39a6e22dd247c72cfaa57d68ea660e94bb2c84ec7db8712af190", size = 206382, upload-time = "2024-10-29T16:40:20.994Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/8e/ca/6a667ccbe649856dcd3458bab80b016681b274399d6211187c6ab969fc50/courlan-1.3.2-py3-none-any.whl", hash = "sha256:d0dab52cf5b5b1000ee2839fbc2837e93b2514d3cb5bb61ae158a55b7a04c6be", size = 33848, upload-time = "2024-10-29T16:40:18.325Z" },
+]
+
 [[package]]
 name = "cryptography"
 version = "46.0.3"
@@ -529,6 +552,21 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/3b/5e/6f8d874366788ad5d549e9ba258037d974dda6e004843be1bda794571701/datasets-4.4.1-py3-none-any.whl", hash = "sha256:c1163de5211e42546079ab355cc0250c7e6db16eb209ac5ac6252f801f596c44", size = 511591, upload-time = "2025-11-05T16:00:36.365Z" },
 ]
 
+[[package]]
+name = "dateparser"
+version = "1.2.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "python-dateutil" },
+    { name = "pytz" },
+    { name = "regex" },
+    { name = "tzlocal" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/a9/30/064144f0df1749e7bb5faaa7f52b007d7c2d08ec08fed8411aba87207f68/dateparser-1.2.2.tar.gz", hash = "sha256:986316f17cb8cdc23ea8ce563027c5ef12fc725b6fb1d137c14ca08777c5ecf7", size = 329840, upload-time = "2025-06-26T09:29:23.211Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/87/22/f020c047ae1346613db9322638186468238bcfa8849b4668a22b97faad65/dateparser-1.2.2-py3-none-any.whl", hash = "sha256:5a5d7211a09013499867547023a2a0c91d5a27d15dd4dbcea676ea9fe66f2482", size = 315453, upload-time = "2025-06-26T09:29:21.412Z" },
+]
+
 [[package]]
 name = "debugpy"
 version = "1.8.17"
@@ -898,6 +936,7 @@ dependencies = [
     { name = "requests" },
     { name = "tenacity" },
     { name = "torch" },
+    { name = "trafilatura" },
     { name = "transformers" },
 ]
 
@@ -917,6 +956,7 @@ requires-dist = [
     { name = "requests", specifier = ">=2.32.5" },
     { name = "tenacity", specifier = ">=8.0.0" },
     { name = "torch", specifier = ">=2.9.1" },
+    { name = "trafilatura", specifier = ">=2.0.0" },
     { name = "transformers", specifier = ">=2.3.0" },
 ]
 
@@ -949,6 +989,22 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/cb/44/870d44b30e1dcfb6a65932e3e1506c103a8a5aea9103c337e7a53180322c/hf_xet-1.2.0-cp37-abi3-win_amd64.whl", hash = "sha256:e6584a52253f72c9f52f9e549d5895ca7a471608495c4ecaa6cc73dba2b24d69", size = 2905735, upload-time = "2025-10-24T19:04:35.928Z" },
 ]
 
+[[package]]
+name = "htmldate"
+version = "1.9.4"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "charset-normalizer" },
+    { name = "dateparser" },
+    { name = "lxml" },
+    { name = "python-dateutil" },
+    { name = "urllib3" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/9d/10/ead9dabc999f353c3aa5d0dc0835b1e355215a5ecb489a7f4ef2ddad5e33/htmldate-1.9.4.tar.gz", hash = "sha256:1129063e02dd0354b74264de71e950c0c3fcee191178321418ccad2074cc8ed0", size = 44690, upload-time = "2025-11-04T17:46:44.983Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/a1/bd/adfcdaaad5805c0c5156aeefd64c1e868c05e9c1cd6fd21751f168cd88c7/htmldate-1.9.4-py3-none-any.whl", hash = "sha256:1b94bcc4e08232a5b692159903acf95548b6a7492dddca5bb123d89d6325921c", size = 31558, upload-time = "2025-11-04T17:46:43.258Z" },
+]
+
 [[package]]
 name = "httpcore"
 version = "1.0.9"
@@ -1391,6 +1447,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" },
 ]
 
+[[package]]
+name = "justext"
+version = "3.0.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "lxml", extra = ["html-clean"] },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/49/f3/45890c1b314f0d04e19c1c83d534e611513150939a7cf039664d9ab1e649/justext-3.0.2.tar.gz", hash = "sha256:13496a450c44c4cd5b5a75a5efcd9996066d2a189794ea99a49949685a0beb05", size = 828521, upload-time = "2025-02-25T20:21:49.934Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/f2/ac/52f4e86d1924a7fc05af3aeb34488570eccc39b4af90530dd6acecdf16b5/justext-3.0.2-py2.py3-none-any.whl", hash = "sha256:62b1c562b15c3c6265e121cc070874243a443bfd53060e869393f09d6b6cc9a7", size = 837940, upload-time = "2025-02-25T20:21:44.179Z" },
+]
+
 [[package]]
 name = "keyring"
 version = "25.7.0"
@@ -1497,6 +1565,103 @@ all = [
     { name = "opentelemetry-instrumentation-weaviate" },
 ]
 
+[[package]]
+name = "lxml"
+version = "6.0.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/aa/88/262177de60548e5a2bfc46ad28232c9e9cbde697bd94132aeb80364675cb/lxml-6.0.2.tar.gz", hash = "sha256:cd79f3367bd74b317dda655dc8fcfa304d9eb6e4fb06b7168c5cf27f96e0cd62", size = 4073426, upload-time = "2025-09-22T04:04:59.287Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/f3/c8/8ff2bc6b920c84355146cd1ab7d181bc543b89241cfb1ebee824a7c81457/lxml-6.0.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a59f5448ba2ceccd06995c95ea59a7674a10de0810f2ce90c9006f3cbc044456", size = 8661887, upload-time = "2025-09-22T04:01:17.265Z" },
+    { url = "https://files.pythonhosted.org/packages/37/6f/9aae1008083bb501ef63284220ce81638332f9ccbfa53765b2b7502203cf/lxml-6.0.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e8113639f3296706fbac34a30813929e29247718e88173ad849f57ca59754924", size = 4667818, upload-time = "2025-09-22T04:01:19.688Z" },
+    { url = "https://files.pythonhosted.org/packages/f1/ca/31fb37f99f37f1536c133476674c10b577e409c0a624384147653e38baf2/lxml-6.0.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a8bef9b9825fa8bc816a6e641bb67219489229ebc648be422af695f6e7a4fa7f", size = 4950807, upload-time = "2025-09-22T04:01:21.487Z" },
+    { url = "https://files.pythonhosted.org/packages/da/87/f6cb9442e4bada8aab5ae7e1046264f62fdbeaa6e3f6211b93f4c0dd97f1/lxml-6.0.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:65ea18d710fd14e0186c2f973dc60bb52039a275f82d3c44a0e42b43440ea534", size = 5109179, upload-time = "2025-09-22T04:01:23.32Z" },
+    { url = "https://files.pythonhosted.org/packages/c8/20/a7760713e65888db79bbae4f6146a6ae5c04e4a204a3c48896c408cd6ed2/lxml-6.0.2-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c371aa98126a0d4c739ca93ceffa0fd7a5d732e3ac66a46e74339acd4d334564", size = 5023044, upload-time = "2025-09-22T04:01:25.118Z" },
+    { url = "https://files.pythonhosted.org/packages/a2/b0/7e64e0460fcb36471899f75831509098f3fd7cd02a3833ac517433cb4f8f/lxml-6.0.2-cp312-cp312-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:700efd30c0fa1a3581d80a748157397559396090a51d306ea59a70020223d16f", size = 5359685, upload-time = "2025-09-22T04:01:27.398Z" },
+    { url = "https://files.pythonhosted.org/packages/b9/e1/e5df362e9ca4e2f48ed6411bd4b3a0ae737cc842e96877f5bf9428055ab4/lxml-6.0.2-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c33e66d44fe60e72397b487ee92e01da0d09ba2d66df8eae42d77b6d06e5eba0", size = 5654127, upload-time = "2025-09-22T04:01:29.629Z" },
+    { url = "https://files.pythonhosted.org/packages/c6/d1/232b3309a02d60f11e71857778bfcd4acbdb86c07db8260caf7d008b08f8/lxml-6.0.2-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90a345bbeaf9d0587a3aaffb7006aa39ccb6ff0e96a57286c0cb2fd1520ea192", size = 5253958, upload-time = "2025-09-22T04:01:31.535Z" },
+    { url = "https://files.pythonhosted.org/packages/35/35/d955a070994725c4f7d80583a96cab9c107c57a125b20bb5f708fe941011/lxml-6.0.2-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:064fdadaf7a21af3ed1dcaa106b854077fbeada827c18f72aec9346847cd65d0", size = 4711541, upload-time = "2025-09-22T04:01:33.801Z" },
+    { url = "https://files.pythonhosted.org/packages/1e/be/667d17363b38a78c4bd63cfd4b4632029fd68d2c2dc81f25ce9eb5224dd5/lxml-6.0.2-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fbc74f42c3525ac4ffa4b89cbdd00057b6196bcefe8bce794abd42d33a018092", size = 5267426, upload-time = "2025-09-22T04:01:35.639Z" },
+    { url = "https://files.pythonhosted.org/packages/ea/47/62c70aa4a1c26569bc958c9ca86af2bb4e1f614e8c04fb2989833874f7ae/lxml-6.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6ddff43f702905a4e32bc24f3f2e2edfe0f8fde3277d481bffb709a4cced7a1f", size = 5064917, upload-time = "2025-09-22T04:01:37.448Z" },
+    { url = "https://files.pythonhosted.org/packages/bd/55/6ceddaca353ebd0f1908ef712c597f8570cc9c58130dbb89903198e441fd/lxml-6.0.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:6da5185951d72e6f5352166e3da7b0dc27aa70bd1090b0eb3f7f7212b53f1bb8", size = 4788795, upload-time = "2025-09-22T04:01:39.165Z" },
+    { url = "https://files.pythonhosted.org/packages/cf/e8/fd63e15da5e3fd4c2146f8bbb3c14e94ab850589beab88e547b2dbce22e1/lxml-6.0.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:57a86e1ebb4020a38d295c04fc79603c7899e0df71588043eb218722dabc087f", size = 5676759, upload-time = "2025-09-22T04:01:41.506Z" },
+    { url = "https://files.pythonhosted.org/packages/76/47/b3ec58dc5c374697f5ba37412cd2728f427d056315d124dd4b61da381877/lxml-6.0.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:2047d8234fe735ab77802ce5f2297e410ff40f5238aec569ad7c8e163d7b19a6", size = 5255666, upload-time = "2025-09-22T04:01:43.363Z" },
+    { url = "https://files.pythonhosted.org/packages/19/93/03ba725df4c3d72afd9596eef4a37a837ce8e4806010569bedfcd2cb68fd/lxml-6.0.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f91fd2b2ea15a6800c8e24418c0775a1694eefc011392da73bc6cef2623b322", size = 5277989, upload-time = "2025-09-22T04:01:45.215Z" },
+    { url = "https://files.pythonhosted.org/packages/c6/80/c06de80bfce881d0ad738576f243911fccf992687ae09fd80b734712b39c/lxml-6.0.2-cp312-cp312-win32.whl", hash = "sha256:3ae2ce7d6fedfb3414a2b6c5e20b249c4c607f72cb8d2bb7cc9c6ec7c6f4e849", size = 3611456, upload-time = "2025-09-22T04:01:48.243Z" },
+    { url = "https://files.pythonhosted.org/packages/f7/d7/0cdfb6c3e30893463fb3d1e52bc5f5f99684a03c29a0b6b605cfae879cd5/lxml-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:72c87e5ee4e58a8354fb9c7c84cbf95a1c8236c127a5d1b7683f04bed8361e1f", size = 4011793, upload-time = "2025-09-22T04:01:50.042Z" },
+    { url = "https://files.pythonhosted.org/packages/ea/7b/93c73c67db235931527301ed3785f849c78991e2e34f3fd9a6663ffda4c5/lxml-6.0.2-cp312-cp312-win_arm64.whl", hash = "sha256:61cb10eeb95570153e0c0e554f58df92ecf5109f75eacad4a95baa709e26c3d6", size = 3672836, upload-time = "2025-09-22T04:01:52.145Z" },
+    { url = "https://files.pythonhosted.org/packages/53/fd/4e8f0540608977aea078bf6d79f128e0e2c2bba8af1acf775c30baa70460/lxml-6.0.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:9b33d21594afab46f37ae58dfadd06636f154923c4e8a4d754b0127554eb2e77", size = 8648494, upload-time = "2025-09-22T04:01:54.242Z" },
+    { url = "https://files.pythonhosted.org/packages/5d/f4/2a94a3d3dfd6c6b433501b8d470a1960a20ecce93245cf2db1706adf6c19/lxml-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:6c8963287d7a4c5c9a432ff487c52e9c5618667179c18a204bdedb27310f022f", size = 4661146, upload-time = "2025-09-22T04:01:56.282Z" },
+    { url = "https://files.pythonhosted.org/packages/25/2e/4efa677fa6b322013035d38016f6ae859d06cac67437ca7dc708a6af7028/lxml-6.0.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1941354d92699fb5ffe6ed7b32f9649e43c2feb4b97205f75866f7d21aa91452", size = 4946932, upload-time = "2025-09-22T04:01:58.989Z" },
+    { url = "https://files.pythonhosted.org/packages/ce/0f/526e78a6d38d109fdbaa5049c62e1d32fdd70c75fb61c4eadf3045d3d124/lxml-6.0.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bb2f6ca0ae2d983ded09357b84af659c954722bbf04dea98030064996d156048", size = 5100060, upload-time = "2025-09-22T04:02:00.812Z" },
+    { url = "https://files.pythonhosted.org/packages/81/76/99de58d81fa702cc0ea7edae4f4640416c2062813a00ff24bd70ac1d9c9b/lxml-6.0.2-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eb2a12d704f180a902d7fa778c6d71f36ceb7b0d317f34cdc76a5d05aa1dd1df", size = 5019000, upload-time = "2025-09-22T04:02:02.671Z" },
+    { url = "https://files.pythonhosted.org/packages/b5/35/9e57d25482bc9a9882cb0037fdb9cc18f4b79d85df94fa9d2a89562f1d25/lxml-6.0.2-cp313-cp313-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:6ec0e3f745021bfed19c456647f0298d60a24c9ff86d9d051f52b509663feeb1", size = 5348496, upload-time = "2025-09-22T04:02:04.904Z" },
+    { url = "https://files.pythonhosted.org/packages/a6/8e/cb99bd0b83ccc3e8f0f528e9aa1f7a9965dfec08c617070c5db8d63a87ce/lxml-6.0.2-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:846ae9a12d54e368933b9759052d6206a9e8b250291109c48e350c1f1f49d916", size = 5643779, upload-time = "2025-09-22T04:02:06.689Z" },
+    { url = "https://files.pythonhosted.org/packages/d0/34/9e591954939276bb679b73773836c6684c22e56d05980e31d52a9a8deb18/lxml-6.0.2-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ef9266d2aa545d7374938fb5c484531ef5a2ec7f2d573e62f8ce722c735685fd", size = 5244072, upload-time = "2025-09-22T04:02:08.587Z" },
+    { url = "https://files.pythonhosted.org/packages/8d/27/b29ff065f9aaca443ee377aff699714fcbffb371b4fce5ac4ca759e436d5/lxml-6.0.2-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:4077b7c79f31755df33b795dc12119cb557a0106bfdab0d2c2d97bd3cf3dffa6", size = 4718675, upload-time = "2025-09-22T04:02:10.783Z" },
+    { url = "https://files.pythonhosted.org/packages/2b/9f/f756f9c2cd27caa1a6ef8c32ae47aadea697f5c2c6d07b0dae133c244fbe/lxml-6.0.2-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a7c5d5e5f1081955358533be077166ee97ed2571d6a66bdba6ec2f609a715d1a", size = 5255171, upload-time = "2025-09-22T04:02:12.631Z" },
+    { url = "https://files.pythonhosted.org/packages/61/46/bb85ea42d2cb1bd8395484fd72f38e3389611aa496ac7772da9205bbda0e/lxml-6.0.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8f8d0cbd0674ee89863a523e6994ac25fd5be9c8486acfc3e5ccea679bad2679", size = 5057175, upload-time = "2025-09-22T04:02:14.718Z" },
+    { url = "https://files.pythonhosted.org/packages/95/0c/443fc476dcc8e41577f0af70458c50fe299a97bb6b7505bb1ae09aa7f9ac/lxml-6.0.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:2cbcbf6d6e924c28f04a43f3b6f6e272312a090f269eff68a2982e13e5d57659", size = 4785688, upload-time = "2025-09-22T04:02:16.957Z" },
+    { url = "https://files.pythonhosted.org/packages/48/78/6ef0b359d45bb9697bc5a626e1992fa5d27aa3f8004b137b2314793b50a0/lxml-6.0.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:dfb874cfa53340009af6bdd7e54ebc0d21012a60a4e65d927c2e477112e63484", size = 5660655, upload-time = "2025-09-22T04:02:18.815Z" },
+    { url = "https://files.pythonhosted.org/packages/ff/ea/e1d33808f386bc1339d08c0dcada6e4712d4ed8e93fcad5f057070b7988a/lxml-6.0.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:fb8dae0b6b8b7f9e96c26fdd8121522ce5de9bb5538010870bd538683d30e9a2", size = 5247695, upload-time = "2025-09-22T04:02:20.593Z" },
+    { url = "https://files.pythonhosted.org/packages/4f/47/eba75dfd8183673725255247a603b4ad606f4ae657b60c6c145b381697da/lxml-6.0.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:358d9adae670b63e95bc59747c72f4dc97c9ec58881d4627fe0120da0f90d314", size = 5269841, upload-time = "2025-09-22T04:02:22.489Z" },
+    { url = "https://files.pythonhosted.org/packages/76/04/5c5e2b8577bc936e219becb2e98cdb1aca14a4921a12995b9d0c523502ae/lxml-6.0.2-cp313-cp313-win32.whl", hash = "sha256:e8cd2415f372e7e5a789d743d133ae474290a90b9023197fd78f32e2dc6873e2", size = 3610700, upload-time = "2025-09-22T04:02:24.465Z" },
+    { url = "https://files.pythonhosted.org/packages/fe/0a/4643ccc6bb8b143e9f9640aa54e38255f9d3b45feb2cbe7ae2ca47e8782e/lxml-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:b30d46379644fbfc3ab81f8f82ae4de55179414651f110a1514f0b1f8f6cb2d7", size = 4010347, upload-time = "2025-09-22T04:02:26.286Z" },
+    { url = "https://files.pythonhosted.org/packages/31/ef/dcf1d29c3f530577f61e5fe2f1bd72929acf779953668a8a47a479ae6f26/lxml-6.0.2-cp313-cp313-win_arm64.whl", hash = "sha256:13dcecc9946dca97b11b7c40d29fba63b55ab4170d3c0cf8c0c164343b9bfdcf", size = 3671248, upload-time = "2025-09-22T04:02:27.918Z" },
+    { url = "https://files.pythonhosted.org/packages/03/15/d4a377b385ab693ce97b472fe0c77c2b16ec79590e688b3ccc71fba19884/lxml-6.0.2-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:b0c732aa23de8f8aec23f4b580d1e52905ef468afb4abeafd3fec77042abb6fe", size = 8659801, upload-time = "2025-09-22T04:02:30.113Z" },
+    { url = "https://files.pythonhosted.org/packages/c8/e8/c128e37589463668794d503afaeb003987373c5f94d667124ffd8078bbd9/lxml-6.0.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4468e3b83e10e0317a89a33d28f7aeba1caa4d1a6fd457d115dd4ffe90c5931d", size = 4659403, upload-time = "2025-09-22T04:02:32.119Z" },
+    { url = "https://files.pythonhosted.org/packages/00/ce/74903904339decdf7da7847bb5741fc98a5451b42fc419a86c0c13d26fe2/lxml-6.0.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:abd44571493973bad4598a3be7e1d807ed45aa2adaf7ab92ab7c62609569b17d", size = 4966974, upload-time = "2025-09-22T04:02:34.155Z" },
+    { url = "https://files.pythonhosted.org/packages/1f/d3/131dec79ce61c5567fecf82515bd9bc36395df42501b50f7f7f3bd065df0/lxml-6.0.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:370cd78d5855cfbffd57c422851f7d3864e6ae72d0da615fca4dad8c45d375a5", size = 5102953, upload-time = "2025-09-22T04:02:36.054Z" },
+    { url = "https://files.pythonhosted.org/packages/3a/ea/a43ba9bb750d4ffdd885f2cd333572f5bb900cd2408b67fdda07e85978a0/lxml-6.0.2-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:901e3b4219fa04ef766885fb40fa516a71662a4c61b80c94d25336b4934b71c0", size = 5055054, upload-time = "2025-09-22T04:02:38.154Z" },
+    { url = "https://files.pythonhosted.org/packages/60/23/6885b451636ae286c34628f70a7ed1fcc759f8d9ad382d132e1c8d3d9bfd/lxml-6.0.2-cp314-cp314-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:a4bf42d2e4cf52c28cc1812d62426b9503cdb0c87a6de81442626aa7d69707ba", size = 5352421, upload-time = "2025-09-22T04:02:40.413Z" },
+    { url = "https://files.pythonhosted.org/packages/48/5b/fc2ddfc94ddbe3eebb8e9af6e3fd65e2feba4967f6a4e9683875c394c2d8/lxml-6.0.2-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b2c7fdaa4d7c3d886a42534adec7cfac73860b89b4e5298752f60aa5984641a0", size = 5673684, upload-time = "2025-09-22T04:02:42.288Z" },
+    { url = "https://files.pythonhosted.org/packages/29/9c/47293c58cc91769130fbf85531280e8cc7868f7fbb6d92f4670071b9cb3e/lxml-6.0.2-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98a5e1660dc7de2200b00d53fa00bcd3c35a3608c305d45a7bbcaf29fa16e83d", size = 5252463, upload-time = "2025-09-22T04:02:44.165Z" },
+    { url = "https://files.pythonhosted.org/packages/9b/da/ba6eceb830c762b48e711ded880d7e3e89fc6c7323e587c36540b6b23c6b/lxml-6.0.2-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:dc051506c30b609238d79eda75ee9cab3e520570ec8219844a72a46020901e37", size = 4698437, upload-time = "2025-09-22T04:02:46.524Z" },
+    { url = "https://files.pythonhosted.org/packages/a5/24/7be3f82cb7990b89118d944b619e53c656c97dc89c28cfb143fdb7cd6f4d/lxml-6.0.2-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8799481bbdd212470d17513a54d568f44416db01250f49449647b5ab5b5dccb9", size = 5269890, upload-time = "2025-09-22T04:02:48.812Z" },
+    { url = "https://files.pythonhosted.org/packages/1b/bd/dcfb9ea1e16c665efd7538fc5d5c34071276ce9220e234217682e7d2c4a5/lxml-6.0.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9261bb77c2dab42f3ecd9103951aeca2c40277701eb7e912c545c1b16e0e4917", size = 5097185, upload-time = "2025-09-22T04:02:50.746Z" },
+    { url = "https://files.pythonhosted.org/packages/21/04/a60b0ff9314736316f28316b694bccbbabe100f8483ad83852d77fc7468e/lxml-6.0.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:65ac4a01aba353cfa6d5725b95d7aed6356ddc0a3cd734de00124d285b04b64f", size = 4745895, upload-time = "2025-09-22T04:02:52.968Z" },
+    { url = "https://files.pythonhosted.org/packages/d6/bd/7d54bd1846e5a310d9c715921c5faa71cf5c0853372adf78aee70c8d7aa2/lxml-6.0.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:b22a07cbb82fea98f8a2fd814f3d1811ff9ed76d0fc6abc84eb21527596e7cc8", size = 5695246, upload-time = "2025-09-22T04:02:54.798Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/32/5643d6ab947bc371da21323acb2a6e603cedbe71cb4c99c8254289ab6f4e/lxml-6.0.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:d759cdd7f3e055d6bc8d9bec3ad905227b2e4c785dc16c372eb5b5e83123f48a", size = 5260797, upload-time = "2025-09-22T04:02:57.058Z" },
+    { url = "https://files.pythonhosted.org/packages/33/da/34c1ec4cff1eea7d0b4cd44af8411806ed943141804ac9c5d565302afb78/lxml-6.0.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:945da35a48d193d27c188037a05fec5492937f66fb1958c24fc761fb9d40d43c", size = 5277404, upload-time = "2025-09-22T04:02:58.966Z" },
+    { url = "https://files.pythonhosted.org/packages/82/57/4eca3e31e54dc89e2c3507e1cd411074a17565fa5ffc437c4ae0a00d439e/lxml-6.0.2-cp314-cp314-win32.whl", hash = "sha256:be3aaa60da67e6153eb15715cc2e19091af5dc75faef8b8a585aea372507384b", size = 3670072, upload-time = "2025-09-22T04:03:38.05Z" },
+    { url = "https://files.pythonhosted.org/packages/e3/e0/c96cf13eccd20c9421ba910304dae0f619724dcf1702864fd59dd386404d/lxml-6.0.2-cp314-cp314-win_amd64.whl", hash = "sha256:fa25afbadead523f7001caf0c2382afd272c315a033a7b06336da2637d92d6ed", size = 4080617, upload-time = "2025-09-22T04:03:39.835Z" },
+    { url = "https://files.pythonhosted.org/packages/d5/5d/b3f03e22b3d38d6f188ef044900a9b29b2fe0aebb94625ce9fe244011d34/lxml-6.0.2-cp314-cp314-win_arm64.whl", hash = "sha256:063eccf89df5b24e361b123e257e437f9e9878f425ee9aae3144c77faf6da6d8", size = 3754930, upload-time = "2025-09-22T04:03:41.565Z" },
+    { url = "https://files.pythonhosted.org/packages/5e/5c/42c2c4c03554580708fc738d13414801f340c04c3eff90d8d2d227145275/lxml-6.0.2-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:6162a86d86893d63084faaf4ff937b3daea233e3682fb4474db07395794fa80d", size = 8910380, upload-time = "2025-09-22T04:03:01.645Z" },
+    { url = "https://files.pythonhosted.org/packages/bf/4f/12df843e3e10d18d468a7557058f8d3733e8b6e12401f30b1ef29360740f/lxml-6.0.2-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:414aaa94e974e23a3e92e7ca5b97d10c0cf37b6481f50911032c69eeb3991bba", size = 4775632, upload-time = "2025-09-22T04:03:03.814Z" },
+    { url = "https://files.pythonhosted.org/packages/e4/0c/9dc31e6c2d0d418483cbcb469d1f5a582a1cd00a1f4081953d44051f3c50/lxml-6.0.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:48461bd21625458dd01e14e2c38dd0aea69addc3c4f960c30d9f59d7f93be601", size = 4975171, upload-time = "2025-09-22T04:03:05.651Z" },
+    { url = "https://files.pythonhosted.org/packages/e7/2b/9b870c6ca24c841bdd887504808f0417aa9d8d564114689266f19ddf29c8/lxml-6.0.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:25fcc59afc57d527cfc78a58f40ab4c9b8fd096a9a3f964d2781ffb6eb33f4ed", size = 5110109, upload-time = "2025-09-22T04:03:07.452Z" },
+    { url = "https://files.pythonhosted.org/packages/bf/0c/4f5f2a4dd319a178912751564471355d9019e220c20d7db3fb8307ed8582/lxml-6.0.2-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5179c60288204e6ddde3f774a93350177e08876eaf3ab78aa3a3649d43eb7d37", size = 5041061, upload-time = "2025-09-22T04:03:09.297Z" },
+    { url = "https://files.pythonhosted.org/packages/12/64/554eed290365267671fe001a20d72d14f468ae4e6acef1e179b039436967/lxml-6.0.2-cp314-cp314t-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:967aab75434de148ec80597b75062d8123cadf2943fb4281f385141e18b21338", size = 5306233, upload-time = "2025-09-22T04:03:11.651Z" },
+    { url = "https://files.pythonhosted.org/packages/7a/31/1d748aa275e71802ad9722df32a7a35034246b42c0ecdd8235412c3396ef/lxml-6.0.2-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d100fcc8930d697c6561156c6810ab4a508fb264c8b6779e6e61e2ed5e7558f9", size = 5604739, upload-time = "2025-09-22T04:03:13.592Z" },
+    { url = "https://files.pythonhosted.org/packages/8f/41/2c11916bcac09ed561adccacceaedd2bf0e0b25b297ea92aab99fd03d0fa/lxml-6.0.2-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2ca59e7e13e5981175b8b3e4ab84d7da57993eeff53c07764dcebda0d0e64ecd", size = 5225119, upload-time = "2025-09-22T04:03:15.408Z" },
+    { url = "https://files.pythonhosted.org/packages/99/05/4e5c2873d8f17aa018e6afde417c80cc5d0c33be4854cce3ef5670c49367/lxml-6.0.2-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:957448ac63a42e2e49531b9d6c0fa449a1970dbc32467aaad46f11545be9af1d", size = 4633665, upload-time = "2025-09-22T04:03:17.262Z" },
+    { url = "https://files.pythonhosted.org/packages/0f/c9/dcc2da1bebd6275cdc723b515f93edf548b82f36a5458cca3578bc899332/lxml-6.0.2-cp314-cp314t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b7fc49c37f1786284b12af63152fe1d0990722497e2d5817acfe7a877522f9a9", size = 5234997, upload-time = "2025-09-22T04:03:19.14Z" },
+    { url = "https://files.pythonhosted.org/packages/9c/e2/5172e4e7468afca64a37b81dba152fc5d90e30f9c83c7c3213d6a02a5ce4/lxml-6.0.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e19e0643cc936a22e837f79d01a550678da8377d7d801a14487c10c34ee49c7e", size = 5090957, upload-time = "2025-09-22T04:03:21.436Z" },
+    { url = "https://files.pythonhosted.org/packages/a5/b3/15461fd3e5cd4ddcb7938b87fc20b14ab113b92312fc97afe65cd7c85de1/lxml-6.0.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:1db01e5cf14345628e0cbe71067204db658e2fb8e51e7f33631f5f4735fefd8d", size = 4764372, upload-time = "2025-09-22T04:03:23.27Z" },
+    { url = "https://files.pythonhosted.org/packages/05/33/f310b987c8bf9e61c4dd8e8035c416bd3230098f5e3cfa69fc4232de7059/lxml-6.0.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:875c6b5ab39ad5291588aed6925fac99d0097af0dd62f33c7b43736043d4a2ec", size = 5634653, upload-time = "2025-09-22T04:03:25.767Z" },
+    { url = "https://files.pythonhosted.org/packages/70/ff/51c80e75e0bc9382158133bdcf4e339b5886c6ee2418b5199b3f1a61ed6d/lxml-6.0.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:cdcbed9ad19da81c480dfd6dd161886db6096083c9938ead313d94b30aadf272", size = 5233795, upload-time = "2025-09-22T04:03:27.62Z" },
+    { url = "https://files.pythonhosted.org/packages/56/4d/4856e897df0d588789dd844dbed9d91782c4ef0b327f96ce53c807e13128/lxml-6.0.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:80dadc234ebc532e09be1975ff538d154a7fa61ea5031c03d25178855544728f", size = 5257023, upload-time = "2025-09-22T04:03:30.056Z" },
+    { url = "https://files.pythonhosted.org/packages/0f/85/86766dfebfa87bea0ab78e9ff7a4b4b45225df4b4d3b8cc3c03c5cd68464/lxml-6.0.2-cp314-cp314t-win32.whl", hash = "sha256:da08e7bb297b04e893d91087df19638dc7a6bb858a954b0cc2b9f5053c922312", size = 3911420, upload-time = "2025-09-22T04:03:32.198Z" },
+    { url = "https://files.pythonhosted.org/packages/fe/1a/b248b355834c8e32614650b8008c69ffeb0ceb149c793961dd8c0b991bb3/lxml-6.0.2-cp314-cp314t-win_amd64.whl", hash = "sha256:252a22982dca42f6155125ac76d3432e548a7625d56f5a273ee78a5057216eca", size = 4406837, upload-time = "2025-09-22T04:03:34.027Z" },
+    { url = "https://files.pythonhosted.org/packages/92/aa/df863bcc39c5e0946263454aba394de8a9084dbaff8ad143846b0d844739/lxml-6.0.2-cp314-cp314t-win_arm64.whl", hash = "sha256:bb4c1847b303835d89d785a18801a883436cdfd5dc3d62947f9c49e24f0f5a2c", size = 3822205, upload-time = "2025-09-22T04:03:36.249Z" },
+]
+
+[package.optional-dependencies]
+html-clean = [
+    { name = "lxml-html-clean" },
+]
+
+[[package]]
+name = "lxml-html-clean"
+version = "0.4.3"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "lxml" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/d9/cb/c9c5bb2a9c47292e236a808dd233a03531f53b626f36259dcd32b49c76da/lxml_html_clean-0.4.3.tar.gz", hash = "sha256:c9df91925b00f836c807beab127aac82575110eacff54d0a75187914f1bd9d8c", size = 21498, upload-time = "2025-10-02T20:49:24.895Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/10/4a/63a9540e3ca73709f4200564a737d63a4c8c9c4dd032bab8535f507c190a/lxml_html_clean-0.4.3-py3-none-any.whl", hash = "sha256:63fd7b0b9c3a2e4176611c2ca5d61c4c07ffca2de76c14059a81a2825833731e", size = 14177, upload-time = "2025-10-02T20:49:23.749Z" },
+]
+
 [[package]]
 name = "markdown-it-py"
 version = "4.0.0"
@@ -3699,6 +3864,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/af/df/c7891ef9d2712ad774777271d39fdef63941ffba0a9d59b7ad1fd2765e57/tiktoken-0.12.0-cp314-cp314t-win_amd64.whl", hash = "sha256:f61c0aea5565ac82e2ec50a05e02a6c44734e91b51c10510b084ea1b8e633a71", size = 920667, upload-time = "2025-10-06T20:22:34.444Z" },
 ]
 
+[[package]]
+name = "tld"
+version = "0.13.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/df/a1/5723b07a70c1841a80afc9ac572fdf53488306848d844cd70519391b0d26/tld-0.13.1.tar.gz", hash = "sha256:75ec00936cbcf564f67361c41713363440b6c4ef0f0c1592b5b0fbe72c17a350", size = 462000, upload-time = "2025-05-21T22:18:29.341Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/dc/70/b2f38360c3fc4bc9b5e8ef429e1fde63749144ac583c2dbdf7e21e27a9ad/tld-0.13.1-py2.py3-none-any.whl", hash = "sha256:a2d35109433ac83486ddf87e3c4539ab2c5c2478230e5d9c060a18af4b03aa7c", size = 274718, upload-time = "2025-05-21T22:18:25.811Z" },
+]
+
 [[package]]
 name = "tokenizers"
 version = "0.22.1"
@@ -3788,6 +3962,24 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2", size = 78540, upload-time = "2024-11-24T20:12:19.698Z" },
 ]
 
+[[package]]
+name = "trafilatura"
+version = "2.0.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "certifi" },
+    { name = "charset-normalizer" },
+    { name = "courlan" },
+    { name = "htmldate" },
+    { name = "justext" },
+    { name = "lxml" },
+    { name = "urllib3" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/06/25/e3ebeefdebfdfae8c4a4396f5a6ea51fc6fa0831d63ce338e5090a8003dc/trafilatura-2.0.0.tar.gz", hash = "sha256:ceb7094a6ecc97e72fea73c7dba36714c5c5b577b6470e4520dca893706d6247", size = 253404, upload-time = "2024-12-03T15:23:24.16Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/8a/b6/097367f180b6383a3581ca1b86fcae284e52075fa941d1232df35293363c/trafilatura-2.0.0-py3-none-any.whl", hash = "sha256:77eb5d1e993747f6f20938e1de2d840020719735690c840b9a1024803a4cd51d", size = 132557, upload-time = "2024-12-03T15:23:21.41Z" },
+]
+
 [[package]]
 name = "transformers"
 version = "2.3.0"
@@ -3861,6 +4053,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8", size = 347839, upload-time = "2025-03-23T13:54:41.845Z" },
 ]
 
+[[package]]
+name = "tzlocal"
+version = "5.3.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "tzdata", marker = "sys_platform == 'win32'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/8b/2e/c14812d3d4d9cd1773c6be938f89e5735a1f11a9f184ac3639b93cef35d5/tzlocal-5.3.1.tar.gz", hash = "sha256:cceffc7edecefea1f595541dbd6e990cb1ea3d19bf01b2809f362a03dd7921fd", size = 30761, upload-time = "2025-03-05T21:17:41.549Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/c2/14/e2a54fabd4f08cd7af1c07030603c3356b74da07f7cc056e600436edfa17/tzlocal-5.3.1-py3-none-any.whl", hash = "sha256:eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d", size = 18026, upload-time = "2025-03-05T21:17:39.857Z" },
+]
+
 [[package]]
 name = "uc-micro-py"
 version = "1.0.3"