feat: Live Compra Ágil web scraper integration for real-time discovery

backend/app/routers/tenders.py

@@ -59,3 +59,8 @@ def get_tenders_count(db: Session = Depends(get_db)):
 @router.post("/tenders/sync")
 async def manual_sync(keyword: Optional[str] = None, db: Session = Depends(get_db)):
     return await sync_tenders_to_db(db, keyword=keyword)
+
+@router.get("/tenders/scrape", response_model=List[Tender])
+async def live_scrape(keyword: str):
+    from app.services.scraper import scrape_compra_agil
+    return await scrape_compra_agil(keyword)

backend/app/services/scraper.py

@@ -0,0 +1,76 @@
+import httpx
+from bs4 import BeautifulSoup
+from typing import List
+from app.schemas.tender import Tender
+from datetime import datetime
+import re
+
+async def scrape_compra_agil(keywords: str) -> List[Tender]:
+    """
+    Scrapes the Mercado Público Compra Ágil search results page.
+    """
+    # Base URL for Compra Ágil search
+    url = f"https://buscador.mercadopublico.cl/compra-agil?keywords={keywords}&status=2&order_by=recent"
+    
+    headers = {
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+    }
+
+    try:
+        async with httpx.AsyncClient(timeout=30.0, follow_redirects=True) as client:
+            print(f"[Scraper] Navigating to: {url}")
+            response = await client.get(url, headers=headers)
+            response.raise_for_status()
+            
+            soup = BeautifulSoup(response.text, 'html.parser')
+            
+            # Find tender cards (this is based on standard Mercado Público searcher structure)
+            # Note: The classes might change, but usually they are consistent
+            tenders = []
+            
+            # Find elements that look like tender containers
+            cards = soup.select('.card-tender, .item-busqueda, .search-result-item') 
+            
+            if not cards:
+                # Fallback: try to find any link with a COT26 pattern
+                all_text = soup.get_text()
+                codes = re.findall(r'[0-9]+-[0-9]+-COT26', all_text)
+                print(f"[Scraper] Found {len(codes)} codes via regex fallback.")
+                # We would need more info to build a full Tender object from regex
+            
+            for card in cards[:20]: # Limit for performance
+                try:
+                    name_elem = card.select_one('.title-tender, h3, .name')
+                    code_elem = card.select_one('.code-tender, .code, span:contains("COT26")')
+                    buyer_elem = card.select_one('.buyer-name, .organismo')
+                    
+                    if not name_elem or not code_elem:
+                        continue
+                        
+                    name = name_elem.get_text(strip=True)
+                    code = code_elem.get_text(strip=True)
+                    buyer = buyer_elem.get_text(strip=True) if buyer_elem else "Unknown"
+                    
+                    tenders.append(Tender(
+                        code=code,
+                        name=name,
+                        description=name,
+                        buyer=buyer,
+                        status="Publicada",
+                        closing_date=datetime.now().strftime("%Y-%m-%d"),
+                        estimated_amount=0,
+                        source="Mercado Público (Scraped)",
+                        region="Nacional",
+                        sector="Compra Ágil",
+                        items=[],
+                        attachments=[]
+                    ))
+                except Exception as e:
+                    print(f"Error parsing card: {e}")
+                    
+            print(f"[Scraper] Successfully scraped {len(tenders)} tenders.")
+            return tenders
+            
+    except Exception as e:
+        print(f"❌ Scraper error: {e}")
+        return []

frontend/app/page.tsx

@@ -114,9 +114,19 @@ export default function HomePage() {
     window.history.pushState({}, '', `?tab=tender_search&q=${encodeURIComponent(value)}`);
   };
 
-  const handleSearch = async (params: { keyword?: string; buyer?: string; provider_code?: string; date?: string; skip?: number; limit?: number }) => {
-    const results = await searchTenders(params);
-    setTenders(results);
+  const handleSearch = async (params: { keyword?: string; buyer?: string; provider_code?: string; date?: string; skip?: number; limit?: number; isAgile?: boolean }) => {
+    try {
+      let results: Tender[];
+      if (params.isAgile && params.keyword) {
+        const { scrapeTenders } = await import("../lib/api");
+        results = await scrapeTenders(params.keyword);
+      } else {
+        results = await searchTenders(params);
+      }
+      setTenders(results);
+    } catch (error) {
+      console.error("Search error:", error);
+    }
   };
 
   const handleProfileSave = async (profile: CompanyProfileType) => {

frontend/components/TenderSearch.tsx

@@ -7,7 +7,7 @@ import { Language, translations } from "../lib/translations";
 
 type Props = {
   tenders: Tender[];
-  onSearch: (params: { keyword?: string; buyer?: string; provider_code?: string; date?: string; skip?: number; limit?: number }) => void;
+  onSearch: (params: { keyword?: string; buyer?: string; provider_code?: string; date?: string; skip?: number; limit?: number; isAgile?: boolean }) => void;
   onAnalyze: (tender: Tender) => void;
   forceShowFollowed?: boolean;
   initialKeyword?: string;
@@ -83,7 +83,8 @@ export default function TenderSearch({ tenders, onSearch, onAnalyze, forceShowFo
         buyer: buyerCode, 
         date,
         skip: 0, 
-        limit: itemsPerPage 
+        limit: itemsPerPage,
+        isAgile: isAgileMode
       });
     } catch (error) {
       console.error(error);

frontend/lib/api.ts

@@ -114,3 +114,11 @@ export async function syncDatabase() {
   }
   return res.json();
 }
+
+export async function scrapeTenders(keyword: string): Promise<Tender[]> {
+  const res = await fetch(`${API_BASE}/api/tenders/scrape?keyword=${encodeURIComponent(keyword)}`);
+  if (!res.ok) {
+    throw new Error("Error scraping tenders");
+  }
+  return res.json();
+}