feat: Scraper 2.0 with Deep Text Scan and regex code detection

backend/app/services/scraper.py

@@ -4,73 +4,88 @@ from typing import List
 from app.schemas.tender import Tender
 from datetime import datetime
 import re
+import json
 
 async def scrape_compra_agil(keywords: str) -> List[Tender]:
     """
-    Scrapes the Mercado Público Compra Ágil search results page.
+    Enhanced scraper for Mercado Público Compra Ágil.
+    Uses greedy regex and multiple CSS strategies.
     """
-    # Base URL for Compra Ágil search
+    # Optimized URL for agile purchase search
     url = f"https://buscador.mercadopublico.cl/compra-agil?keywords={keywords}&status=2&order_by=recent"
     
     headers = {
-        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
+        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
+        "Accept-Language": "es-ES,es;q=0.9,en;q=0.8",
+        "Referer": "https://buscador.mercadopublico.cl/"
     }
 
     try:
-        async with httpx.AsyncClient(timeout=30.0, follow_redirects=True) as client:
-            print(f"[Scraper] Navigating to: {url}")
+        async with httpx.AsyncClient(timeout=45.0, follow_redirects=True) as client:
+            print(f"[Scraper] Launching deep scan on: {url}")
             response = await client.get(url, headers=headers)
             response.raise_for_status()
             
-            soup = BeautifulSoup(response.text, 'html.parser')
+            html = response.text
+            soup = BeautifulSoup(html, 'html.parser')
             
-            # Find tender cards (this is based on standard Mercado Público searcher structure)
-            # Note: The classes might change, but usually they are consistent
             tenders = []
+            seen_codes = set()
             
-            # Find elements that look like tender containers
-            cards = soup.select('.card-tender, .item-busqueda, .search-result-item') 
+            # Strategy 1: Find all links that look like tender details
+            # Pattern: ficha?code=XXXX-XXXX-XXXX
+            links = soup.find_all('a', href=re.compile(r'code='))
             
-            if not cards:
-                # Fallback: try to find any link with a COT26 pattern
-                all_text = soup.get_text()
-                codes = re.findall(r'[0-9]+-[0-9]+-COT26', all_text)
-                print(f"[Scraper] Found {len(codes)} codes via regex fallback.")
-                # We would need more info to build a full Tender object from regex
-            
-            for card in cards[:20]: # Limit for performance
-                try:
-                    name_elem = card.select_one('.title-tender, h3, .name')
-                    code_elem = card.select_one('.code-tender, .code, span:contains("COT26")')
-                    buyer_elem = card.select_one('.buyer-name, .organismo')
-                    
-                    if not name_elem or not code_elem:
-                        continue
+            for link in links:
+                href = link.get('href')
+                code_match = re.search(r'code=([0-9a-zA-Z-]+)', href)
+                if code_match:
+                    code = code_match.group(1)
+                    if code not in seen_codes:
+                        # Try to find the title nearby
+                        # Often the link text is the name, or it's in a nearby div
+                        name = link.get_text(strip=True) or "Licitación Compra Ágil"
+                        
+                        # Clean code
+                        code = code.strip()
                         
-                    name = name_elem.get_text(strip=True)
-                    code = code_elem.get_text(strip=True)
-                    buyer = buyer_elem.get_text(strip=True) if buyer_elem else "Unknown"
-                    
-                    tenders.append(Tender(
-                        code=code,
-                        name=name,
-                        description=name,
-                        buyer=buyer,
-                        status="Publicada",
-                        closing_date=datetime.now().strftime("%Y-%m-%d"),
-                        estimated_amount=0,
-                        source="Mercado Público (Scraped)",
-                        region="Nacional",
-                        sector="Compra Ágil",
-                        items=[],
-                        attachments=[]
-                    ))
-                except Exception as e:
-                    print(f"Error parsing card: {e}")
-                    
-            print(f"[Scraper] Successfully scraped {len(tenders)} tenders.")
-            return tenders
+                        tenders.append(Tender(
+                            code=code,
+                            name=name if len(name) > 5 else f"Compra Ágil {code}",
+                            description=name,
+                            buyer="Mercado Público",
+                            status="Publicada",
+                            closing_date=datetime.now().strftime("%Y-%m-%d"),
+                            estimated_amount=0,
+                            source="MP Web Live",
+                            region="Nacional",
+                            sector="Agile",
+                            items=[],
+                            attachments=[]
+                        ))
+                        seen_codes.add(code)
+
+            # Strategy 2: Search for code patterns in the whole text if Strategy 1 found nothing
+            if not tenders:
+                codes = re.findall(r'[0-9]+-[0-9]+-[a-zA-Z0-9]+', html)
+                for code in list(set(codes)):
+                    if 'COT26' in code.upper() or len(code) > 10:
+                        tenders.append(Tender(
+                            code=code,
+                            name=f"Oportunidad Detectada: {code}",
+                            description="Detectada vía escaneo de texto profundo.",
+                            buyer="Chile Compra",
+                            status="Activa",
+                            closing_date="TBD",
+                            estimated_amount=None,
+                            source="MP Text Scan",
+                            sector="Compra Ágil"
+                        ))
+
+            print(f"[Scraper] Scan finished. Found {len(tenders)} opportunities.")
+            return tenders[:30] # Limit to top 30
             
     except Exception as e:
-        print(f"❌ Scraper error: {e}")
+        print(f"❌ Scraper critical failure: {e}")
         return []