fix: implement rate-limiting and retry logic for Mercado Público APIs

backend/app/services/mercado_publico.py

@@ -1,9 +1,13 @@
+import asyncio
 import httpx
 from typing import List, Optional, Dict, Any
 from app.config import settings
 from app.schemas.tender import Tender, TenderItem
 from datetime import datetime
 
+# Global semaphore to avoid "peticiones simultáneas" error from MP API
+mp_api_semaphore = asyncio.Semaphore(1)
+
 API_BASE = "https://api.mercadopublico.cl/servicios/v1/publico/licitaciones.json"
 
 # Constants from documentation
@@ -73,7 +77,6 @@ TIME_UNITS = {
 
 def map_raw_to_tender(item: Dict[str, Any]) -> Tender:
     """Maps raw API item to Tender schema."""
-    # Handle Items
     items_list = []
     raw_items = item.get("Items", {})
     if isinstance(raw_items, dict) and "Listado" in raw_items:
@@ -88,7 +91,6 @@ def map_raw_to_tender(item: Dict[str, Any]) -> Tender:
                 unit=i.get("UnidadMedida", "")
             ))
 
-    # Handle dates
     fechas = item.get("Fechas", {})
     closing_date = fechas.get("FechaCierre") or item.get("FechaCierre")
     pub_date = fechas.get("FechaPublicacion")
@@ -110,32 +112,50 @@ def map_raw_to_tender(item: Dict[str, Any]) -> Tender:
         region=item.get("Comprador", {}).get("RegionUnidad", "Nacional"),
         sector="Public",
         items=items_list,
-        attachments=[], # API v1 doesn't seem to provide attachments directly in this list
+        attachments=[],
         raw_data=item
     )
 
-async def _fetch(params: Dict[str, str]) -> List[Tender]:
-    """Helper to perform the actual API request."""
+async def _fetch(params: Dict[str, str], retries: int = 3) -> List[Tender]:
+    """Helper to perform the actual API request with rate limit handling."""
     if not settings.mercado_publico_ticket:
         print("⚠️ No Mercado Público Ticket configured.")
         return []
 
     params["ticket"] = settings.mercado_publico_ticket
 
-    try:
-        async with httpx.AsyncClient(timeout=30.0) as client:
-            response = await client.get(API_BASE, params=params)
-            response.raise_for_status()
-            data = response.json()
-            
-            raw_list = data.get("Listado", [])
-            if raw_list is None: # Sometimes API returns null for Listado
-                return []
-                
-            return [map_raw_to_tender(item) for item in raw_list]
-    except Exception as e:
-        print(f"❌ API Error fetching with params {params}: {e}")
-        return []
+    async with mp_api_semaphore:
+        for attempt in range(retries):
+            try:
+                async with httpx.AsyncClient(timeout=45.0) as client:
+                    response = await client.get(API_BASE, params=params)
+                    
+                    if response.status_code == 500:
+                        print(f"⚠️ API 500 for {response.url} - Likely no data or MP glitch.")
+                        return []
+                        
+                    response.raise_for_status()
+                    data = response.json()
+                    
+                    # Check for "peticiones simultáneas" error in the payload
+                    if data.get("Mensaje") and "simultáneas" in data.get("Mensaje", ""):
+                        wait_time = (attempt + 1) * 2
+                        print(f"🔄 Concurrent request error. Retrying in {wait_time}s... (Attempt {attempt+1}/{retries})")
+                        await asyncio.sleep(wait_time)
+                        continue
+
+                    raw_list = data.get("Listado", [])
+                    if raw_list is None:
+                        return []
+                        
+                    return [map_raw_to_tender(item) for item in raw_list]
+            except Exception as e:
+                print(f"❌ API Error (Attempt {attempt+1}): {e}")
+                if attempt < retries - 1:
+                    await asyncio.sleep(1)
+                else:
+                    return []
+    return []
 
 async def get_active_tenders() -> List[Tender]:
     """Shows all tenders published on the day of the query."""
@@ -146,35 +166,24 @@ async def get_tenders_by_date(date: str) -> List[Tender]:
     return await _fetch({"fecha": date})
 
 async def get_tenders_by_status_and_date(status: str, date: Optional[str] = None) -> List[Tender]:
-    """
-    Fetches tenders by status and date.
-    Status can be: publicada, cerrada, desierta, adjudicada, revocada, suspendida, todos.
-    """
     params = {"estado": status}
     if date:
         params["fecha"] = date
     return await _fetch(params)
 
 async def get_tender_by_code(code: str) -> Optional[Tender]:
-    """Fetches detailed information for a specific tender code."""
     results = await _fetch({"codigo": code})
     return results[0] if results else None
 
 async def get_tenders_by_provider(provider_code: str, date: str) -> List[Tender]:
-    """Fetches tenders for a specific provider on a specific date."""
     return await _fetch({"CodigoProveedor": provider_code, "fecha": date})
 
 async def get_tenders_by_org(org_code: str, date: str) -> List[Tender]:
-    """Fetches tenders for a specific public organization on a specific date."""
     return await _fetch({"CodigoOrganismo": org_code, "fecha": date})
 
 async def fetch_tenders(keyword: Optional[str] = None, date: Optional[str] = None) -> List[Tender]:
-    """
-    Legacy/Wrapper function for general search.
-    """
     search_date = date if date else datetime.now().strftime("%d%m%Y")
     
-    # We use active tenders as default if no date is provided
     if not date:
         tenders = await get_active_tenders()
     else:

backend/app/services/mercado_publico_oc.py

@@ -1,9 +1,13 @@
+import asyncio
 import httpx
 from typing import List, Optional, Dict, Any
 from app.config import settings
 from app.schemas.oc import PurchaseOrder, OCItem
 from datetime import datetime
 
+# Global semaphore to avoid "peticiones simultáneas" error from MP API
+mp_api_semaphore = asyncio.Semaphore(1)
+
 API_BASE_OC = "https://api.mercadopublico.cl/servicios/v1/publico/ordenesdecompra.json"
 
 OC_STATUS_CODES = {
@@ -74,35 +78,46 @@ def map_raw_to_oc(item: Dict[str, Any]) -> PurchaseOrder:
         raw_data=item
     )
 
-async def _fetch_oc(params: Dict[str, str]) -> List[PurchaseOrder]:
+async def _fetch_oc(params: Dict[str, str], retries: int = 3) -> List[PurchaseOrder]:
     if not settings.mercado_publico_ticket:
         return []
     
     params["ticket"] = settings.mercado_publico_ticket
     
-    # Cleaning params: if status is "todos", it's better to omit it when fecha is present
     if params.get("estado") == "todos":
         del params["estado"]
 
-    try:
-        async with httpx.AsyncClient(timeout=45.0) as client:
-            response = await client.get(API_BASE_OC, params=params)
-            # Handle MP API quirks: sometimes it returns 500 when no data is found for a date
-            if response.status_code == 500:
-                print(f"⚠️ API returned 500 for {response.url} - Likely no data for this query.")
-                return []
-                
-            response.raise_for_status()
-            data = response.json()
-            
-            raw_list = data.get("Listado", [])
-            if not raw_list:
-                return []
-            
-            return [map_raw_to_oc(item) for item in raw_list]
-    except Exception as e:
-        print(f"❌ OC API Error: {e}")
-        return []
+    async with mp_api_semaphore:
+        for attempt in range(retries):
+            try:
+                async with httpx.AsyncClient(timeout=45.0) as client:
+                    response = await client.get(API_BASE_OC, params=params)
+                    
+                    if response.status_code == 500:
+                        print(f"⚠️ API 500 for {response.url} - Likely no data or MP glitch.")
+                        return []
+                        
+                    response.raise_for_status()
+                    data = response.json()
+                    
+                    if data.get("Mensaje") and "simultáneas" in data.get("Mensaje", ""):
+                        wait_time = (attempt + 1) * 2
+                        print(f"🔄 OC Concurrent request error. Retrying in {wait_time}s... (Attempt {attempt+1}/{retries})")
+                        await asyncio.sleep(wait_time)
+                        continue
+
+                    raw_list = data.get("Listado", [])
+                    if not raw_list:
+                        return []
+                    
+                    return [map_raw_to_oc(item) for item in raw_list]
+            except Exception as e:
+                print(f"❌ OC API Error (Attempt {attempt+1}): {e}")
+                if attempt < retries - 1:
+                    await asyncio.sleep(1)
+                else:
+                    return []
+    return []
 
 async def get_oc_by_code(code: str) -> Optional[PurchaseOrder]:
     results = await _fetch_oc({"codigo": code})
@@ -110,14 +125,10 @@ async def get_oc_by_code(code: str) -> Optional[PurchaseOrder]:
 
 async def get_ocs_by_date(date: str, status: str = "todos") -> List[PurchaseOrder]:
     params = {"estado": status}
-    
-    # Logic based on documentation examples:
-    # If it's today, the doc suggests using estado=todos without fecha
     today_str = datetime.now().strftime("%d%m%Y")
     if date == today_str and status == "todos":
         return await _fetch_oc({"estado": "todos"})
     
-    # Otherwise, use the date
     params["fecha"] = date
     return await _fetch_oc(params)