Restore original code with detailed logging

app.py

@@ -17,68 +17,116 @@ for i in range(1, 20):
 
 key_status = {}
 for idx, k in enumerate(OLLAMA_KEYS, 1):
-    key_status[k] = {"index": idx, "prefix": k[:8]+"...", "success": 0, "healthy": True}
+    key_status[k] = {"index": idx, "prefix": k[:8]+"...", "failures": 0, "success": 0, "last_error": None, "healthy": True}
 
 def log(msg):
     print(f"[{time.strftime('%H:%M:%S')}] {msg}")
 
 @app.get("/")
 def root():
-    healthy = [k for k, v in key_status.items() if v["healthy"]]
-    return {"status": "ok", "keys_loaded": len(OLLAMA_KEYS), "healthy": len(healthy),
-            "keys_status": {v["prefix"]: {"s": v["success"], "h": v["healthy"]} for v in key_status.values()}}
+    return {
+        "status": "ok",
+        "base_url": BASE_URL,
+        "master_key": MASTER_API_KEY[:8] + "...",
+        "keys_loaded": len(OLLAMA_KEYS),
+        "healthy_keys": sum(1 for v in key_status.values() if v["healthy"]),
+        "keys_status": {v["prefix"]: {"failures": v["failures"], "success": v["success"], "healthy": v["healthy"]} for v in key_status.values()}
+    }
 
-def get_best_keys(max_failures=2, limit=2):
-    """Get best healthy keys sorted by fewest failures"""
-    healthy = sorted([(k, v) for k, v in key_status.items() if v["healthy"]], key=lambda x: x[1]["success"])
-    return [k for k, v in healthy[:limit]]
+def get_healthy_keys(max_failures=2):
+    """Get keys that are healthy (low failures)"""
+    healthy = [k for k, v in key_status.items() if v["failures"] < max_failures and v["healthy"]]
+    if not healthy:
+        for v in key_status.values():
+            v["failures"] = 0
+            v["healthy"] = True
+        return OLLAMA_KEYS[:2]
+    return healthy[:2]
 
 @app.post("/v1/chat/completions")
 async def chat(req: Request):
     auth_key = req.headers.get("Authorization", "").replace("Bearer ", "")
     if auth_key != MASTER_API_KEY:
+        log(f"AUTH FAIL: received '{auth_key[:8]}...' expected '{MASTER_API_KEY[:8]}...'")
         return JSONResponse({"error": "Unauthorized"}, status_code=401)
     
     body = await req.json()
+    is_stream = body.get("stream", False)
     model = body.get("model", "?")
+    log(f"REQUEST: model='{model}'")
     
-    selected = get_best_keys(max_failures=2, limit=2)
-    if not selected:
-        for v in key_status.values(): v["healthy"] = True
-        selected = OLLAMA_KEYS[:2]
+    candidate_keys = get_healthy_keys(max_failures=2)
+    log(f"Using top {len(candidate_keys)} keys")
     
-    log(f"REQ: {model} -> {[key_status[k]['prefix'] for k in selected]}")
-    
-    for key in selected:
+    for attempt, key in enumerate(candidate_keys):
         ki = key_status[key]
+        log(f"TRY #{attempt+1}: using key#{ki['index']} ({ki['prefix']})")
+        
         try:
-            start = time.time()
+            start_time = time.time()
             async with httpx.AsyncClient(timeout=30.0) as client:
                 resp = await client.post(f"{BASE_URL}/v1/chat/completions", json=body, headers={"Authorization": f"Bearer {key}"})
+            elapsed = time.time() - start_time
             
             if resp.status_code == 200:
                 ki["success"] += 1
-                log(f"OK: key#{ki['index']} in {time.time()-start:.2f}s")
+                ki["failures"] = 0
+                ki["healthy"] = True
+                log(f"SUCCESS: key#{ki['index']} responded in {elapsed:.2f}s")
                 return Response(resp.content, status_code=200)
+            
             elif resp.status_code == 429:
+                ki["failures"] += 1
                 ki["healthy"] = False
+                log(f"RATE LIMIT: key#{ki['index']} - skip to next")
                 continue
-        except Exception as e:
+            
+            elif resp.status_code >= 500:
+                ki["failures"] += 1
+                ki["last_error"] = f"http {resp.status_code}"
+                log(f"SERVER ERROR: key#{ki['index']} got {resp.status_code}, trying next")
+                continue
+            
+            else:
+                ki["last_error"] = f"http {resp.status_code}"
+                log(f"ERROR: key#{ki['index']} got {resp.status_code}")
+                return Response(resp.content, status_code=resp.status_code)
+                
+        except httpx.TimeoutException:
+            ki["failures"] += 1
             ki["healthy"] = False
+            ki["last_error"] = "timeout after 30s"
+            log(f"TIMEOUT: key#{ki['index']} - already healthy=False, try next")
+            continue
+            
+        except Exception as e:
+            ki["failures"] += 1
+            ki["last_error"] = str(e)[:50]
+            log(f"EXCEPTION: key#{ki['index']} error: {e}")
             continue
     
-    return JSONResponse({"error": "all failed"}, status_code=500)
+    log(f"ALL KEYS FAILED for model='{model}'")
+    return JSONResponse({"error": "all keys failed after 2 attempts", "model": model,
+        "keys_status": {v["prefix"]: {"failures": v["failures"], "last_error": v["last_error"]} for v in key_status.values()}}, status_code=500)
 
 @app.get("/v1/models")
 def models(req: Request):
     auth_key = req.headers.get("Authorization", "").replace("Bearer ", "")
     if auth_key != MASTER_API_KEY:
         return JSONResponse({"error": "Unauthorized"}, status_code=401)
-    for key in get_best_keys(limit=2):
+    
+    log("REQUEST: GET /v1/models")
+    
+    for key in get_healthy_keys(max_failures=2):
+        ki = key_status[key]
         try:
             resp = httpx.get(f"{BASE_URL}/v1/models", headers={"Authorization": f"Bearer {key}"}, timeout=10)
             if resp.status_code == 200:
+                ki["success"] += 1
+                log(f"MODELS OK: key#{ki['index']}")
                 return Response(resp.content, status_code=200)
-        except:
-            pass
-    return JSONResponse({"error": "no keys"}, status_code=500)
+        except Exception as e:
+            ki["last_error"] = str(e)[:50]
+            log(f"MODELS FAIL: key#{ki['index']} - {e}")
+    
+    return JSONResponse({"error": "no keys available"}, status_code=500)