server/scripts/generate_ieee_results.py

import asyncio
import sys
import os
import time
from typing import List, Dict

# Add server directory to path
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))

from app.services.ai_engine import AIEngine
from app.core.database import SessionLocal
from app.models.website import Website

async def generate_table():
    print("🚀 Initializing AI Engine (Warm-up)...")
    engine = AIEngine()
    
    # Get a real website ID - mimicking the one in logs if possible, or first available
    db = SessionLocal()
    # Log used ID 29. Let's try to find it or fallback to first.
    website = db.query(Website).filter(Website.id == 29).first()
    if not website:
        website = db.query(Website).first()
    
    website_id = website.id if website else 1
    db.close()
    
    print(f"Using Website ID: {website_id}")
    
    # Initialize Model Registry (Critical for AIEngine)
    from app.core.model_registry import get_model_registry
    print("⚙️ Initializing Model Registry...")
    get_model_registry().initialize()
    
    # Test Queries (Mix of questions from logs and standard ones)
    queries = [
        "Why Choose IQRA University?",
        "Do iqra university provide Master’s Program (MBA)?",
        "How many degree programs does iqra university offers?",
        "How many campuses does iqra university have?",
        "How can i apply to iqra university?",
        "What are the admission requirements?", # General
        "Tell me about the computer science faculty.", # Specific
        "Is financial aid available?", # Important
        "Where is the main campus located?", # Location
        "Contact phone number?" # Contact
    ]
    
    results = []

    print("\n🧪 STARTING EXPERIMENTS")
    print("=" * 60)
    
    # Warmup query (to load models)
    try:
        print("Warmup...")
        await engine.generate_response("hello", [], website_id=website_id)
    except Exception as e:
        print(f"Warmup failed: {e}")

    for i, query in enumerate(queries):
        print(f"Processing {i+1}/{len(queries)}: '{query}'...")
        
        start_time = time.time()
        try:
            # Generate response
            response, confidence, needs_review = await engine.generate_response(
                query=query, 
                context=[], 
                website_id=website_id
            )
        except Exception as e:
            response = f"ERROR: {str(e)}"
            confidence = 0.0
            needs_review = True
        
        end_time = time.time()
        latency_ms = (end_time - start_time) * 1000
        
        # Truncate response for table
        short_response = response.replace('\n', ' ')[:100] + "..." if len(response) > 100 else response.replace('\n', ' ')
        
        results.append({
            "query": query,
            "response": short_response,
            "latency": f"{latency_ms:.2f} ms",
            "confidence": f"{confidence:.4f}",
            "review": "Yes" if needs_review else "No"
        })
        
        # Small delay to be nice to the system
        await asyncio.sleep(0.5)

    print("\n" + "=" * 60)
    print("RESULTS TABLE (Markdown)")
    print("=" * 60)
    
    # Print Markdown Table
    headers = ["Question Asked", "System Response (Truncated)", "Responsiveness (Latency)", "Confidence Score"]
    
    # Header
    print(f"| {headers[0]} | {headers[1]} | {headers[2]} | {headers[3]} |")
    print(f"|{'---'}|{'---'}|{'---'}|{'---'}|")
    
    # Rows
    for r in results:
        print(f"| {r['query']} | {r['response']} | {r['latency']} | {r['confidence']} |")
        
    print("\n" + "=" * 60)
    print("Done.")

if __name__ == "__main__":
    asyncio.run(generate_table())