server/tests/test_scraper.py

#!/usr/bin/env python3
import asyncio
from app.services.scraper import WebScraper

async def test_scraper():
    # Test with a real website
    test_url = "https://example.com"  # Simple test site
    
    async with WebScraper() as scraper:
        print(f"Testing scraper with: {test_url}")
        results = await scraper.scrape_website(test_url)
        
        print(f"\nFound {len(results)} pages:")
        for result in results:
            print(f"\nURL: {result['url']}")
            print(f"Title: {result['title']}")
            print(f"Content: {result['content'][:200]}...")

if __name__ == "__main__":
    asyncio.run(test_scraper())