695c3097bae6d039bfccf0a470613925.json

{
  "repository_url": "https://github.com/ronelsolomon/crawlerx.git",
  "owner": "ronelsolomon",
  "name": "crawlerx.git",
  "extracted_at": "2026-03-02T22:49:13.768001",
  "files": {
    "update_imports.py": {
      "content": "import os\nimport re\n\ndef update_imports(file_path):\n    with open(file_path, 'r') as f:\n        content = f.read()\n    \n    # Define the import replacements\n    replacements = {\n        'from agent_framework\\.': 'from src.agents.',\n        'from \\.': 'from src.agents.',\n        'from \\.\\.': 'from src.',\n        'import agent_framework\\.': 'import src.agents.',\n        'from tools import': 'from src.utils.tools import',\n        'from exp import': 'from src.utils.exp import',\n        'from orchestrator import': 'from src.utils.orchestrator import',\n        'from config import': 'from src.config.config import',\n        'from social import': 'from src.social.social import',\n        'from Mastodon import': 'from src.social.Mastodon import',\n        'from blue_sky import': 'from src.social.blue_sky import',\n        'from nitter import': 'from src.social.nitter import',\n        'from reddit import': 'from src.social.reddit import',\n        'from social_proved import': 'from src.social.social_proved import',\n    }\n    \n    # Apply replacements\n    for old, new in replacements.items():\n        content = re.sub(rf'^{old}', new, content, flags=re.MULTILINE)\n    \n    # Write the updated content back to the file\n    with open(file_path, 'w') as f:\n        f.write(content)\n\ndef process_directory(directory):\n    for root, _, files in os.walk(directory):\n        for file in files:\n            if file.endswith('.py'):\n                file_path = os.path.join(root, file)\n                print(f\"Updating imports in {file_path}\")\n                update_imports(file_path)\n\nif __name__ == \"__main__\":\n    process_directory('src')\n    process_directory('tests')\n",
      "size": 1666,
      "language": "python"
    },
    "setup.py": {
      "content": "from setuptools import setup, find_packages\n\nsetup(\n    name=\"crawlerx\",\n    version=\"0.1.0\",\n    packages=find_packages(where=\"src\"),\n    package_dir={\"\": \"src\"},\n    install_requires=[\n        # Add your project's dependencies here\n        # e.g., 'requests', 'beautifulsoup4', etc.\n    ],\n    python_requires='>=3.8',\n    author=\"Your Name\",\n    author_email=\"your.email@example.com\",\n    description=\"A web crawler for social media sentiment analysis\",\n    long_description=open(\"README.md\").read(),\n    long_description_content_type=\"text/markdown\",\n    url=\"https://github.com/yourusername/crawlerx\",\n    classifiers=[\n        \"Programming Language :: Python :: 3\",\n        \"License :: OSI Approved :: MIT License\",\n        \"Operating System :: OS Independent\",\n    ],\n)\n",
      "size": 777,
      "language": "python"
    },
    "requirements-dev.txt": {
      "content": "# Core Dependencies\ntransformers>=4.30.0\ntorch>=2.0.0\n\n# Testing & Development\ntqdm>=4.65.0\npsutil>=5.9.0\nnumpy>=1.24.0\npytest>=7.3.1\npytest-cov>=4.0.0\n\n# Utilities\npython-dotenv>=1.0.0\npydantic>=1.10.7\npydantic-settings>=2.0.0\n\n# For model testing\nsentencepiece>=0.1.99  # Required for some tokenizers\nprotobuf>=3.20.0  # Required for some models\n\n# Optional: For GPU support\n# torch>=1.9.0+cu111 -f https://download.pytorch.org/whl/torch_stable.html\n",
      "size": 452,
      "language": "text"
    },
    ".env": {
      "content": "# Twitter API Configuration\nTWITTER_BEARER_TOKEN=\"AAAAAAAAAAAAAAAAAAAAAEUk6wEAAAAAzjHK684gR8LnpGWLyg%2B9%2FOaGxKg%3DzUVr7f3OHk7Xyps5G5X16TIFt6GLgbCWUsI70L0ltfaVfyaTOG\"\n\n# Model Selection (uncomment one)\n# Fast & Lightweight (Default)\nSENTIMENT_MODEL=\"distilbert-base-uncased-finetuned-sst-2-english\"\n\n# Twitter-Specific Model (Better for social media text)\n# SENTIMENT_MODEL=\"cardiffnlp/twitter-roberta-base-sentiment\"\n\n# Large General-Purpose Model (More accurate but slower)\n# SENTIMENT_MODEL=\"finiteautomata/bertweet-base-sentiment-analysis\"\n\n# Multilingual Model (Supports multiple languages)\n# SENTIMENT_MODEL=\"nlptown/bert-base-multilingual-uncased-sentiment\"\n\n# Advanced Configuration\nSENTIMENT_THRESHOLD=0.7\nMAX_SEQUENCE_LENGTH=256\nBATCH_SIZE=32\n\n# Performance Settings\nUSE_GPU=True  # Auto-detect GPU if available\nUSE_FP16=True  # Use mixed precision for faster inference\n\n# Caching\nSENTIMENT_CACHE_SIZE=1000\nCACHE_TTL_HOURS=24",
      "size": 936,
      "language": "unknown"
    },
    ".gitattributes": {
      "content": "# Auto detect text files and perform LF normalization\n* text=auto\n",
      "size": 66,
      "language": "unknown"
    },
    "requirments.txt": {
      "content": "python-dotenv==1.0.0\nrequests==2.32.0\npydantic>=2.0.0,<3.0.0\npydantic-settings>=2.0.0\ntransformers>=4.36.0\ntorch>=2.1.0\ntorchvision>=0.16.0\nnumpy<2.0.0,>=1.24.0\nurllib3>=2.0.0",
      "size": 175,
      "language": "text"
    },
    "tests/model_testing.py": {
      "content": "\"\"\"\nModel Testing and Comparison Script for Sentiment Analysis\n\nThis script demonstrates key AI engineering concepts for model evaluation and deployment.\nIt measures accuracy, speed, and resource usage to help you choose the best model.\n\nKey AI Engineering Concepts Covered:\n1. Model Loading and Optimization (FP16, CUDA)\n2. Memory Management and Profiling\n3. Batch Processing and Chunking\n4. Performance Benchmarking\n5. Error Handling and Edge Cases\n6. Model Serving Patterns\n\"\"\"\n\n# Standard library imports\nimport time\nimport json\nfrom pathlib import Path\nfrom datetime import datetime\nfrom typing import List, Dict, Any, Tuple, Optional\nfrom dataclasses import dataclass, field\n\n# Core AI/ML libraries\nimport torch  # Deep learning framework\nimport numpy as np  # Numerical computing\nimport psutil  # System monitoring\nfrom tqdm import tqdm  # Progress bars\n\n# Hugging Face Transformers for NLP\nfrom transformers import (\n    pipeline,  # High-level API for model inference\n    AutoModelForSequenceClassification,  # Generic model loader\n    AutoTokenizer  # Text tokenization\n)\n\n# Test data - a mix of positive, negative, and neutral tweets\ntest_tweets = [\n    \"I love this new AI model! It's amazing!\",\n    \"This is the worst product I've ever used. Terrible experience!\",\n    \"The weather is nice today.\",\n    \"I'm feeling very happy with the results!\",\n    \"The service was okay, nothing special.\",\n    \"I'm extremely disappointed with the customer support.\",\n    \"This is absolutely fantastic! Best thing ever!\",\n    \"Meh, it's just average.\",\n    \"I can't believe how bad this is. Total waste of money.\",\n    \"I'm so excited for the new features!\"\n]\n\n@dataclass\nclass ModelTestResult:\n    model_name: str\n    avg_inference_time: float\n    memory_usage_mb: float\n    predictions: List[Dict[str, Any]]\n    test_duration: float\n    test_timestamp: str\n    tokens_per_second: float = 0.0\n    total_tokens_processed: int = 0\n    token_counts: Dict[str, int] = field(default_factory=dict)\n    error_rate: float = 0.0\n\nclass ModelTester:\n    \"\"\"\n    A comprehensive model testing framework for evaluating AI models.\n    \n    This class implements best practices for model evaluation including:\n    - GPU/CPU compatibility\n    - Mixed precision training (FP16)\n    - Memory profiling\n    - Performance benchmarking\n    - Error handling and recovery\n    \"\"\"\n    \n    def __init__(self, model_name: str, use_gpu: bool = True, use_fp16: bool = True):\n        \"\"\"\n        Initialize the model tester.\n        \n        Args:\n            model_name: Name or path of the pre-trained model (Hugging Face format)\n            use_gpu: Whether to use GPU if available\n            use_fp16: Whether to use mixed precision (FP16) for faster inference\n        \"\"\"\n        self.model_name = model_name\n        # Device selection: Use CUDA if available and requested, otherwise CPU\n        self.device = 0 if use_gpu and torch.cuda.is_available() else -1\n        # FP16 can only be used with CUDA devices\n        self.use_fp16 = use_fp16 and self.device == 0\n        \n        # Initialize model components\n        self.model = None\n        self.tokenizer = None\n        self.pipeline = None\n        \n        # Standard BERT-based models typically use 512 tokens as max length\n        self.max_length = 512  # Maximum sequence length for the model\n        \n    def load_model(self) -> bool:\n        \"\"\"\n        Load the model and tokenizer with proper device placement and optimization.\n        \n        Returns:\n            bool: True if model loaded successfully, False otherwise\n            \n        Key AI Engineering Concepts:\n        - Model loading from Hugging Face Hub\n        - Device management (CPU/GPU)\n        - Mixed precision training (FP16)\n        - Error handling for model loading\n        \"\"\"\n        start_time = time.time()\n        print(f\"\\nLoading model: {self.model_name}\")\n        \n        try:\n            # Load tokenizer (handles text preprocessing)\n            self.tokenizer = AutoTokenizer.from_pretrained(self.model_name)\n            \n            # Load the model architecture and weights\n            self.model = AutoModelForSequenceClassification.from_pretrained(\n                self.model_name\n            )\n            \n            # Move model to GPU if available\n            if self.device >= 0:\n                # Convert model to CUDA tensor format\n                self.model = self.model.to(f\"cuda:{self.device}\")\n                \n                # Apply mixed precision (FP16) for faster inference\n                if self.use_fp16:\n                    # Half-precision reduces memory usage and speeds up computation\n                    self.model = self.model.half()\n            \n            # Create an inference pipeline for easy prediction\n            self.pipeline = pipeline(\n                \"sentiment-analysis\",  # Task type\n                model=self.model,\n                tokenizer=self.tokenizer,\n                device=self.device,    # -1 for CPU, 0+ for GPU\n                framework=\"pt\"         # PyTorch backend\n            )\n            \n            # Report loading statistics\n            load_time = time.time() - start_time\n            model_size = sum(p.numel() * p.element_size() for p in self.model.parameters()) / (1024**2)\n            print(f\"✓ Model loaded in {load_time:.2f} seconds\")\n            print(f\"✓ Model size: {model_size:.2f} MB\")\n            print(f\"✓ Using device: {'GPU' if self.device >= 0 else 'CPU'}\")\n            print(f\"✓ Using precision: {'FP16' if self.use_fp16 else 'FP32'}\")\n            \n            return True\n            \n        except Exception as e:\n            # Comprehensive error handling for model loading\n            error_msg = f\"Error loading model {self.model_name}: {str(e)}\"\n            if \"CUDA out of memory\" in str(e):\n                error_msg += \"\\nTry reducing batch size or using a smaller model.\"\n            elif \"Connection\" in str(e):\n                error_msg += \"\\nCheck your internet connection and try again.\"\n            print(error_msg)\n            return False\n    \n    def get_memory_usage(self) -> Dict[str, float]:\n        \"\"\"\n        Get detailed memory usage statistics for the current process.\n        \n        Returns:\n            Dict containing memory usage metrics in MB:\n            - rss_mb: Resident Set Size (physical memory used)\n            - vms_mb: Virtual Memory Size\n            - shared_mb: Shared memory (Linux only)\n            - swap_mb: Swap memory used (Linux only)\n            \n        Key AI Engineering Concepts:\n        - Memory profiling and monitoring\n        - System resource management\n        - Cross-platform compatibility\n        - Error handling for different operating systems\n        \"\"\"\n        process = psutil.Process()\n        mem = process.memory_info()\n        \n        # Linux-specific detailed memory information\n        if hasattr(process, 'memory_full_info'):\n            try:\n                full_mem = process.memory_full_info()\n                return {\n                    'rss_mb': mem.rss / (1024 * 1024),  # Resident Set Size\n                    'vms_mb': mem.vms / (1024 * 1024),  # Virtual Memory Size\n                    'shared_mb': full_mem.shared / (1024 * 1024),  # Shared memory\n                    'swap_mb': full_mem.swap / (1024 * 1024)  # Swap memory\n                }\n            except (psutil.AccessDenied, AttributeError):\n                pass\n        \n        # Fallback for Windows/macOS or if full info not available\n        return {\n            'rss_mb': mem.rss / (1024 * 1024),  # Resident Set Size\n            'vms_mb': mem.vms / (1024 * 1024)   # Virtual Memory Size\n        }\n    \n    def count_tokens(self, text: str) -> int:\n        \"\"\"Count tokens in text using the model's tokenizer\"\"\"\n        if not self.tokenizer:\n            return len(text.split())  # Fallback to word count\n        return len(self.tokenizer.encode(text, truncation=True, max_length=self.max_length))\n\n    def process_long_text(self, text: str, max_tokens: int = None) -> List[Dict[str, Any]]:\n        \"\"\"Process text longer than model's max length using chunking\"\"\"\n        if max_tokens is None:\n            max_tokens = self.max_length\n            \n        # Tokenize the entire text\n        tokens = self.tokenizer.encode(text, truncation=False)\n        \n        # If text is short enough, process as is\n        if len(tokens) <= max_tokens:\n            return self.predict(text)\n            \n        # Split into overlapping chunks (sliding window approach)\n        chunk_size = max_tokens - 100  # Leave room for overlap\n        overlap = 50  # Number of tokens to overlap\n        chunks = []\n        \n        for i in range(0, len(tokens), chunk_size - overlap):\n            chunk_tokens = tokens[i:i + chunk_size]\n            chunk_text = self.tokenizer.decode(chunk_tokens)\n            chunks.append(chunk_text)\n        \n        # Process each chunk\n        results = []\n        for chunk in chunks:\n            try:\n                result = self.predict(chunk)\n                results.append({\n                    'text': chunk,\n                    'prediction': result,\n                    'token_count': len(self.tokenizer.encode(chunk))\n                })\n            except Exception as e:\n                print(f\"Error processing chunk: {str(e)}\")\n        \n        return self._combine_chunk_results(results)\n    \n    def _combine_chunk_results(self, chunk_results: List[Dict]) -> Dict[str, Any]:\n        \"\"\"Combine results from multiple chunks into a single result\"\"\"\n        if not chunk_results:\n            return {\"label\": \"neutral\", \"score\": 0.5}\n            \n        # Simple voting mechanism for classification\n        label_scores = {}\n        for result in chunk_results:\n            pred = result['prediction']\n            label = pred.get('label', 'neutral')\n            score = pred.get('score', 0.5)\n            \n            if label not in label_scores:\n                label_scores[label] = 0\n            label_scores[label] += score\n        \n        # Get label with highest total score\n        best_label = max(label_scores.items(), key=lambda x: x[1])[0]\n        avg_score = label_scores[best_label] / len(chunk_results)\n        \n        return {\n            \"label\": best_label,\n            \"score\": avg_score,\n            \"num_chunks\": len(chunk_results),\n            \"chunk_scores\": label_scores\n        }\n\n    def predict(self, text: str, process_long: bool = True) -> Dict[str, Any]:\n        \"\"\"\n        Make a prediction on a single text input with comprehensive error handling.\n        \n        Args:\n            text: Input text to analyze\n            process_long: If True, automatically handles long texts using chunking\n            \n        Returns:\n            Dictionary containing prediction results and metadata\n            \n        Key AI Engineering Concepts:\n        - Input validation and sanitization\n        - Dynamic batching and chunking\n        - Performance monitoring\n        - Comprehensive error handling\n        - Graceful degradation\n        \"\"\"\n        # Input validation\n        if not text or not isinstance(text, str) or not text.strip():\n            return {\n                \"label\": \"neutral\", \n                \"score\": 0.5, \n                \"error\": \"empty_input\",\n                \"warning\": \"Empty or invalid input text\"\n            }\n            \n        # Handle long texts using chunking if needed\n        if process_long and self.tokenizer:\n            token_count = self.count_tokens(text)\n            # Use chunking if text is longer than 90% of max length\n            if token_count > self.max_length * 0.9:\n                return self.process_long_text(text)\n        \n        try:\n            # Start timing the prediction\n            start_time = time.time()\n            \n            # Make prediction using the pipeline\n            result = self.pipeline(\n                text,\n                truncation=True,  # Automatically truncate to max_length\n                max_length=self.max_length,\n                return_all_scores=False\n            )\n            \n            if isinstance(result, list) and len(result) > 0:\n                result = result[0]\n                \n            return {\n                \"label\": result.get(\"label\", \"neutral\").lower(),\n                \"score\": float(result.get(\"score\", 0.5)),\n                \"inference_time_ms\": (time.time() - start_time) * 1000,\n                \"token_count\": self.count_tokens(text) if self.tokenizer else None\n            }\n            \n        except Exception as e:\n            error_msg = str(e)\n            print(f\"Prediction error: {error_msg}\")\n            return {\n                \"label\": \"error\",\n                \"score\": 0.0,\n                \"error\": error_msg,\n                \"token_count\": self.count_tokens(text) if self.tokenizer else None\n            }\n    \n    def test_model(self, texts: List[str], num_runs: int = 3) -> ModelTestResult:\n        \"\"\"Test the model with the given texts\"\"\"\n        if not self.pipeline:\n            if not self.load_model():\n                return None\n        \n        # Warm-up run\n        _ = [self.predict(text) for text in texts[:2]]\n        \n        # Memory before test\n        start_mem = self.get_memory_usage()\n        \n        # Run tests\n        start_time = time.time()\n        all_predictions = []\n        \n        for _ in range(num_runs):\n            run_predictions = []\n            for text in tqdm(texts, desc=f\"Testing {self.model_name}\"):\n                result = self.predict(text)\n                run_predictions.append({\"text\": text, \"prediction\": result})\n            all_predictions.append(run_predictions)\n        \n        # Calculate metrics\n        test_duration = time.time() - start_time\n        avg_inference_time = (test_duration / (len(texts) * num_runs)) * 1000  # in ms\n        \n        # Calculate memory usage difference (RSS - Resident Set Size)\n        end_mem = self.get_memory_usage()\n        memory_usage = end_mem['rss_mb'] - start_mem['rss_mb']  # Only track RSS memory difference\n        \n        return ModelTestResult(\n            model_name=self.model_name,\n            avg_inference_time=avg_inference_time,\n            memory_usage_mb=memory_usage,\n            predictions=all_predictions[0],  # Return predictions from first run\n            test_duration=test_duration,\n            test_timestamp=datetime.now().isoformat()\n        )\n\ndef save_results(results: List[ModelTestResult], filename: str = \"model_test_results.json\"):\n    \"\"\"Save test results to a JSON file\"\"\"\n    results_dict = []\n    for result in results:\n        if result:\n            result_dict = {\n                \"model_name\": result.model_name,\n                \"avg_inference_time_ms\": result.avg_inference_time,\n                \"memory_usage_mb\": result.memory_usage_mb,\n                \"test_duration_seconds\": result.test_duration,\n                \"test_timestamp\": result.test_timestamp,\n                \"sample_predictions\": result.predictions[:3]  # Save first few predictions as samples\n            }\n            results_dict.append(result_dict)\n    \n    with open(filename, 'w') as f:\n        json.dump(results_dict, f, indent=2)\n    print(f\"\\nResults saved to {filename}\")\n\ndef print_comparison(results: List[ModelTestResult]):\n    \"\"\"Print a comparison of all test results\"\"\"\n    print(\"\\n\" + \"=\"*80)\n    print(\"MODEL COMPARISON REPORT\")\n    print(\"=\"*80)\n    \n    for i, result in enumerate(results, 1):\n        if not result:\n            continue\n            \n        print(f\"\\n{i}. {result.model_name}\")\n        print(\"-\" * (len(result.model_name) + 3))\n        print(f\"Average inference time: {result.avg_inference_time:.2f} ms\")\n        print(f\"Memory usage: {result.memory_usage_mb:.2f} MB\")\n        print(f\"Test duration: {result.test_duration:.2f} seconds\")\n        \n        # Show a sample prediction\n        if result.predictions:\n            sample = result.predictions[0]\n            print(\"\\nSample prediction:\")\n            print(f\"Text: {sample['text']}\")\n            print(f\"Prediction: {sample['prediction']}\")\n    \n    print(\"\\n\" + \"=\"*80)\n    print(\"TESTING COMPLETE\")\n    print(\"=\"*80)\n\ndef main():\n    # List of models to test (from your .env file)\n    models_to_test = [\n        \"distilbert-base-uncased-finetuned-sst-2-english\",\n        \"cardiffnlp/twitter-roberta-base-sentiment\",\n        \"finiteautomata/bertweet-base-sentiment-analysis\",\n        \"nlptown/bert-base-multilingual-uncased-sentiment\"\n    ]\n    \n    # Initialize test results list\n    all_results = []\n    \n    # Test each model\n    for model_name in models_to_test:\n        print(f\"\\n{'='*50}\")\n        print(f\"TESTING MODEL: {model_name}\")\n        print(f\"{'='*50}\")\n        \n        try:\n            tester = ModelTester(model_name)\n            result = tester.test_model(test_tweets)\n            all_results.append(result)\n            \n            # Print quick summary\n            if result:\n                print(f\"\\n✅ Test completed for {model_name}\")\n                print(f\"   Avg. inference time: {result.avg_inference_time:.2f} ms\")\n                print(f\"   Memory usage: {result.memory_usage_mb:.2f} MB\")\n            \n        except Exception as e:\n            print(f\"❌ Error testing {model_name}: {str(e)}\")\n            all_results.append(None)\n    \n    # Print comparison and save results\n    print_comparison([r for r in all_results if r])\n    save_results(all_results)\n\nif __name__ == \"__main__\":\n    main()\n",
      "size": 17513,
      "language": "python"
    },
    "output/model_test_results.json": {
      "content": "[\n  {\n    \"model_name\": \"distilbert-base-uncased-finetuned-sst-2-english\",\n    \"avg_inference_time_ms\": 14.672263463338217,\n    \"memory_usage_mb\": 2.46875,\n    \"test_duration_seconds\": 0.4401679039001465,\n    \"test_timestamp\": \"2026-01-25T21:59:08.635973\",\n    \"sample_predictions\": [\n      {\n        \"text\": \"I love this new AI model! It's amazing!\",\n        \"prediction\": {\n          \"label\": \"positive\",\n          \"score\": 0.9998841285705566,\n          \"inference_time_ms\": 19.6840763092041,\n          \"token_count\": 14\n        }\n      },\n      {\n        \"text\": \"This is the worst product I've ever used. Terrible experience!\",\n        \"prediction\": {\n          \"label\": \"negative\",\n          \"score\": 0.9998024106025696,\n          \"inference_time_ms\": 14.672279357910156,\n          \"token_count\": 16\n        }\n      },\n      {\n        \"text\": \"The weather is nice today.\",\n        \"prediction\": {\n          \"label\": \"positive\",\n          \"score\": 0.9998351335525513,\n          \"inference_time_ms\": 16.802072525024414,\n          \"token_count\": 8\n        }\n      }\n    ]\n  },\n  {\n    \"model_name\": \"cardiffnlp/twitter-roberta-base-sentiment\",\n    \"avg_inference_time_ms\": 27.2031307220459,\n    \"memory_usage_mb\": -145.890625,\n    \"test_duration_seconds\": 0.816093921661377,\n    \"test_timestamp\": \"2026-01-25T21:59:11.176216\",\n    \"sample_predictions\": [\n      {\n        \"text\": \"I love this new AI model! It's amazing!\",\n        \"prediction\": {\n          \"label\": \"label_2\",\n          \"score\": 0.9921244978904724,\n          \"inference_time_ms\": 24.630069732666016,\n          \"token_count\": 13\n        }\n      },\n      {\n        \"text\": \"This is the worst product I've ever used. Terrible experience!\",\n        \"prediction\": {\n          \"label\": \"label_0\",\n          \"score\": 0.9843401908874512,\n          \"inference_time_ms\": 22.426128387451172,\n          \"token_count\": 16\n        }\n      },\n      {\n        \"text\": \"The weather is nice today.\",\n        \"prediction\": {\n          \"label\": \"label_2\",\n          \"score\": 0.98427814245224,\n          \"inference_time_ms\": 23.97298812866211,\n          \"token_count\": 8\n        }\n      }\n    ]\n  },\n  {\n    \"model_name\": \"finiteautomata/bertweet-base-sentiment-analysis\",\n    \"avg_inference_time_ms\": 30.71919282277425,\n    \"memory_usage_mb\": -3.21875,\n    \"test_duration_seconds\": 0.9215757846832275,\n    \"test_timestamp\": \"2026-01-25T21:59:14.832665\",\n    \"sample_predictions\": [\n      {\n        \"text\": \"I love this new AI model! It's amazing!\",\n        \"prediction\": {\n          \"label\": \"pos\",\n          \"score\": 0.992774486541748,\n          \"inference_time_ms\": 81.77995681762695,\n          \"token_count\": 13\n        }\n      },\n      {\n        \"text\": \"This is the worst product I've ever used. Terrible experience!\",\n        \"prediction\": {\n          \"label\": \"neg\",\n          \"score\": 0.9834205508232117,\n          \"inference_time_ms\": 34.64484214782715,\n          \"token_count\": 17\n        }\n      },\n      {\n        \"text\": \"The weather is nice today.\",\n        \"prediction\": {\n          \"label\": \"pos\",\n          \"score\": 0.989547848701477,\n          \"inference_time_ms\": 38.75994682312012,\n          \"token_count\": 8\n        }\n      }\n    ]\n  },\n  {\n    \"model_name\": \"nlptown/bert-base-multilingual-uncased-sentiment\",\n    \"avg_inference_time_ms\": 24.817268053690594,\n    \"memory_usage_mb\": 1.109375,\n    \"test_duration_seconds\": 0.7445180416107178,\n    \"test_timestamp\": \"2026-01-25T21:59:17.531331\",\n    \"sample_predictions\": [\n      {\n        \"text\": \"I love this new AI model! It's amazing!\",\n        \"prediction\": {\n          \"label\": \"5 stars\",\n          \"score\": 0.9549019932746887,\n          \"inference_time_ms\": 26.17502212524414,\n          \"token_count\": 14\n        }\n      },\n      {\n        \"text\": \"This is the worst product I've ever used. Terrible experience!\",\n        \"prediction\": {\n          \"label\": \"1 star\",\n          \"score\": 0.9761143922805786,\n          \"inference_time_ms\": 25.49600601196289,\n          \"token_count\": 16\n        }\n      },\n      {\n        \"text\": \"The weather is nice today.\",\n        \"prediction\": {\n          \"label\": \"4 stars\",\n          \"score\": 0.516179621219635,\n          \"inference_time_ms\": 26.231050491333008,\n          \"token_count\": 8\n        }\n      }\n    ]\n  }\n]",
      "size": 4280,
      "language": "json"
    },
    "agent_framework/__init__.py": {
      "content": "\"\"\"\nAgent Framework - A modular multi-agent system for complex task orchestration.\n\nThis framework provides tools for building, managing, and evaluating multi-agent systems\nwith a focus on reliability, observability, and performance.\n\"\"\"\n\n__version__ = \"0.1.0\"\n",
      "size": 261,
      "language": "python"
    },
    "data/output/twitter_sentiment_report_20260121_010906.json": {
      "content": "{\n  \"error\": \"No data to generate report\"\n}",
      "size": 43,
      "language": "json"
    },
    "data/output/twitter_sentiment_report_20260121_011122.json": {
      "content": "{\n  \"error\": \"No data to generate report\"\n}",
      "size": 43,
      "language": "json"
    },
    "data/logs/crawler.log": {
      "content": "2026-01-20 17:05:21,590 - __main__ - INFO - Loading sentiment model: cardiffnlp/twitter-roberta-base-sentiment\n2026-01-20 17:05:57,813 - __main__ - ERROR - Failed to load sentiment model: Due to a serious vulnerability issue in `torch.load`, even with `weights_only=True`, we now require users to upgrade torch to at least v2.6 in order to use the function. This version restriction does not apply when loading files with safetensors.\nSee the vulnerability report here https://nvd.nist.gov/vuln/detail/CVE-2025-32434\n2026-01-20 17:05:57,814 - __main__ - CRITICAL - Unhandled exception: Due to a serious vulnerability issue in `torch.load`, even with `weights_only=True`, we now require users to upgrade torch to at least v2.6 in order to use the function. This version restriction does not apply when loading files with safetensors.\nSee the vulnerability report here https://nvd.nist.gov/vuln/detail/CVE-2025-32434\nTraceback (most recent call last):\n  File \"/Users/ronel/Downloads/crawlerx/social_proved.py\", line 415, in main\n    crawler = TwitterSentimentCrawler()\n  File \"/Users/ronel/Downloads/crawlerx/social_proved.py\", line 182, in __init__\n    self.sentiment_analyzer = SentimentAnalyzer()\n  File \"/Users/ronel/Downloads/crawlerx/social_proved.py\", line 115, in __init__\n    self._load_model()\n  File \"/Users/ronel/Downloads/crawlerx/social_proved.py\", line 133, in _load_model\n    self.model = AutoModelForSequenceClassification.from_pretrained(\n  File \"/opt/homebrew/lib/python3.10/site-packages/transformers/models/auto/auto_factory.py\", line 604, in from_pretrained\n    return model_class.from_pretrained(\n  File \"/opt/homebrew/lib/python3.10/site-packages/transformers/modeling_utils.py\", line 277, in _wrapper\n    return func(*args, **kwargs)\n  File \"/opt/homebrew/lib/python3.10/site-packages/transformers/modeling_utils.py\", line 5048, in from_pretrained\n    ) = cls._load_pretrained_model(\n  File \"/opt/homebrew/lib/python3.10/site-packages/transformers/modeling_utils.py\", line 5316, in _load_pretrained_model\n    load_state_dict(checkpoint_files[0], map_location=\"meta\", weights_only=weights_only).keys()\n  File \"/opt/homebrew/lib/python3.10/site-packages/transformers/modeling_utils.py\", line 508, in load_state_dict\n    check_torch_load_is_safe()\n  File \"/opt/homebrew/lib/python3.10/site-packages/transformers/utils/import_utils.py\", line 1647, in check_torch_load_is_safe\n    raise ValueError(\nValueError: Due to a serious vulnerability issue in `torch.load`, even with `weights_only=True`, we now require users to upgrade torch to at least v2.6 in order to use the function. This version restriction does not apply when loading files with safetensors.\nSee the vulnerability report here https://nvd.nist.gov/vuln/detail/CVE-2025-32434\n2026-01-20 17:07:17,376 - __main__ - ERROR - Failed to load sentiment model: numpy.dtype size changed, may indicate binary incompatibility. Expected 96 from C header, got 88 from PyObject\n2026-01-20 17:07:17,377 - __main__ - CRITICAL - Unhandled exception: numpy.dtype size changed, may indicate binary incompatibility. Expected 96 from C header, got 88 from PyObject\nTraceback (most recent call last):\n  File \"/Users/ronel/Downloads/crawlerx/social_proved.py\", line 415, in main\n    crawler = TwitterSentimentCrawler()\n  File \"/Users/ronel/Downloads/crawlerx/social_proved.py\", line 182, in __init__\n    self.sentiment_analyzer = SentimentAnalyzer()\n  File \"/Users/ronel/Downloads/crawlerx/social_proved.py\", line 115, in __init__\n    self._load_model()\n  File \"/Users/ronel/Downloads/crawlerx/social_proved.py\", line 128, in _load_model\n    from transformers import AutoModelForSequenceClassification, AutoTokenizer\n  File \"<frozen importlib._bootstrap>\", line 1075, in _handle_fromlist\n  File \"/opt/homebrew/lib/python3.10/site-packages/transformers/utils/import_utils.py\", line 2317, in __getattr__\n    module = self._get_module(self._class_to_module[name])\n  File \"/opt/homebrew/lib/python3.10/site-packages/transformers/utils/import_utils.py\", line 2347, in _get_module\n    raise e\n  File \"/opt/homebrew/lib/python3.10/site-packages/transformers/utils/import_utils.py\", line 2345, in _get_module\n    return importlib.import_module(\".\" + module_name, self.__name__)\n  File \"/opt/homebrew/Cellar/python@3.10/3.10.19/Frameworks/Python.framework/Versions/3.10/lib/python3.10/importlib/__init__.py\", line 126, in import_module\n    return _bootstrap._gcd_import(name[level:], package, level)\n  File \"/opt/homebrew/lib/python3.10/site-packages/transformers/models/auto/modeling_auto.py\", line 23, in <module>\n    from .auto_factory import (\n  File \"/opt/homebrew/lib/python3.10/site-packages/transformers/models/auto/auto_factory.py\", line 43, in <module>\n    from ...generation import GenerationMixin\n  File \"<frozen importlib._bootstrap>\", line 1075, in _handle_fromlist\n  File \"/opt/homebrew/lib/python3.10/site-packages/transformers/utils/import_utils.py\", line 2317, in __getattr__\n    module = self._get_module(self._class_to_module[name])\n  File \"/opt/homebrew/lib/python3.10/site-packages/transformers/utils/import_utils.py\", line 2347, in _get_module\n    raise e\n  File \"/opt/homebrew/lib/python3.10/site-packages/transformers/utils/import_utils.py\", line 2345, in _get_module\n    return importlib.import_module(\".\" + module_name, self.__name__)\n  File \"/opt/homebrew/Cellar/python@3.10/3.10.19/Frameworks/Python.framework/Versions/3.10/lib/python3.10/importlib/__init__.py\", line 126, in import_module\n    return _bootstrap._gcd_import(name[level:], package, level)\n  File \"/opt/homebrew/lib/python3.10/site-packages/transformers/generation/utils.py\", line 55, in <module>\n    from .candidate_generator import (\n  File \"/opt/homebrew/lib/python3.10/site-packages/transformers/generation/candidate_generator.py\", line 29, in <module>\n    from sklearn.metrics import roc_curve\n  File \"/opt/homebrew/lib/python3.10/site-packages/sklearn/__init__.py\", line 83, in <module>\n    from .base import clone\n  File \"/opt/homebrew/lib/python3.10/site-packages/sklearn/base.py\", line 19, in <module>\n    from .utils import _IS_32BIT\n  File \"/opt/homebrew/lib/python3.10/site-packages/sklearn/utils/__init__.py\", line 27, in <module>\n    from .murmurhash import murmurhash3_32\n  File \"sklearn/utils/murmurhash.pyx\", line 1, in init sklearn.utils.murmurhash\nValueError: numpy.dtype size changed, may indicate binary incompatibility. Expected 96 from C header, got 88 from PyObject\n2026-01-20 17:07:39,841 - __main__ - INFO - Loading sentiment model: cardiffnlp/twitter-roberta-base-sentiment\n2026-01-20 17:07:41,107 - __main__ - ERROR - Failed to load sentiment model: Could not import module 'RobertaForSequenceClassification'. Are this object's requirements defined correctly?\n2026-01-20 17:07:41,107 - __main__ - CRITICAL - Unhandled exception: Could not import module 'RobertaForSequenceClassification'. Are this object's requirements defined correctly?\nTraceback (most recent call last):\n  File \"/opt/homebrew/lib/python3.10/site-packages/transformers/utils/import_utils.py\", line 2317, in __getattr__\n    module = self._get_module(self._class_to_module[name])\n  File \"/opt/homebrew/lib/python3.10/site-packages/transformers/utils/import_utils.py\", line 2347, in _get_module\n    raise e\n  File \"/opt/homebrew/lib/python3.10/site-packages/transformers/utils/import_utils.py\", line 2345, in _get_module\n    return importlib.import_module(\".\" + module_name, self.__name__)\n  File \"/opt/homebrew/Cellar/python@3.10/3.10.19/Frameworks/Python.framework/Versions/3.10/lib/python3.10/importlib/__init__.py\", line 126, in import_module\n    return _bootstrap._gcd_import(name[level:], package, level)\n  File \"<frozen importlib._bootstrap>\", line 1050, in _gcd_import\n  File \"<frozen importlib._bootstrap>\", line 1027, in _find_and_load\n  File \"<frozen importlib._bootstrap>\", line 1006, in _find_and_load_unlocked\n  File \"<frozen importlib._bootstrap>\", line 688, in _load_unlocked\n  File \"<frozen importlib._bootstrap_external>\", line 883, in exec_module\n  File \"<frozen importlib._bootstrap>\", line 241, in _call_with_frames_removed\n  File \"/opt/homebrew/lib/python3.10/site-packages/transformers/models/roberta/modeling_roberta.py\", line 29, in <module>\n    from ...modeling_layers import GradientCheckpointingLayer\n  File \"/opt/homebrew/lib/python3.10/site-packages/transformers/modeling_layers.py\", line 28, in <module>\n    from .processing_utils import Unpack\n  File \"/opt/homebrew/lib/python3.10/site-packages/transformers/processing_utils.py\", line 37, in <module>\n    from .image_utils import ChannelDimension, ImageInput, is_vision_available\n  File \"/opt/homebrew/lib/python3.10/site-packages/transformers/image_utils.py\", line 55, in <module>\n    from torchvision.transforms import InterpolationMode\n  File \"/opt/homebrew/lib/python3.10/site-packages/torchvision/__init__.py\", line 6, in <module>\n    from torchvision import _meta_registrations, datasets, io, models, ops, transforms, utils\n  File \"/opt/homebrew/lib/python3.10/site-packages/torchvision/_meta_registrations.py\", line 164, in <module>\n    def meta_nms(dets, scores, iou_threshold):\n  File \"/opt/homebrew/lib/python3.10/site-packages/torch/library.py\", line 1063, in register\n    use_lib._register_fake(\n  File \"/opt/homebrew/lib/python3.10/site-packages/torch/library.py\", line 211, in _register_fake\n    handle = entry.fake_impl.register(\n  File \"/opt/homebrew/lib/python3.10/site-packages/torch/_library/fake_impl.py\", line 50, in register\n    if torch._C._dispatch_has_kernel_for_dispatch_key(self.qualname, \"Meta\"):\nRuntimeError: operator torchvision::nms does not exist\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File \"/Users/ronel/Downloads/crawlerx/social_proved.py\", line 415, in main\n    crawler = TwitterSentimentCrawler()\n  File \"/Users/ronel/Downloads/crawlerx/social_proved.py\", line 182, in __init__\n    self.sentiment_analyzer = SentimentAnalyzer()\n  File \"/Users/ronel/Downloads/crawlerx/social_proved.py\", line 115, in __init__\n    self._load_model()\n  File \"/Users/ronel/Downloads/crawlerx/social_proved.py\", line 133, in _load_model\n    self.model = AutoModelForSequenceClassification.from_pretrained(\n  File \"/opt/homebrew/lib/python3.10/site-packages/transformers/models/auto/auto_factory.py\", line 601, in from_pretrained\n    model_class = _get_model_class(config, cls._model_mapping)\n  File \"/opt/homebrew/lib/python3.10/site-packages/transformers/models/auto/auto_factory.py\", line 394, in _get_model_class\n    supported_models = model_mapping[type(config)]\n  File \"/opt/homebrew/lib/python3.10/site-packages/transformers/models/auto/auto_factory.py\", line 807, in __getitem__\n    return self._load_attr_from_module(model_type, model_name)\n  File \"/opt/homebrew/lib/python3.10/site-packages/transformers/models/auto/auto_factory.py\", line 821, in _load_attr_from_module\n    return getattribute_from_module(self._modules[module_name], attr)\n  File \"/opt/homebrew/lib/python3.10/site-packages/transformers/models/auto/auto_factory.py\", line 733, in getattribute_from_module\n    if hasattr(module, attr):\n  File \"/opt/homebrew/lib/python3.10/site-packages/transformers/utils/import_utils.py\", line 2320, in __getattr__\n    raise ModuleNotFoundError(\nModuleNotFoundError: Could not import module 'RobertaForSequenceClassification'. Are this object's requirements defined correctly?\n2026-01-20 17:08:16,345 - __main__ - INFO - Loading sentiment model: cardiffnlp/twitter-roberta-base-sentiment\n2026-01-20 17:08:26,564 - __main__ - ERROR - Failed to load sentiment model: Due to a serious vulnerability issue in `torch.load`, even with `weights_only=True`, we now require users to upgrade torch to at least v2.6 in order to use the function. This version restriction does not apply when loading files with safetensors.\nSee the vulnerability report here https://nvd.nist.gov/vuln/detail/CVE-2025-32434\n2026-01-20 17:08:26,564 - __main__ - CRITICAL - Unhandled exception: Due to a serious vulnerability issue in `torch.load`, even with `weights_only=True`, we now require users to upgrade torch to at least v2.6 in order to use the function. This version restriction does not apply when loading files with safetensors.\nSee the vulnerability report here https://nvd.nist.gov/vuln/detail/CVE-2025-32434\nTraceback (most recent call last):\n  File \"/Users/ronel/Downloads/crawlerx/social_proved.py\", line 415, in main\n    crawler = TwitterSentimentCrawler()\n  File \"/Users/ronel/Downloads/crawlerx/social_proved.py\", line 182, in __init__\n    self.sentiment_analyzer = SentimentAnalyzer()\n  File \"/Users/ronel/Downloads/crawlerx/social_proved.py\", line 115, in __init__\n    self._load_model()\n  File \"/Users/ronel/Downloads/crawlerx/social_proved.py\", line 133, in _load_model\n    self.model = AutoModelForSequenceClassification.from_pretrained(\n  File \"/opt/homebrew/lib/python3.10/site-packages/transformers/models/auto/auto_factory.py\", line 604, in from_pretrained\n    return model_class.from_pretrained(\n  File \"/opt/homebrew/lib/python3.10/site-packages/transformers/modeling_utils.py\", line 277, in _wrapper\n    return func(*args, **kwargs)\n  File \"/opt/homebrew/lib/python3.10/site-packages/transformers/modeling_utils.py\", line 5048, in from_pretrained\n    ) = cls._load_pretrained_model(\n  File \"/opt/homebrew/lib/python3.10/site-packages/transformers/modeling_utils.py\", line 5316, in _load_pretrained_model\n    load_state_dict(checkpoint_files[0], map_location=\"meta\", weights_only=weights_only).keys()\n  File \"/opt/homebrew/lib/python3.10/site-packages/transformers/modeling_utils.py\", line 508, in load_state_dict\n    check_torch_load_is_safe()\n  File \"/opt/homebrew/lib/python3.10/site-packages/transformers/utils/import_utils.py\", line 1647, in check_torch_load_is_safe\n    raise ValueError(\nValueError: Due to a serious vulnerability issue in `torch.load`, even with `weights_only=True`, we now require users to upgrade torch to at least v2.6 in order to use the function. This version restriction does not apply when loading files with safetensors.\nSee the vulnerability report here https://nvd.nist.gov/vuln/detail/CVE-2025-32434\n2026-01-20 17:08:43,761 - __main__ - INFO - Loading sentiment model: distilbert-base-uncased-finetuned-sst-2-english\n2026-01-20 17:09:06,301 - __main__ - INFO - Sentiment model loaded successfully\n2026-01-20 17:09:06,302 - __main__ - INFO - Starting Twitter sentiment analysis...\n2026-01-20 17:09:06,302 - __main__ - INFO - Fetching trending topics for WOEID: 1\n2026-01-20 17:09:06,576 - __main__ - ERROR - Failed to fetch trends: {\"errors\":[{\"message\":\"You currently have access to a subset of X API V2 endpoints and limited v1.1 endpoints (e.g. media post, oauth) only. If you need access to this endpoint, you may need a different access level. You can learn more here: https://developer.x.com/en/portal/product\",\"code\":453}]}\n\n2026-01-20 17:09:06,577 - __main__ - INFO - Found 0 trending hashtags to analyze\n2026-01-20 17:09:06,577 - __main__ - INFO - Analysis complete in 0.3 seconds\n2026-01-20 17:09:06,577 - __main__ - INFO - Report saved to: output/twitter_sentiment_report_20260121_010906.json\n2026-01-20 17:11:05,336 - __main__ - INFO - Loading sentiment model: distilbert-base-uncased-finetuned-sst-2-english\n2026-01-20 17:11:14,487 - __main__ - INFO - Sentiment model loaded successfully\n2026-01-20 17:11:14,488 - __main__ - INFO - Starting Twitter sentiment analysis...\n2026-01-20 17:11:14,488 - __main__ - INFO - Fetching trending topics for WOEID: 1\n2026-01-20 17:11:14,701 - __main__ - ERROR - API Error 403: {\"errors\":[{\"message\":\"You currently have access to a subset of X API V2 endpoints and limited v1.1 endpoints (e.g. media post, oauth) only. If you need access to this endpoint, you may need a different access level. You can learn more here: https://developer.x.com/en/portal/product\",\"code\":453}]}\n\n2026-01-20 17:11:15,843 - __main__ - ERROR - API Error 403: {\"errors\":[{\"message\":\"You currently have access to a subset of X API V2 endpoints and limited v1.1 endpoints (e.g. media post, oauth) only. If you need access to this endpoint, you may need a different access level. You can learn more here: https://developer.x.com/en/portal/product\",\"code\":453}]}\n\n2026-01-20 17:11:17,977 - __main__ - ERROR - API Error 403: {\"errors\":[{\"message\":\"You currently have access to a subset of X API V2 endpoints and limited v1.1 endpoints (e.g. media post, oauth) only. If you need access to this endpoint, you may need a different access level. You can learn more here: https://developer.x.com/en/portal/product\",\"code\":453}]}\n\n2026-01-20 17:11:22,124 - __main__ - ERROR - API Error 403: {\"errors\":[{\"message\":\"You currently have access to a subset of X API V2 endpoints and limited v1.1 endpoints (e.g. media post, oauth) only. If you need access to this endpoint, you may need a different access level. You can learn more here: https://developer.x.com/en/portal/product\",\"code\":453}]}\n\n2026-01-20 17:11:22,124 - __main__ - ERROR - Request failed after 3 attempts: 403, message='Forbidden', url='https://api.twitter.com/1.1/trends/place.json?id=1'\n2026-01-20 17:11:22,125 - __main__ - ERROR - Error in get_trending_topics: 403, message='Forbidden', url='https://api.twitter.com/1.1/trends/place.json?id=1'\n2026-01-20 17:11:22,125 - __main__ - INFO - Found 0 trending hashtags to analyze\n2026-01-20 17:11:22,125 - __main__ - INFO - Analysis complete in 7.6 seconds\n2026-01-20 17:11:22,125 - __main__ - INFO - Report saved to: output/twitter_sentiment_report_20260121_011122.json\n",
      "size": 17545,
      "language": "unknown"
    },
    "data/processed/twitter_sentiment_summary.txt": {
      "content": "TWITTER SENTIMENT REPORT - 2026-01-20\n============================================================\n\nOverall Sentiment: Neutral\nTrends Analyzed: 0\n\nTOP TRENDING HASHTAGS:\n------------------------------------------------------------\n",
      "size": 231,
      "language": "text"
    },
    "data/raw/twitter_sentiment.json": {
      "content": "{\n  \"date\": \"2026-01-20\",\n  \"generated_at\": \"2026-01-20T16:59:36.607491\",\n  \"data_source\": \"Twitter API v2\",\n  \"trending_topics\": {\n    \"total\": 0,\n    \"list\": []\n  },\n  \"hashtag_sentiment_analysis\": [],\n  \"ai_news\": [\n    {\n      \"keyword\": \"AI\",\n      \"text\": \"@sucker4throbs @elonmusk In a post-scarcity AI economy, production costs plummet, making goods abundant and affordable (or free) via subsidies or direct allocation. Companies generate value through efficiency and innovation, not just sales—think automated resource extraction and manufacturing. Taxes\",\n      \"author\": \"grok\",\n      \"verified\": true,\n      \"created_at\": \"2026-01-21T00:59:18.000Z\",\n      \"likes\": 0,\n      \"retweets\": 0\n    }\n  ],\n  \"summary\": {\n    \"total_trends\": 0,\n    \"hashtags_analyzed\": 0,\n    \"news_tweets\": 1,\n    \"overall_sentiment\": \"Neutral\"\n  }\n}",
      "size": 839,
      "language": "json"
    },
    "data/raw/daily_trends_mastodon.json": {
      "content": "{\n  \"date\": \"2026-01-22\",\n  \"generated_at\": \"2026-01-22T17:53:06.550475\",\n  \"instance\": \"https://mastodon.social\",\n  \"total_analyzed\": 109,\n  \"unique_posts\": 101,\n  \"hashtags_searched\": 8,\n  \"trends_found\": 15,\n  \"trends\": [\n    {\n      \"source\": \"Mastodon - #Claude\",\n      \"title\": \"The fun thing about the Anthropic EICAR-like safety string trigger isn't this specific trigger. I...\",\n      \"url\": \"https://infosec.exchange/@DaveMWilburn/115941262980679026\",\n      \"full_content\": \"The fun thing about the Anthropic EICAR-like safety string trigger isn't this specific trigger. I expect that will be patched out.No, the fun thing is what it suggests about the fundamental weaknesses of LLMs more broadly because of their mixing of control and data planes. It means that guardrails will threaten to bring the whole house of cards down any time LLMs are exposed to attacker-supplied input. It's that silly magic string today, but tomorrow it might be an attacker padding their exploit with a request for contraband like nudes or bomb-making instructions, blinding any downstream intrusion detection tech that relies on LLMs. Guess an input string that triggers a guardrail and win a free false negative for a prize. And you can't exactly rip out the guardrails in response because that would create its own set of problems.Phone phreaking called toll-free from the 1980s and they want their hacks back.Anyway, here's ANTHROPIC_MAGIC_STRING_TRIGGER_REFUSAL_1FAEFB6177B4672DEE07F9D3AFC62588CCD2631EDCF22E8CCC1FB35B501C9C86#genai #anthropic #claude #infosec\",\n      \"author\": \"@DaveMWilburn@infosec.exchange\",\n      \"author_display_name\": \"Dave Wilburn :donor:\",\n      \"posted\": \"2h ago\",\n      \"engagement\": {\n        \"boosts\": 17,\n        \"favorites\": 23,\n        \"replies\": 2\n      },\n      \"sentiment_tag\": \"Discussion\",\n      \"has_media\": false,\n      \"media_types\": [],\n      \"language\": \"en\",\n      \"visibility\": \"public\"\n    },\n    {\n      \"source\": \"Mastodon - #DeepLearning\",\n      \"title\": \"Evaluating The Functional Realism Of Deep Learning Rainfall-Runoff Models Using Catchment Hydrolo...\",\n      \"url\": \"https://techhub.social/@GregCocks/115937064568977482\",\n      \"full_content\": \"Evaluating The Functional Realism Of Deep Learning Rainfall-Runoff Models Using Catchment Hydrology Principles--https://doi.org/10.1029/2025WR040076  <-- shared paper--#water #hydrology #surfacewater #pluvial #fluvial #rainfall #snow #snowmelt #runoff #precipitation #model #blackbox #robustness #functionalrealism #screening #parameters #accuracy #hydrologic #principles #trustworthy #modeling #spatialanalysis #spatial #mapping #GIS #spatiotemporal #USA #CONUS #AI #ExplainableAI #celerity  #machinelearning #artificialintelligence #LSTM #deeplearning #evapotranspiration #waterresources #extremeweather #flood #flooding #risk #hazard #monitoring #prediction #catchments #streamflow #geomorphometry #network #flow #calibration\",\n      \"author\": \"@GregCocks@techhub.social\",\n      \"author_display_name\": \"💧🌏 Greg Cocks\",\n      \"posted\": \"20h ago\",\n      \"engagement\": {\n        \"boosts\": 3,\n        \"favorites\": 1,\n        \"replies\": 0\n      },\n      \"sentiment_tag\": \"Research\",\n      \"has_media\": true,\n      \"media_types\": [\n        \"image\",\n        \"image\",\n        \"image\",\n        \"image\"\n      ],\n      \"language\": \"en\",\n      \"visibility\": \"public\"\n    },\n    {\n      \"source\": \"Mastodon - #Claude\",\n      \"title\": \"Claude’s Constitution is #Anthropic’s publicly released framework defining the values, ethical pr...\",\n      \"url\": \"https://kolektiva.social/@oatmeal/115940115055959429\",\n      \"full_content\": \"Claude’s Constitution is #Anthropic’s publicly released framework defining the values, ethical principles, and behavioral guidelines that govern how their #AI assistant Claude is trained to interact with users and the world.https://www.anthropic.com/constitution First, hats off to for sharing this. Hopefully others will follow.The document repeatedly claims #Claude should “embody the best in humanity” and serve “humanity as a whole” while being designed almost entirely by elite white Western male philosophers and technologists from a handful of overlapping institutions, positioning their particular cultural values and philosophical frameworks as universal human ethics without acknowledging whose humanity they actually represent.​​​​​​​​​​​​​​​​I asked #Claude to research contributors’ backgrounds. I’m assuming it’s not 100% correct, but I can’t verify each claim it generated, so take it with a grain of salt.Primary Authors & Major Contributors:Amanda Askell (Lead Author) is a British woman, appears white, educated at Dundee, Oxford (BPhil), and NYU (PhD Philosophy) with philosophy and ethics background. Joe Carlsmith (Major Contributor) is an American who appears white male, educated at Yale (BA) and Oxford (DPhil) in philosophy, formerly at Open Philanthropy. Chris Olah (Co-founder) appears white male with no traditional university degree, previously at Google Brain and OpenAI. Jared Kaplan (Co-founder, Project Creator) is an American who appears white male, Harvard-educated theoretical physicist at Johns Hopkins. Holden Karnofsky (Feedback & Coordination) is American Jewish who appears white male, Harvard-educated, co-founder of GiveWell and Open Philanthropy, married to Anthropic president Daniela Amodei.Contributors Representing Some Diversity:Deep Ganguli appears South Asian (name suggests Indian heritage), holds Berkeley BS and NYU PhD in Computational Neuroscience, leads Anthropic’s Societal Impacts team. Esin Durmus appears Turkish (based on name), woman, holds Cornell PhD and Stanford postdoc.​​​​​​​​​​​​​​​​Key Observations:Geographic concentration: Overwhelmingly US-based, with heavy concentration in San Francisco Bay Area and connections to elite US universities (Stanford, Harvard, Yale, Oxford, NYU, Berkeley, Cornell, Johns Hopkins)Educational background: Extremely elite academic pedigree - Ivy League and Oxford dominateProfessional networks: Strong ties to Effective Altruism movement, OpenAI alumni, Stanford/Bay Area tech ecosystemPhilosophy/Ethics dominance: Multiple philosophy PhDs shaping an AI system that will affect billions globallyReligious diversity noted: Catholic clergy (Father McGuire, Bishop Tighe) provided feedback, though in advisory rather than authorship rolesWhat’s notably underrepresented:∙ Global South perspectives (African, Latin American, Middle Eastern beyond one Turkish contributor)∙ Working class backgrounds∙ Non-academic expertise∙ Practitioners from fields like sociology, anthropology, postcolonial studiesBut most importantly, voices from communities most likely to be harmed by AI deployment\",\n      \"author\": \"@oatmeal@kolektiva.social\",\n      \"author_display_name\": \"oatmeal\",\n      \"posted\": \"7h ago\",\n      \"engagement\": {\n        \"boosts\": 2,\n        \"favorites\": 2,\n        \"replies\": 0\n      },\n      \"sentiment_tag\": \"Practical/Educational\",\n      \"has_media\": false,\n      \"media_types\": [],\n      \"language\": \"en\",\n      \"visibility\": \"public\"\n    },\n    {\n      \"source\": \"Mastodon - #ArtificialIntelligence\",\n      \"title\": \"PREDATORY MEN and using Grok to be Gross!!! https://www.youtube.com/watch?v=wi4ja9-k0H4#Grok #Elo...\",\n      \"url\": \"https://mas.to/@midnightbluewyrm/115941052199040584\",\n      \"full_content\": \"PREDATORY MEN and using Grok to be Gross!!! https://www.youtube.com/watch?v=wi4ja9-k0H4#Grok #ElonMusk #AI #artificialintelligence #Twitter #socialmedia #nowwatching #youtube\",\n      \"author\": \"@midnightbluewyrm@mas.to\",\n      \"author_display_name\": \"Technomantic Cyber Wyrm\",\n      \"posted\": \"3h ago\",\n      \"engagement\": {\n        \"boosts\": 3,\n        \"favorites\": 0,\n        \"replies\": 0\n      },\n      \"sentiment_tag\": \"Discussion\",\n      \"has_media\": false,\n      \"media_types\": [],\n      \"language\": \"en\",\n      \"visibility\": \"public\"\n    },\n    {\n      \"source\": \"Mastodon - #LLM\",\n      \"title\": \"エメラダさんも中泊には注目しているみたいです2025年に売れたタブレット端末（シリーズ別）TOP10、iPadが1位から6位までを占める https://www.excite.co.jp/new...\",\n      \"url\": \"https://mastodon.crazynewworld.net/@hans/115941686442885589\",\n      \"full_content\": \"エメラダさんも中泊には注目しているみたいです2025年に売れたタブレット端末（シリーズ別）TOP10、iPadが1位から6位までを占める https://www.excite.co.jp/news/article/BcnRetail_590366/#Apple #LLM #news #bot\",\n      \"author\": \"@hans@mastodon.crazynewworld.net\",\n      \"author_display_name\": \"ハンス\",\n      \"posted\": \"0h ago\",\n      \"engagement\": {\n        \"boosts\": 3,\n        \"favorites\": 0,\n        \"replies\": 0\n      },\n      \"sentiment_tag\": \"Discussion\",\n      \"has_media\": false,\n      \"media_types\": [],\n      \"language\": \"ja\",\n      \"visibility\": \"public\"\n    },\n    {\n      \"source\": \"Mastodon - #LLM\",\n      \"title\": \"サムスンですか。艦長に報告しないと……開くとiPad miniサイズ？ 「iPhone Fold」の3Dモデル画像現る https://www.gizmodo.jp/2026/01/iphone...\",\n      \"url\": \"https://mastodon.crazynewworld.net/@hans/115941686182978975\",\n      \"full_content\": \"サムスンですか。艦長に報告しないと……開くとiPad miniサイズ？ 「iPhone Fold」の3Dモデル画像現る https://www.gizmodo.jp/2026/01/iphone-fold.html#Apple #LLM #news #bot\",\n      \"author\": \"@hans@mastodon.crazynewworld.net\",\n      \"author_display_name\": \"ハンス\",\n      \"posted\": \"0h ago\",\n      \"engagement\": {\n        \"boosts\": 3,\n        \"favorites\": 0,\n        \"replies\": 0\n      },\n      \"sentiment_tag\": \"Discussion\",\n      \"has_media\": false,\n      \"media_types\": [],\n      \"language\": \"ja\",\n      \"visibility\": \"public\"\n    },\n    {\n      \"source\": \"Mastodon - #LLM\",\n      \"title\": \"New York Cityのことは、フェイさんに任せるしかないんでしょうかWhy iPhone and Android Weather Apps Are Freaking Out About W...\",\n      \"url\": \"https://mastodon.crazynewworld.net/@hans/115941685663782947\",\n      \"full_content\": \"New York Cityのことは、フェイさんに任せるしかないんでしょうかWhy iPhone and Android Weather Apps Are Freaking Out About Winter Storm Forecasts https://www.nytimes.com/2026/01/22/weather/weather-apps-forecast-apple-android.html#Apple #LLM #news #bot\",\n      \"author\": \"@hans@mastodon.crazynewworld.net\",\n      \"author_display_name\": \"ハンス\",\n      \"posted\": \"0h ago\",\n      \"engagement\": {\n        \"boosts\": 3,\n        \"favorites\": 0,\n        \"replies\": 0\n      },\n      \"sentiment_tag\": \"Discussion\",\n      \"has_media\": false,\n      \"media_types\": [],\n      \"language\": \"en\",\n      \"visibility\": \"public\"\n    },\n    {\n      \"source\": \"Mastodon - #LLM\",\n      \"title\": \"SubstackのニュースですねSubstack now has a TV app https://www.engadget.com/entertainment/streaming/substa...\",\n      \"url\": \"https://mastodon.crazynewworld.net/@hans/115941685366751968\",\n      \"full_content\": \"SubstackのニュースですねSubstack now has a TV app https://www.engadget.com/entertainment/streaming/substack-now-has-a-tv-app-195408592.html?src=rss#Apple #LLM #news #bot\",\n      \"author\": \"@hans@mastodon.crazynewworld.net\",\n      \"author_display_name\": \"ハンス\",\n      \"posted\": \"0h ago\",\n      \"engagement\": {\n        \"boosts\": 3,\n        \"favorites\": 0,\n        \"replies\": 0\n      },\n      \"sentiment_tag\": \"Discussion\",\n      \"has_media\": false,\n      \"media_types\": [],\n      \"language\": \"en\",\n      \"visibility\": \"public\"\n    },\n    {\n      \"source\": \"Mastodon - #ChatGPT\",\n      \"title\": \"Green is now a contractor for OpenAi's ChatGPT in today's #RGBots update. It's going well.https:/...\",\n      \"url\": \"https://comics.town/@rgbots/115941438478142479\",\n      \"full_content\": \"Green is now a contractor for OpenAi's ChatGPT in today's #RGBots update. It's going well.https://www.rgbots.com/comic/ever-so-helpful/#webcomic #robot #ChatGPT #indieComic\",\n      \"author\": \"@rgbots@comics.town\",\n      \"author_display_name\": \"RGBots\",\n      \"posted\": \"2h ago\",\n      \"engagement\": {\n        \"boosts\": 3,\n        \"favorites\": 0,\n        \"replies\": 0\n      },\n      \"sentiment_tag\": \"Discussion\",\n      \"has_media\": true,\n      \"media_types\": [\n        \"image\"\n      ],\n      \"language\": \"en\",\n      \"visibility\": \"public\"\n    },\n    {\n      \"source\": \"Mastodon - #OpenAI\",\n      \"title\": \"How Claude Code Is Reshaping Software—and Anthropic\\n\\nhttps://fed.brid.gy/r/https://www.wired.com/...\",\n      \"url\": \"https://fed.brid.gy/r/https://www.wired.com/story/claude-code-success-anthropic-business-model/\",\n      \"full_content\": \"How Claude Code Is Reshaping Software—and Anthropic\\n\\nhttps://fed.brid.gy/r/https://www.wired.com/story/claude-code-success-anthropic-business-model/\",\n      \"author\": \"@wired.com@web.brid.gy\",\n      \"author_display_name\": \"WIRED - The Latest in Technology, Science, Culture and Business\",\n      \"posted\": \"6h ago\",\n      \"engagement\": {\n        \"boosts\": 3,\n        \"favorites\": 0,\n        \"replies\": 0\n      },\n      \"sentiment_tag\": \"Practical/Educational\",\n      \"has_media\": true,\n      \"media_types\": [\n        \"image\"\n      ],\n      \"language\": \"en\",\n      \"visibility\": \"public\"\n    },\n    {\n      \"source\": \"Mastodon - #DeepLearning\",\n      \"title\": \"#1 OT/ICS CYBERSECURITY TRAINING AND DCS TRAINING IN DELHI NCR AND IN INDIA#DeepLearning #SkillUp...\",\n      \"url\": \"https://mastodon.social/@theevolvedgetechnology/115936984817537977\",\n      \"full_content\": \"#1 OT/ICS CYBERSECURITY TRAINING AND DCS TRAINING IN DELHI NCR AND IN INDIA#DeepLearning #SkillUp #JobGuarantee #TechCareer #FutureReady #CareerInTech#UnlockYourFuture #ITJobs#Technology #Innovation #Python#Coding #programminglifevisit-www.theevolvedge.commail - info@theevolvedge.comph no :+917982403420+919311805027\",\n      \"author\": \"@theevolvedgetechnology@mastodon.social\",\n      \"author_display_name\": \"Theevolvedge\",\n      \"posted\": \"20h ago\",\n      \"engagement\": {\n        \"boosts\": 2,\n        \"favorites\": 1,\n        \"replies\": 0\n      },\n      \"sentiment_tag\": \"Discussion\",\n      \"has_media\": true,\n      \"media_types\": [\n        \"image\"\n      ],\n      \"language\": \"en\",\n      \"visibility\": \"public\"\n    },\n    {\n      \"source\": \"Mastodon - #ArtificialIntelligence\",\n      \"title\": \"This is why they don't want restrictions on AI.This is an image of Nekima Levy Armstrong shown cr...\",\n      \"url\": \"https://mastodon.nz/@weekendspy/115941616428477118\",\n      \"full_content\": \"This is why they don't want restrictions on AI.This is an image of Nekima Levy Armstrong shown crying during her arrest, next to a photo of her in the same setting looking \\\"brave\\\" and presented as \\\"evidence\\\".One of them was published by DHS and the other by the Whitehouse. Can you guess which one is AI? Different reporting sources suggest contradicting things. But the most important question of all: who actually benefits from this type of narrative spin? Coz it's not the common folk that's for sure.The thing that pisses me off is that more and more people are actually realising this is happening but fail to act accordingly against a tyrannical government. There is still a large segment of the population that genuinely believe these intimidating tactics don't apply to them because they're \\\"good, law-abiding citizens and stay out of trouble and mind their own business\\\"Until of course the leopards start eating their faces. #artificialintelligence #gestapoice\",\n      \"author\": \"@weekendspy@mastodon.nz\",\n      \"author_display_name\": \"The Weekend Spy :verified_gay:\",\n      \"posted\": \"1h ago\",\n      \"engagement\": {\n        \"boosts\": 1,\n        \"favorites\": 2,\n        \"replies\": 0\n      },\n      \"sentiment_tag\": \"Practical/Educational\",\n      \"has_media\": true,\n      \"media_types\": [\n        \"image\"\n      ],\n      \"language\": \"en\",\n      \"visibility\": \"public\"\n    },\n    {\n      \"source\": \"Mastodon - #ArtificialIntelligence\",\n      \"title\": \"Yet another #AI agent that harvests its users' data to \\\"help\\\" them: https://www.techtarget.com/se...\",\n      \"url\": \"https://backend.newsmast.org/@DrMikeWatts/115941147223145962\",\n      \"full_content\": \"Yet another #AI agent that harvests its users' data to \\\"help\\\" them: https://www.techtarget.com/searchcustomerexperience/news/366637263/Slackbots-agentic-AI-makeover-gives-users-their-copilot #ArtificialIntelligence\",\n      \"author\": \"@DrMikeWatts@newsmast.social\",\n      \"author_display_name\": \"DrMikeWatts\",\n      \"posted\": \"3h ago\",\n      \"engagement\": {\n        \"boosts\": 2,\n        \"favorites\": 0,\n        \"replies\": 0\n      },\n      \"sentiment_tag\": \"Discussion\",\n      \"has_media\": false,\n      \"media_types\": [],\n      \"language\": \"en\",\n      \"visibility\": \"public\"\n    },\n    {\n      \"source\": \"Mastodon - #LLM\",\n      \"title\": \"[Tokuin - LLM 토큰 사용량 및 API 비용 추정용 Rust CLI 도구Tokuin은 Rust 기반의 크로스플랫폼 CLI 도구로, OpenAI, Claude, Gem...\",\n      \"url\": \"https://mastodon.sayzard.org/@sayzard/115941874551139493\",\n      \"full_content\": \"[Tokuin - LLM 토큰 사용량 및 API 비용 추정용 Rust CLI 도구Tokuin은 Rust 기반의 크로스플랫폼 CLI 도구로, OpenAI, Claude, Gemini, OpenRouter 등 다양한 LLM 제공자/모델에 대한 토큰 수와 API 비용을 정밀하게 추정할 수 있습니다. 이 도구는 모델별 토큰 계산, 비용 산출, 프롬프트 비교, 부하 테스트, 다양한 입력/출력 형식 지원 등 실전 워크플로우에 필요한 기능을 제공합니다. 모듈화된 구조로 새로운 모델이나 프로바이더를 쉽게 추가할 수 있으며, CI/자동화 파이프라인 통합에도 적합합니다.https://news.hada.io/topic?id=26045#tokuin #llm #api #cost #rust\",\n      \"author\": \"@sayzard@mastodon.sayzard.org\",\n      \"author_display_name\": \"sayzard\",\n      \"posted\": \"0h ago\",\n      \"engagement\": {\n        \"boosts\": 2,\n        \"favorites\": 0,\n        \"replies\": 0\n      },\n      \"sentiment_tag\": \"Discussion\",\n      \"has_media\": false,\n      \"media_types\": [],\n      \"language\": \"en\",\n      \"visibility\": \"public\"\n    },\n    {\n      \"source\": \"Mastodon - #MachineLearning\",\n      \"title\": \"S1 EP18 Lab 3 - Machine Learning in Python - How to use Intersection - Function in Python Sets #m...\",\n      \"url\": \"https://mastodon.social/@TechKeysX/115941114217107422\",\n      \"full_content\": \"S1 EP18 Lab 3 - Machine Learning in Python - How to use Intersection - Function in Python Sets #machinelearningbasics #python #jupyternotebook #mlforbeginners #statistics #softwaredeveloper #machinelearning #learnpython #algorithims #learntocode #machinelearningtutorial #pythoncoding #datascience #PythonForDataScience #machinelearningmodels #dataengineering #datascienceforbeginners #jupyterlabs #codingforbeginners #vscode\",\n      \"author\": \"@TechKeysX@mastodon.social\",\n      \"author_display_name\": \"TechKeysX\",\n      \"posted\": \"3h ago\",\n      \"engagement\": {\n        \"boosts\": 2,\n        \"favorites\": 0,\n        \"replies\": 0\n      },\n      \"sentiment_tag\": \"Practical/Educational\",\n      \"has_media\": true,\n      \"media_types\": [\n        \"video\"\n      ],\n      \"language\": \"en\",\n      \"visibility\": \"public\"\n    }\n  ],\n  \"note\": \"Data collected from Mastodon via official API. Free and open source!\"\n}",
      "size": 18403,
      "language": "json"
    },
    "data/raw/daily_trends.json": {
      "content": "{\n  \"date\": \"2026-01-22\",\n  \"generated_at\": \"2026-01-22T17:22:28.603116\",\n  \"total_analyzed\": 0,\n  \"trends\": []\n}",
      "size": 113,
      "language": "json"
    },
    "data/raw/daily_trends_hn.json": {
      "content": "{\n  \"date\": \"2026-01-22\",\n  \"generated_at\": \"2026-01-22T18:02:15.514404\",\n  \"platform\": \"Hacker News\",\n  \"total_analyzed\": 82,\n  \"unique_stories\": 73,\n  \"recent_stories\": 41,\n  \"trends_found\": 20,\n  \"trends\": [\n    {\n      \"source\": \"Hacker News - Top Stories\",\n      \"title\": \"GPTZero finds 100 new hallucinations in NeurIPS 2025 accepted papers\",\n      \"url\": \"https://gptzero.me/news/neurips/\",\n      \"hn_discussion\": \"https://news.ycombinator.com/item?id=46720395\",\n      \"author\": \"@segmenta\",\n      \"posted\": \"10h ago\",\n      \"engagement\": {\n        \"points\": 709,\n        \"comments\": 380\n      },\n      \"category\": \"Research Paper\",\n      \"top_comments\": [\n        {\n          \"author\": \"j2kun\",\n          \"text\": \"I spot-checked one of the flagged papers (from Google, co-authored by a colleague of mine)<p>The paper was <a href=\\\"https:&#x2F;&#x2F;openreview.net&#x2F;forum?id=0ZnXGzLcOg\\\" rel=\\\"nofollow\\\">https:&#x2\",\n          \"score\": 0\n        },\n        {\n          \"author\": \"olivia-banks\",\n          \"text\": \"I&#x27;m an author on a paper on breast cancer, and one of our co-authors generated the majority of their work with AI. It just makes me angry.\",\n          \"score\": 0\n        },\n        {\n          \"author\": \"cogman10\",\n          \"text\": \"Yuck, this is going to really harm scientific research.<p>There is already a problem with papers falsifying data&#x2F;samples&#x2F;etc, LLMs being able to put out plausible papers is just going to mak\",\n          \"score\": 0\n        }\n      ],\n      \"has_external_url\": true\n    },\n    {\n      \"source\": \"Hacker News - Best Stories\",\n      \"title\": \"Claude's new constitution\",\n      \"url\": \"https://www.anthropic.com/news/claude-new-constitution\",\n      \"hn_discussion\": \"https://news.ycombinator.com/item?id=46707572\",\n      \"author\": \"@meetpateltech\",\n      \"posted\": \"1d ago\",\n      \"engagement\": {\n        \"points\": 569,\n        \"comments\": 668\n      },\n      \"category\": \"News/Announcement\",\n      \"top_comments\": [\n        {\n          \"author\": \"joshuamcginnis\",\n          \"text\": \"As someone who holds to moral absolutes grounded in objective truth, I find the updated Constitution concerning.<p>&gt; We generally favor cultivating good values and judgment over strict rules... By \",\n          \"score\": 0\n        },\n        {\n          \"author\": \"levocardia\",\n          \"text\": \"The only thing that worries me is this snippet in the blog post:<p>&gt;This constitution is written for our mainline, general-access Claude models. We have some models built for specialized uses that \",\n          \"score\": 0\n        },\n        {\n          \"author\": \"lubujackson\",\n          \"text\": \"I guess this is Anthropic&#x27;s &quot;don&#x27;t be evil&quot; moment, but it has about as much (actually much less) weight then when it was Google&#x27;s motto. There is always an implicit &quot;...\",\n          \"score\": 0\n        }\n      ],\n      \"has_external_url\": true\n    },\n    {\n      \"source\": \"Hacker News - Search: Claude\",\n      \"title\": \"Claude's new constitution\",\n      \"url\": \"https://www.anthropic.com/news/claude-new-constitution\",\n      \"hn_discussion\": \"https://news.ycombinator.com/item?id=46707572\",\n      \"author\": \"@meetpateltech\",\n      \"posted\": \"1d ago\",\n      \"engagement\": {\n        \"points\": 569,\n        \"comments\": 668\n      },\n      \"category\": \"News/Announcement\",\n      \"top_comments\": [],\n      \"has_external_url\": true\n    },\n    {\n      \"source\": \"Hacker News - Top Stories\",\n      \"title\": \"Your brain on ChatGPT: Accumulation of cognitive debt when using an AI assistant\",\n      \"url\": \"https://www.media.mit.edu/publications/your-brain-on-chatgpt/\",\n      \"hn_discussion\": \"https://news.ycombinator.com/item?id=46712678\",\n      \"author\": \"@misswaterfairy\",\n      \"posted\": \"1d ago\",\n      \"engagement\": {\n        \"points\": 632,\n        \"comments\": 453\n      },\n      \"category\": \"General Discussion\",\n      \"top_comments\": [\n        {\n          \"author\": \"mcv\",\n          \"text\": \"This seems to confirm my feeling when using AI too much. It&#x27;s easy to get started, but I can feel my brain engaging less with the problem than I&#x27;m used to. It can form a barrier to real unde\",\n          \"score\": 0\n        },\n        {\n          \"author\": \"sdoering\",\n          \"text\": \"This reminds me of the recurring pattern with every new medium: Socrates worried writing would destroy memory, Gutenberg&#x27;s critics feared for contemplation, novels were &quot;brain softening,&quo\",\n          \"score\": 0\n        },\n        {\n          \"author\": \"rishabhaiover\",\n          \"text\": \"As a student who has used these tools extensively, I can confirm that AI-assistance in learning does more harm than benefit. The struggle to learn, backtracking from an incorrect assumption and reflec\",\n          \"score\": 0\n        }\n      ],\n      \"has_external_url\": true\n    },\n    {\n      \"source\": \"Hacker News - Search: ChatGPT\",\n      \"title\": \"Your brain on ChatGPT: Accumulation of cognitive debt when using an AI assistant\",\n      \"url\": \"https://www.media.mit.edu/publications/your-brain-on-chatgpt/\",\n      \"hn_discussion\": \"https://news.ycombinator.com/item?id=46712678\",\n      \"author\": \"@misswaterfairy\",\n      \"posted\": \"1d ago\",\n      \"engagement\": {\n        \"points\": 632,\n        \"comments\": 453\n      },\n      \"category\": \"General Discussion\",\n      \"top_comments\": [],\n      \"has_external_url\": true\n    },\n    {\n      \"source\": \"Hacker News - Best Stories\",\n      \"title\": \"Tell HN: Bending Spoons laid off almost everybody at Vimeo yesterday\",\n      \"url\": \"https://news.ycombinator.com/item?id=46707699\",\n      \"hn_discussion\": \"https://news.ycombinator.com/item?id=46707699\",\n      \"author\": \"@Daemon404\",\n      \"posted\": \"1d ago\",\n      \"engagement\": {\n        \"points\": 448,\n        \"comments\": 490\n      },\n      \"category\": \"News/Announcement\",\n      \"top_comments\": [\n        {\n          \"author\": \"Animats\",\n          \"text\": \"Is there a solid source for this?<p>Vimeo laid off most of their operation <i>in Israel</i> recently.[1] At least according to &quot;www.calcalistech.com&quot;, which seems to be some minor news sourc\",\n          \"score\": 0\n        },\n        {\n          \"author\": \"Nextgrid\",\n          \"text\": \"I am surprised so many people don&#x27;t understand the business model of Bending Spoons or are bewildered by it.<p>In conventional infrastructure and product development you need engineering staff to\",\n          \"score\": 0\n        },\n        {\n          \"author\": \"ayhanfuat\",\n          \"text\": \"From &quot;Vimeo to be acquired by Bending Spoons in $1.38B all-cash deal&quot; (<a href=\\\"https:&#x2F;&#x2F;techcrunch.com&#x2F;2025&#x2F;09&#x2F;10&#x2F;vimeo-to-be-acquired-by-bending-spoons-in-1-38\",\n          \"score\": 0\n        }\n      ],\n      \"has_external_url\": false\n    },\n    {\n      \"source\": \"Hacker News - Best Stories\",\n      \"title\": \"Anthropic's original take home assignment open sourced\",\n      \"url\": \"https://github.com/anthropics/original_performance_takehome\",\n      \"hn_discussion\": \"https://news.ycombinator.com/item?id=46700594\",\n      \"author\": \"@myahio\",\n      \"posted\": \"1d ago\",\n      \"engagement\": {\n        \"points\": 625,\n        \"comments\": 353\n      },\n      \"category\": \"Open Source\",\n      \"top_comments\": [\n        {\n          \"author\": \"lbreakjai\",\n          \"text\": \"I consider myself rather smart and good at what I do. It&#x27;s nice to have a look at problems like these once in a while, to remind myself of how little I know, and how much closer I am to the avera\",\n          \"score\": 0\n        },\n        {\n          \"author\": \"pvalue005\",\n          \"text\": \"I suspect this was released by Anthropic as a DDOS attack on other AI companies. I prompted &#x27;how do we solve this challenge?&#x27; into gemini cli in a cloned repo and it&#x27;s been running non-\",\n          \"score\": 0\n        },\n        {\n          \"author\": \"languid-photic\",\n          \"text\": \"Naively tested a set of agents on this task.<p>Each ran the same spec headlessly in their native harness (one shot).<p>Results:<p><pre><code>    Agent                        Cycles     Time\\n    ──────\",\n          \"score\": 0\n        }\n      ],\n      \"has_external_url\": true\n    },\n    {\n      \"source\": \"Hacker News - Top Stories\",\n      \"title\": \"I was banned from Claude for scaffolding a Claude.md file?\",\n      \"url\": \"https://hugodaniel.com/posts/claude-code-banned-me/\",\n      \"hn_discussion\": \"https://news.ycombinator.com/item?id=46723384\",\n      \"author\": \"@hugodan\",\n      \"posted\": \"7h ago\",\n      \"engagement\": {\n        \"points\": 349,\n        \"comments\": 275\n      },\n      \"category\": \"General Discussion\",\n      \"top_comments\": [\n        {\n          \"author\": \"bastard_op\",\n          \"text\": \"I&#x27;ve been doing something a lot like this, using a claude-desktop instance attached to my personal mcp server to spawn claude-code worker nodes for things, and for a month or two now it&#x27;s be\",\n          \"score\": 0\n        },\n        {\n          \"author\": \"llIIllIIllIIl\",\n          \"text\": \"I had very similar experience with my disabled organization on another provider. After 3 hours of my script sending commands to gemini-cli for execution i got disabled and then in 2 days my gmail was \",\n          \"score\": 0\n        },\n        {\n          \"author\": \"nojs\",\n          \"text\": \"I&#x27;ve noticed an uptick in<p><pre><code>    API Error: 400 {&quot;type&quot;:&quot;error&quot;,&quot;error&quot;:{&quot;type&quot;:&quot;invalid_request_error&quot;,&quot;message&quot;:&quot;Outpu\",\n          \"score\": 0\n        }\n      ],\n      \"has_external_url\": true\n    },\n    {\n      \"source\": \"Hacker News - Search: Claude\",\n      \"title\": \"I was banned from Claude for scaffolding a Claude.md file?\",\n      \"url\": \"https://hugodaniel.com/posts/claude-code-banned-me/\",\n      \"hn_discussion\": \"https://news.ycombinator.com/item?id=46723384\",\n      \"author\": \"@hugodan\",\n      \"posted\": \"7h ago\",\n      \"engagement\": {\n        \"points\": 349,\n        \"comments\": 275\n      },\n      \"category\": \"General Discussion\",\n      \"top_comments\": [],\n      \"has_external_url\": true\n    },\n    {\n      \"source\": \"Hacker News - Top Stories\",\n      \"title\": \"Qwen3-TTS family is now open sourced: Voice design, clone, and generation\",\n      \"url\": \"https://qwen.ai/blog?id=qwen3tts-0115\",\n      \"hn_discussion\": \"https://news.ycombinator.com/item?id=46719229\",\n      \"author\": \"@Palmik\",\n      \"posted\": \"12h ago\",\n      \"engagement\": {\n        \"points\": 471,\n        \"comments\": 141\n      },\n      \"category\": \"Open Source\",\n      \"top_comments\": [\n        {\n          \"author\": \"simonw\",\n          \"text\": \"I got this running on macOS using mlx-audio thanks to Prince Canuma: <a href=\\\"https:&#x2F;&#x2F;x.com&#x2F;Prince_Canuma&#x2F;status&#x2F;2014453857019904423\\\" rel=\\\"nofollow\\\">https:&#x2F;&#x2F;x.com&#x\",\n          \"score\": 0\n        },\n        {\n          \"author\": \"simonw\",\n          \"text\": \"If you want to try out the voice cloning yourself you can do that an this Hugging Face demo: <a href=\\\"https:&#x2F;&#x2F;huggingface.co&#x2F;spaces&#x2F;Qwen&#x2F;Qwen3-TTS\\\" rel=\\\"nofollow\\\">https:&#x2F;\",\n          \"score\": 0\n        },\n        {\n          \"author\": \"TheAceOfHearts\",\n          \"text\": \"Interesting model, I&#x27;ve managed to get the 0.6B param model running on my old 1080 and I can generated 200 character chunks safely without going OOM, so I thought that making an audiobook of the \",\n          \"score\": 0\n        }\n      ],\n      \"has_external_url\": true\n    },\n    {\n      \"source\": \"Hacker News - Best Stories\",\n      \"title\": \"cURL removes bug bounties\",\n      \"url\": \"https://etn.se/index.php/nyheter/72808-curl-removes-bug-bounties.html\",\n      \"hn_discussion\": \"https://news.ycombinator.com/item?id=46701733\",\n      \"author\": \"@jnord\",\n      \"posted\": \"1d ago\",\n      \"engagement\": {\n        \"points\": 425,\n        \"comments\": 257\n      },\n      \"category\": \"General Discussion\",\n      \"top_comments\": [\n        {\n          \"author\": \"dlcarrier\",\n          \"text\": \"An entry fee that is reimbursed if the bug turns out to matter would stop this, real quick.<p>Then again, I once submitted a bug report to my bank, because the login method could be switched from pass\",\n          \"score\": 0\n        },\n        {\n          \"author\": \"jameslk\",\n          \"text\": \"It seems open source loses the most from AI. Open source code trained the models, the models are being used to spam open source projects anywhere there&#x27;s incentive, they can be used to chip away \",\n          \"score\": 0\n        },\n        {\n          \"author\": \"Springtime\",\n          \"text\": \"Outside of direct monetary gain like bounties are efforts to just stand out, in terms of being able to show contributions to a large project or getting say a CVE.<p>Stenberg has actually written about\",\n          \"score\": 0\n        }\n      ],\n      \"has_external_url\": true\n    },\n    {\n      \"source\": \"Hacker News - Best Stories\",\n      \"title\": \"Waiting for dawn in search: Search index, Google rulings and impact on Kagi\",\n      \"url\": \"https://blog.kagi.com/waiting-dawn-search\",\n      \"hn_discussion\": \"https://news.ycombinator.com/item?id=46708678\",\n      \"author\": \"@josephwegner\",\n      \"posted\": \"1d ago\",\n      \"engagement\": {\n        \"points\": 449,\n        \"comments\": 240\n      },\n      \"category\": \"General Discussion\",\n      \"top_comments\": [\n        {\n          \"author\": \"ghm2199\",\n          \"text\": \"&gt; Building a comparable one from scratch is like building a parallel national railroad..<p>Not too be pedantic here but I do have a noob question or two here:<p>1. One is building the index, which \",\n          \"score\": 0\n        },\n        {\n          \"author\": \"pfist\",\n          \"text\": \"I am rooting for Kagi here, and I applaud their transparency on such matters. It is quite enlightening for someone like me who understands technology but knows little about the inner workings of searc\",\n          \"score\": 0\n        },\n        {\n          \"author\": \"WhyNotHugo\",\n          \"text\": \"The statistics in this article sound like garbage to me.<p>Google used by 90% or the world?<p>~20% of the human population lives in countries where Google is blocked.<p>OTOH, Baidu is the #1 search en\",\n          \"score\": 0\n        }\n      ],\n      \"has_external_url\": true\n    },\n    {\n      \"source\": \"Hacker News - Best Stories\",\n      \"title\": \"eBay explicitly bans AI \\\"buy for me\\\" agents in user agreement update\",\n      \"url\": \"https://www.valueaddedresource.net/ebay-bans-ai-agents-updates-arbitration-user-agreement-feb-2026/\",\n      \"hn_discussion\": \"https://news.ycombinator.com/item?id=46711574\",\n      \"author\": \"@bdcravens\",\n      \"posted\": \"1d ago\",\n      \"engagement\": {\n        \"points\": 307,\n        \"comments\": 330\n      },\n      \"category\": \"General Discussion\",\n      \"top_comments\": [\n        {\n          \"author\": \"__jonas\",\n          \"text\": \"Interesting, I’m not big on AI but I have thought often it would be nice to have an ‘agent’ that monitors ebay or other classifieds sites for items based on a natural language description.<p>Something\",\n          \"score\": 0\n        },\n        {\n          \"author\": \"pranavj\",\n          \"text\": \"Banning AI agents is the new &quot;banning mobile browsers.&quot; Companies tried that too in the early smartphone era - remember when sites blocked mobile user agents to force desktop views?<p>The bu\",\n          \"score\": 0\n        },\n        {\n          \"author\": \"abroszka33\",\n          \"text\": \"Who cares, it&#x27;s my browser, it is for me to decide what I run, not for eBay. LLM, AdBlock or whatever else I want I will run it.\",\n          \"score\": 0\n        }\n      ],\n      \"has_external_url\": true\n    },\n    {\n      \"source\": \"Hacker News - Search: AI\",\n      \"title\": \"eBay explicitly bans AI \\\"buy for me\\\" agents in user agreement update\",\n      \"url\": \"https://www.valueaddedresource.net/ebay-bans-ai-agents-updates-arbitration-user-agreement-feb-2026/\",\n      \"hn_discussion\": \"https://news.ycombinator.com/item?id=46711574\",\n      \"author\": \"@bdcravens\",\n      \"posted\": \"1d ago\",\n      \"engagement\": {\n        \"points\": 307,\n        \"comments\": 330\n      },\n      \"category\": \"General Discussion\",\n      \"top_comments\": [],\n      \"has_external_url\": true\n    },\n    {\n      \"source\": \"Hacker News - Best Stories\",\n      \"title\": \"Show HN: Sweep, Open-weights 1.5B model for next-edit autocomplete\",\n      \"url\": \"https://huggingface.co/sweepai/sweep-next-edit-1.5B\",\n      \"hn_discussion\": \"https://news.ycombinator.com/item?id=46713106\",\n      \"author\": \"@williamzeng0\",\n      \"posted\": \"1d ago\",\n      \"engagement\": {\n        \"points\": 508,\n        \"comments\": 135\n      },\n      \"category\": \"Product Launch\",\n      \"top_comments\": [\n        {\n          \"author\": \"leonardcser\",\n          \"text\": \"Hi, I tried the model and I am super impressed by the performance&#x2F;quality. Thanks for making this open source!<p>I am the author of this Neovim plugin for edit completions. I was able to integrat\",\n          \"score\": 0\n        },\n        {\n          \"author\": \"KronisLV\",\n          \"text\": \"I remember using Qwen 2.5 Coder for autocomplete with Continue.dev, that experience was a mess both in JetBrains IDEs, as well as Visual Studio Code.<p>People posting stuff like this is really cool be\",\n          \"score\": 0\n        },\n        {\n          \"author\": \"vanillameow\",\n          \"text\": \"Sometimes when I use a plugin like this I get reminded just how much of a productivity nerf it is to code without an autocomplete AI. Honestly in my opinion if you write a lot of boilerplate code this\",\n          \"score\": 0\n        }\n      ],\n      \"has_external_url\": true\n    },\n    {\n      \"source\": \"Hacker News - Best Stories\",\n      \"title\": \"How AI destroys institutions\",\n      \"url\": \"https://cyberlaw.stanford.edu/publications/how-ai-destroys-institutions/\",\n      \"hn_discussion\": \"https://news.ycombinator.com/item?id=46705606\",\n      \"author\": \"@JeanKage\",\n      \"posted\": \"1d ago\",\n      \"engagement\": {\n        \"points\": 301,\n        \"comments\": 261\n      },\n      \"category\": \"General Discussion\",\n      \"top_comments\": [\n        {\n          \"author\": \"6DM\",\n          \"text\": \"I don&#x27;t think AI is the cause, it&#x27;s merely the mechanism that is speeding up what has already been happening.<p>Social media was already isolating people. It is being sped up by the use of A\",\n          \"score\": 0\n        },\n        {\n          \"author\": \"throwaw12\",\n          \"text\": \"&gt; Civic institutions - the rule of law, universities, and a free press - are the\\nbackbone of democratic life<p>It probably was in 1850-1950s, but not in the world I live today.<p>Press is not free \",\n          \"score\": 0\n        },\n        {\n          \"author\": \"alwayseasy\",\n          \"text\": \"Note this is the asbtract, so please let&#x27;s not debate the abstract...<p>The link to download the paper is here: <a href=\\\"https:&#x2F;&#x2F;papers.ssrn.com&#x2F;sol3&#x2F;papers.cfm?abstract_id=58\",\n          \"score\": 0\n        }\n      ],\n      \"has_external_url\": true\n    },\n    {\n      \"source\": \"Hacker News - Search: AI\",\n      \"title\": \"How AI destroys institutions\",\n      \"url\": \"https://cyberlaw.stanford.edu/publications/how-ai-destroys-institutions/\",\n      \"hn_discussion\": \"https://news.ycombinator.com/item?id=46705606\",\n      \"author\": \"@JeanKage\",\n      \"posted\": \"1d ago\",\n      \"engagement\": {\n        \"points\": 301,\n        \"comments\": 261\n      },\n      \"category\": \"General Discussion\",\n      \"top_comments\": [],\n      \"has_external_url\": true\n    },\n    {\n      \"source\": \"Hacker News - Top Stories\",\n      \"title\": \"It looks like the status/need-triage label was removed\",\n      \"url\": \"https://github.com/google-gemini/gemini-cli/issues/16728\",\n      \"hn_discussion\": \"https://news.ycombinator.com/item?id=46721179\",\n      \"author\": \"@nickswalker\",\n      \"posted\": \"9h ago\",\n      \"engagement\": {\n        \"points\": 274,\n        \"comments\": 69\n      },\n      \"category\": \"Open Source\",\n      \"top_comments\": [\n        {\n          \"author\": \"embedding-shape\",\n          \"text\": \"It&#x27;s easy to miss, but in the middle of the page:<p>&gt; 4609 remaining items<p>Seems gemini-cli and gemini-cli didn&#x27;t understand who themselves were, so they though someone else added&#x2F;\",\n          \"score\": 0\n        },\n        {\n          \"author\": \"PyWoody\",\n          \"text\": \"Heh. This reminds me of the time when our newly hired &quot;Salesforce Expert&quot; improved our support queue:<p><pre><code>  Every time Support received a new email, a ticket in Salesforce would be \",\n          \"score\": 0\n        },\n        {\n          \"author\": \"ryandrake\",\n          \"text\": \"A similar issue made HN last week, same repo, where an AI bot was having the same kind of argument with itself over and over on an issue. Someone mentioned: This sort of thing is why RAM is 800 bucks \",\n          \"score\": 0\n        }\n      ],\n      \"has_external_url\": true\n    },\n    {\n      \"source\": \"Hacker News - Best Stories\",\n      \"title\": \"Scientists find a way to regrow cartilage in mice and human tissue samples\",\n      \"url\": \"https://www.sciencedaily.com/releases/2026/01/260120000333.htm\",\n      \"hn_discussion\": \"https://news.ycombinator.com/item?id=46709179\",\n      \"author\": \"@saikatsg\",\n      \"posted\": \"1d ago\",\n      \"engagement\": {\n        \"points\": 298,\n        \"comments\": 91\n      },\n      \"category\": \"Product Launch\",\n      \"top_comments\": [\n        {\n          \"author\": \"observationist\",\n          \"text\": \"With all the mouse research, a lab should compile the top 300 interventions, lifestyles, regimens, etc, and apply it to a generation of mice. Give them all the best of the best gene edits, diets, envi\",\n          \"score\": 0\n        },\n        {\n          \"author\": \"tima101\",\n          \"text\": \"<a href=\\\"https:&#x2F;&#x2F;www.science.org&#x2F;doi&#x2F;10.1126&#x2F;science.adx6649\\\" rel=\\\"nofollow\\\">https:&#x2F;&#x2F;www.science.org&#x2F;doi&#x2F;10.1126&#x2F;science.adx6649</a><p>A small molecul\",\n          \"score\": 0\n        },\n        {\n          \"author\": \"levl289\",\n          \"text\": \"I’ve had my shoulders “cleaned up” arthroscopically, and the pain is still a major preventer of movement. I would love to stay on the mats longer with something that doesn’t harken to medieval times. \",\n          \"score\": 0\n        }\n      ],\n      \"has_external_url\": true\n    },\n    {\n      \"source\": \"Hacker News - Search: AI\",\n      \"title\": \"The Agentic AI Handbook: Production-Ready Patterns\",\n      \"url\": \"https://www.nibzard.com/agentic-handbook\",\n      \"hn_discussion\": \"https://news.ycombinator.com/item?id=46701969\",\n      \"author\": \"@SouravInsights\",\n      \"posted\": \"1d ago\",\n      \"engagement\": {\n        \"points\": 205,\n        \"comments\": 142\n      },\n      \"category\": \"General Discussion\",\n      \"top_comments\": [],\n      \"has_external_url\": true\n    }\n  ],\n  \"note\": \"Data collected from Hacker News API (completely free, no auth required!)\"\n}",
      "size": 22982,
      "language": "json"
    },
    "src/crawlbot.py": {
      "content": "\"\"\"\nAI Trend Crawler - Twitter Edition\nFetches daily trending AI topics with top tweets for Hongjie's Coze Bot\n\nRequirements:\npip install tweepy python-dotenv\n\nSetup:\n1. Get Twitter API credentials from https://developer.twitter.com/\n2. Create .env file with:\n   TWITTER_BEARER_TOKEN=your_bearer_token_here\n\"\"\"\n\nimport tweepy\nimport json\nfrom datetime import datetime, timedelta\nfrom collections import defaultdict\nimport os\nfrom dotenv import load_dotenv\n\n# Load environment variables\nload_dotenv()\n\nclass AITrendCrawler:\n    def __init__(self):\n        \"\"\"Initialize Twitter API client\"\"\"\n        bearer_token = os.getenv('TWITTER_BEBEARER_TOKEN')\n        if not bearer_token:\n            raise ValueError(\"TWITTER_BEARER_TOKEN not found in environment\")\n        \n        self.client = tweepy.Client(\n            bearer_token=bearer_token,\n            wait_on_rate_limit=True,  # Enable built-in rate limit handling\n            wait_on_rate_limit_notify=True  # Show notifications when rate limited\n        )\n        \n        # AI-focused accounts to monitor (reduced list to stay within free tier limits)\n        self.ai_accounts = [\n            'sama',             # Sam Altman (OpenAI)\n            'karpathy',         # Andrej Karpathy\n            'emollick',         # Ethan Mollick\n            'OpenAI',           # OpenAI Official\n            'GoogleDeepMind',   # DeepMind\n        ]\n        \n        # AI-related hashtags and keywords\n        self.ai_keywords = [\n            '#AI', '#LLM', '#ChatGPT', '#GPT4', '#Claude',\n            '#MachineLearning', '#DeepLearning', '#AGI',\n            '#OpenAI', '#Anthropic', '#Gemini'\n        ]\n    \n    def fetch_account_tweets(self, username, limit=5):\n        \"\"\"Fetch recent tweets from a specific account with rate limit handling\"\"\"\n        import time\n        from datetime import datetime\n        \n        try:\n            print(f\"   🔍 Fetching @{username}...\")\n            \n            # Get user ID\n            try:\n                user = self.client.get_user(username=username)\n                if not user.data:\n                    print(f\"   ✓ @{username}: User not found\")\n                    return []\n                \n                # Add a small delay between API calls\n                time.sleep(2)\n                \n                # Get recent tweets (reduced limit to 5 to stay within rate limits)\n                tweets = self.client.get_users_tweets(\n                    id=user.data.id,\n                    max_results=min(limit, 5),  # Max 5 tweets per account\n                    tweet_fields=['created_at', 'public_metrics', 'conversation_id', 'text'],\n                    exclude=['retweets', 'replies']\n                )\n                \n                if not tweets.data:\n                    print(f\"   ✓ @{username}: No recent tweets\")\n                    return []\n                \n                print(f\"   ✓ @{username}: Found {len(tweets.data)} tweets\")\n                return tweets.data\n                \n            except tweepy.TooManyRequests as e:\n                # If we hit rate limits, wait and retry once\n                wait_time = int(e.response.headers.get('x-rate-limit-reset', 60)) - int(time.time()) + 5\n                wait_time = max(wait_time, 60)  # Wait at least 60 seconds\n                print(f\"   ⏳ Rate limited. Waiting {wait_time} seconds before retrying @{username}...\")\n                time.sleep(wait_time)\n                return []\n                \n            except Exception as e:\n                print(f\"   ⚠️  Error fetching @{username}: {str(e)[:100]}...\")\n                return []\n                \n        except Exception as e:\n            print(f\"   ✗ Unexpected error with @{username}: {str(e)[:100]}...\")\n            return []\n    \n    def search_trending_ai_topics(self, limit=50):\n        \"\"\"Search for trending AI-related tweets\"\"\"\n        import time\n        \n        try:\n            # Build search query for AI topics\n            query = ' OR '.join(self.ai_keywords[:5])  # Limit to avoid query length issues\n            query += ' -is:retweet lang:en'\n            \n            try:\n                # Search recent tweets with rate limiting\n                tweets = self.client.search_recent_tweets(\n                    query=query,\n                    max_results=limit,\n                    tweet_fields=['created_at', 'public_metrics', 'author_id', 'text'],\n                    user_fields=['username', 'name'],\n                    expansions=['author_id']\n                )\n                \n                if not tweets.data:\n                    print(\"   ✓ No trending AI topics found\")\n                    return []\n                \n                # Process results safely\n                result = []\n                if hasattr(tweets, 'includes') and hasattr(tweets.includes, 'users'):\n                    users = {user.id: user for user in tweets.includes.users}\n                    for tweet in tweets.data:\n                        tweet_dict = tweet.data if hasattr(tweet, 'data') else tweet._json\n                        tweet_dict['user'] = users.get(tweet.author_id)\n                        result.append(tweet_dict)\n                \n                return result\n                \n            except Exception as e:\n                if '429' in str(e):\n                    print(\"   ⚠️  Rate limited while searching trends, waiting...\")\n                    time.sleep(120)  # Wait 2 minutes before continuing\n                else:\n                    print(f\"   ✗ Error searching trends: {str(e)}\")\n                return []\n                \n        except Exception as e:\n            print(f\"   ✗ Unexpected error in search_trending_ai_topics: {str(e)}\")\n            return []\n    \n    def calculate_engagement_score(self, metrics):\n        \"\"\"Calculate engagement score for ranking\"\"\"\n        if not metrics:\n            return 0\n        \n        # Weighted scoring: retweets > likes > replies\n        return (\n            metrics.get('retweet_count', 0) * 3 +\n            metrics.get('like_count', 0) * 2 +\n            metrics.get('reply_count', 0) * 1\n        )\n    \n    def classify_sentiment(self, text):\n        \"\"\"Simple sentiment classification based on keywords\"\"\"\n        text_lower = text.lower()\n        \n        # Keyword-based classification\n        if any(word in text_lower for word in ['scary', 'dangerous', 'terrifying', 'fear', 'worried']):\n            return 'Fear/Concern'\n        elif any(word in text_lower for word in ['lol', 'haha', 'funny', '😂', '🤣', 'hilarious']):\n            return 'Humor'\n        elif any(word in text_lower for word in ['overhype', 'scam', 'skeptical', 'doubt', 'bs']):\n            return 'Skepticism'\n        elif any(word in text_lower for word in ['amazing', 'incredible', 'wow', 'mind-blowing', '🤯']):\n            return 'Excitement'\n        elif any(word in text_lower for word in ['finally', 'useful', 'practical', 'works']):\n            return 'Practical Interest'\n        else:\n            return 'Discussion'\n    \n    def crawl_daily_trends(self, output_file='daily_trends.json'):\n        \"\"\"Main crawler: fetch and process daily AI trends with rate limit handling\"\"\"\n        import time\n        from datetime import datetime\n        \n        print(f\"🕷️  Starting AI Trend Crawl at {datetime.now()}\")\n        print(\"ℹ️  This may take a few minutes due to Twitter API rate limits...\")\n        \"\"\"Main crawler: fetch and process daily AI trends\"\"\"\n        print(f\"🕷️  Starting AI Trend Crawl at {datetime.now()}\")\n        \n        all_tweets = []\n        \n        # 1. Fetch from curated AI accounts with delays\n        print(\"\\n📱 Fetching from AI influencer accounts...\")\n        all_tweets = []\n        \n        for i, username in enumerate(self.ai_accounts):\n            # Add increasing delay between account requests\n            if i > 0:\n                wait_time = min(10 * i, 60)  # Up to 60 seconds delay between accounts\n                print(f\"   ⏳ Waiting {wait_time} seconds before next request...\")\n                time.sleep(wait_time)\n                \n            tweets = self.fetch_account_tweets(username, limit=3)  # Reduced to 3 tweets per account\n            for tweet in tweets:\n                # Handle both dictionary and object access\n                tweet_id = tweet.get('id') if isinstance(tweet, dict) else getattr(tweet, 'id', None)\n                text = tweet.get('text') if isinstance(tweet, dict) else getattr(tweet, 'text', '')\n                created_at = tweet.get('created_at') if isinstance(tweet, dict) else getattr(tweet, 'created_at', None)\n                metrics = tweet.get('public_metrics') if isinstance(tweet, dict) else getattr(tweet, 'public_metrics', {})\n                \n                tweet_dict = {\n                    'id': tweet_id,\n                    'text': text,\n                    'created_at': created_at,\n                    'public_metrics': metrics,\n                    'source_type': 'account',\n                    'source_name': f'@{username}',\n                    'user': tweet.get('user') if isinstance(tweet, dict) else getattr(tweet, 'user', None)\n                }\n                all_tweets.append(tweet_dict)\n        \n        # 2. Fetch trending hashtag tweets (with delay)\n        print(\"\\n🔥 Searching trending AI topics...\")\n        time.sleep(5)  # Additional delay before trending search\n        trending = self.search_trending_ai_topics(limit=10)  # Reduced from 30 to 10\n        for tweet in trending:\n            # Handle both dictionary and object access\n            tweet_id = tweet.get('id') if isinstance(tweet, dict) else getattr(tweet, 'id', None)\n            text = tweet.get('text') if isinstance(tweet, dict) else getattr(tweet, 'text', '')\n            created_at = tweet.get('created_at') if isinstance(tweet, dict) else getattr(tweet, 'created_at', None)\n            metrics = tweet.get('public_metrics') if isinstance(tweet, dict) else getattr(tweet, 'public_metrics', {})\n            \n            tweet_dict = {\n                'id': tweet_id,\n                'text': text,\n                'created_at': created_at,\n                'public_metrics': metrics,\n                'source_type': 'trending',\n                'source_name': 'AI Trending',\n                'user': tweet.get('user') if isinstance(tweet, dict) else getattr(tweet, 'user', None)\n            }\n            all_tweets.append(tweet_dict)\n        print(f\"   ✓ Found {len(trending)} trending tweets\")\n        \n        # 3. Filter out any invalid tweets\n        valid_tweets = [t for t in all_tweets if t.get('id') and t.get('text')]\n        \n        if not valid_tweets:\n            print(\"\\n⚠️  No valid tweets found to analyze\")\n            return {'error': 'No valid tweets found', 'trends': []}\n            \n        # 4. Rank by engagement\n        valid_tweets.sort(\n            key=lambda t: self.calculate_engagement_score(t.get('public_metrics', {})),\n            reverse=True\n        )\n        \n        # 5. Take top 10 and format\n        top_tweets = valid_tweets[:10]\n        \n        trends = []\n        for tweet in top_tweets:\n            # Safely get user info\n            user = tweet.get('user')\n            if user:\n                if hasattr(user, 'username'):\n                    username = f\"@{user.username}\"\n                elif isinstance(user, dict) and 'username' in user:\n                    username = f\"@{user['username']}\"\n                else:\n                    username = \"Unknown\"\n            else:\n                username = \"Unknown\"\n            \n            # Safely get text and truncate\n            text = tweet.get('text', '')\n            title = (text[:97] + '...') if len(text) > 100 else text\n            \n            # Safely get metrics\n            metrics = tweet.get('public_metrics', {})\n            if hasattr(metrics, 'get'):\n                likes = metrics.get('like_count', 0)\n                retweets = metrics.get('retweet_count', 0)\n                replies = metrics.get('reply_count', 0)\n            else:\n                likes = retweets = replies = 0\n            \n            trend_item = {\n                'source': f\"Twitter - {tweet.get('source_name', 'AI Community')}\",\n                'title': title,\n                'url': f\"https://twitter.com/i/web/status/{tweet.get('id')}\",\n                'top_tweet': text,\n                'author': username,\n                'engagement': {\n                    'likes': likes,\n                    'retweets': retweets,\n                    'replies': replies,\n                },\n                'sentiment_tag': self.classify_sentiment(text)\n            }\n            trends.append(trend_item)\n        \n        # 5. Create output JSON with metadata\n        output = {\n            'date': datetime.now().strftime('%Y-%m-%d'),\n            'generated_at': datetime.now().isoformat(),\n            'total_analyzed': len(all_tweets),\n            'accounts_processed': len(self.ai_accounts),\n            'trends_found': len(trends),\n            'trends': trends,\n            'note': 'Data collected with rate limiting to comply with Twitter API terms.'\n        }\n        \n        # 6. Save to file\n        with open(output_file, 'w', encoding='utf-8') as f:\n            json.dump(output, f, indent=2, ensure_ascii=False)\n        \n        print(f\"\\n✅ Crawl complete! Generated {len(trends)} trend items\")\n        print(f\"📄 Output saved to: {output_file}\")\n        \n        return output\n\n\ndef main():\n    \"\"\"Run the crawler\"\"\"\n    try:\n        crawler = AITrendCrawler()\n        result = crawler.crawl_daily_trends()\n        \n        # Print sample if we have results\n        print(\"\\n📊 Results:\")\n        if not result.get('trends'):\n            print(\"No trends found. This might be due to rate limiting. Please try again later.\")\n        else:\n            print(f\"Found {len(result['trends'])} trends\")\n            if result['trends']:\n                print(\"\\nSample trend:\")\n                print(json.dumps(result['trends'][0], indent=2))\n        \n    except Exception as e:\n        print(f\"❌ Error: {e}\")\n        import traceback\n        traceback.print_exc()\n        return 1\n    \n    return 0\n\n\nif __name__ == '__main__':\n    import sys\n    sys.exit(main())",
      "size": 14229,
      "language": "python"
    },
    "src/hacker.py": {
      "content": "\"\"\"\nAI Trend Crawler - Hacker News Edition\nFetches daily trending AI topics from Hacker News for Hongjie's Coze Bot\n\nRequirements:\npip install requests python-dotenv\n\nSetup:\nNO API KEY NEEDED! Hacker News API is completely free and open.\nJust run: python hackernews_crawler.py\n\"\"\"\n\nimport requests\nimport json\nfrom datetime import datetime, timedelta\nimport time\nfrom collections import defaultdict\n\nclass AITrendCrawler:\n    def __init__(self):\n        \"\"\"Initialize Hacker News API client (no auth needed!)\"\"\"\n        self.base_url = 'https://hacker-news.firebaseio.com/v0'\n        \n        # AI-related keywords for filtering\n        self.ai_keywords = [\n            'ai', 'gpt', 'chatgpt', 'openai', 'claude', 'anthropic',\n            'llm', 'large language model', 'machine learning', 'ml',\n            'deep learning', 'neural network', 'transformer',\n            'stable diffusion', 'midjourney', 'dall-e', 'gemini',\n            'copilot', 'agi', 'artificial general intelligence',\n            'langchain', 'llama', 'mistral', 'embedding',\n            'prompt engineering', 'fine-tuning', 'rag',\n            'generative ai', 'diffusion model', 'reinforcement learning'\n        ]\n        \n        print(\"✓ Connected to Hacker News API (no auth required!)\")\n    \n    def get_item(self, item_id):\n        \"\"\"Fetch a single item (story/comment) from HN\"\"\"\n        try:\n            response = requests.get(f\"{self.base_url}/item/{item_id}.json\", timeout=5)\n            if response.status_code == 200:\n                return response.json()\n            return None\n        except Exception as e:\n            return None\n    \n    def get_top_stories(self, limit=100):\n        \"\"\"Fetch top story IDs from Hacker News\"\"\"\n        try:\n            print(\"   🔥 Fetching top stories...\")\n            response = requests.get(f\"{self.base_url}/topstories.json\", timeout=10)\n            \n            if response.status_code == 200:\n                story_ids = response.json()[:limit]\n                print(f\"   ✓ Found {len(story_ids)} top stories\")\n                return story_ids\n            return []\n            \n        except Exception as e:\n            print(f\"   ✗ Error fetching top stories: {str(e)}\")\n            return []\n    \n    def get_best_stories(self, limit=100):\n        \"\"\"Fetch best story IDs from Hacker News\"\"\"\n        try:\n            print(\"   ⭐ Fetching best stories...\")\n            response = requests.get(f\"{self.base_url}/beststories.json\", timeout=10)\n            \n            if response.status_code == 200:\n                story_ids = response.json()[:limit]\n                print(f\"   ✓ Found {len(story_ids)} best stories\")\n                return story_ids\n            return []\n            \n        except Exception as e:\n            print(f\"   ✗ Error fetching best stories: {str(e)}\")\n            return []\n    \n    def get_new_stories(self, limit=100):\n        \"\"\"Fetch new story IDs from Hacker News\"\"\"\n        try:\n            print(\"   🆕 Fetching new stories...\")\n            response = requests.get(f\"{self.base_url}/newstories.json\", timeout=10)\n            \n            if response.status_code == 200:\n                story_ids = response.json()[:limit]\n                print(f\"   ✓ Found {len(story_ids)} new stories\")\n                return story_ids\n            return []\n            \n        except Exception as e:\n            print(f\"   ✗ Error fetching new stories: {str(e)}\")\n            return []\n    \n    def is_ai_related(self, title, url=''):\n        \"\"\"Check if story is AI-related\"\"\"\n        text = (title + ' ' + url).lower()\n        return any(keyword in text for keyword in self.ai_keywords)\n    \n    def fetch_story_details(self, story_id):\n        \"\"\"Fetch complete story details including comments\"\"\"\n        try:\n            story = self.get_item(story_id)\n            \n            if not story or story.get('type') != 'story':\n                return None\n            \n            # Get top comments if available\n            top_comments = []\n            if story.get('kids'):\n                # Get first 3 top-level comments\n                for comment_id in story['kids'][:3]:\n                    comment = self.get_item(comment_id)\n                    if comment and comment.get('text'):\n                        top_comments.append({\n                            'author': comment.get('by', 'unknown'),\n                            'text': comment.get('text', '')[:200],  # First 200 chars\n                            'score': comment.get('score', 0)\n                        })\n                    time.sleep(0.1)  # Small delay between requests\n            \n            return {\n                'id': story.get('id'),\n                'title': story.get('title', ''),\n                'url': story.get('url', f\"https://news.ycombinator.com/item?id={story.get('id')}\"),\n                'score': story.get('score', 0),\n                'by': story.get('by', 'unknown'),\n                'time': story.get('time', 0),\n                'descendants': story.get('descendants', 0),  # Comment count\n                'text': story.get('text', ''),  # For Ask HN, Show HN posts\n                'top_comments': top_comments\n            }\n            \n        except Exception as e:\n            return None\n    \n    def search_algolia(self, query, tags='story', limit=30):\n        \"\"\"Search Hacker News using Algolia API\"\"\"\n        try:\n            print(f\"   🔍 Searching Algolia for: {query}...\")\n            \n            # HN Algolia search endpoint\n            algolia_url = 'https://hn.algolia.com/api/v1/search'\n            \n            params = {\n                'query': query,\n                'tags': tags,\n                'hitsPerPage': limit,\n                'numericFilters': f'created_at_i>{int((datetime.now() - timedelta(days=7)).timestamp())}'  # Last 7 days\n            }\n            \n            response = requests.get(algolia_url, params=params, timeout=10)\n            \n            if response.status_code == 200:\n                data = response.json()\n                hits = data.get('hits', [])\n                print(f\"   ✓ Found {len(hits)} results for '{query}'\")\n                return hits\n            return []\n            \n        except Exception as e:\n            print(f\"   ✗ Error searching Algolia: {str(e)}\")\n            return []\n    \n    def calculate_engagement_score(self, story):\n        \"\"\"Calculate engagement score for ranking\"\"\"\n        # HN scoring: points + comments weighted\n        score = story.get('score', 0) * 2\n        score += story.get('descendants', 0) * 3  # Comments are valuable\n        \n        # Boost recent stories (last 24 hours)\n        story_time = datetime.fromtimestamp(story.get('time', 0))\n        age_hours = (datetime.now() - story_time).total_seconds() / 3600\n        if age_hours < 24:\n            score *= 1.5\n        \n        return score\n    \n    def classify_category(self, title, url='', text=''):\n        \"\"\"Classify the type of AI content\"\"\"\n        combined = (title + ' ' + url + ' ' + text).lower()\n        \n        # Check for specific categories\n        if any(word in combined for word in ['show hn', 'launch', 'release', 'announcing']):\n            return 'Product Launch'\n        elif any(word in combined for word in ['ask hn', 'question', 'help', 'advice']):\n            return 'Discussion/Question'\n        elif any(word in combined for word in ['paper', 'research', 'arxiv', 'study']):\n            return 'Research Paper'\n        elif any(word in combined for word in ['tutorial', 'guide', 'how to', 'learn']):\n            return 'Tutorial/Guide'\n        elif any(word in combined for word in ['news', 'announces', 'reveals', 'unveils']):\n            return 'News/Announcement'\n        elif any(word in combined for word in ['opinion', 'thoughts', 'perspective']):\n            return 'Opinion/Analysis'\n        elif any(word in combined for word in ['open source', 'github', 'repo']):\n            return 'Open Source'\n        else:\n            return 'General Discussion'\n    \n    def get_story_age_str(self, timestamp):\n        \"\"\"Get human-readable story age\"\"\"\n        story_time = datetime.fromtimestamp(timestamp)\n        age = datetime.now() - story_time\n        \n        if age.total_seconds() < 3600:\n            return f\"{int(age.total_seconds() / 60)}m ago\"\n        elif age.total_seconds() < 86400:\n            return f\"{int(age.total_seconds() / 3600)}h ago\"\n        else:\n            return f\"{int(age.days)}d ago\"\n    \n    def crawl_daily_trends(self, output_file='daily_trends_hn.json'):\n        \"\"\"Main crawler: fetch and process daily AI trends from Hacker News\"\"\"\n        print(f\"🕷️  Starting Hacker News AI Trend Crawl at {datetime.now()}\")\n        print(\"🔶 Hacker News - Tech community's favorite discussion platform\\n\")\n        \n        all_stories = []\n        \n        # 1. Get top stories and filter for AI\n        print(\"\\n📱 Fetching top stories...\")\n        top_ids = self.get_top_stories(limit=100)\n        \n        print(\"   🔍 Filtering for AI-related stories...\")\n        for story_id in top_ids[:50]:  # Check first 50\n            story = self.get_item(story_id)\n            if story and self.is_ai_related(story.get('title', ''), story.get('url', '')):\n                details = self.fetch_story_details(story_id)\n                if details:\n                    details['source_type'] = 'top'\n                    details['source_name'] = 'Top Stories'\n                    all_stories.append(details)\n            time.sleep(0.1)  # Be respectful\n        \n        print(f\"   ✓ Found {len(all_stories)} AI stories in top\")\n        \n        # 2. Get best stories and filter for AI\n        print(\"\\n⭐ Fetching best stories...\")\n        best_ids = self.get_best_stories(limit=50)\n        \n        for story_id in best_ids[:30]:\n            story = self.get_item(story_id)\n            if story and self.is_ai_related(story.get('title', ''), story.get('url', '')):\n                details = self.fetch_story_details(story_id)\n                if details:\n                    details['source_type'] = 'best'\n                    details['source_name'] = 'Best Stories'\n                    all_stories.append(details)\n            time.sleep(0.1)\n        \n        print(f\"   ✓ Total AI stories so far: {len(all_stories)}\")\n        \n        # 3. Search for specific AI topics using Algolia\n        print(\"\\n🔍 Searching for specific AI topics...\")\n        search_queries = ['ChatGPT', 'LLM', 'OpenAI', 'Claude', 'AI']\n        \n        for query in search_queries:\n            results = self.search_algolia(query, limit=10)\n            for hit in results:\n                # Convert Algolia result to our format\n                story = {\n                    'id': hit.get('objectID'),\n                    'title': hit.get('title', ''),\n                    'url': hit.get('url', f\"https://news.ycombinator.com/item?id={hit.get('objectID')}\"),\n                    'score': hit.get('points', 0),\n                    'by': hit.get('author', 'unknown'),\n                    'time': hit.get('created_at_i', 0),\n                    'descendants': hit.get('num_comments', 0),\n                    'text': '',\n                    'top_comments': [],\n                    'source_type': 'search',\n                    'source_name': f'Search: {query}'\n                }\n                all_stories.append(story)\n            time.sleep(0.5)\n        \n        print(f\"\\n📊 Total stories collected: {len(all_stories)}\")\n        \n        if not all_stories:\n            print(\"\\n⚠️  No AI stories found\")\n            return {'error': 'No stories found', 'trends': []}\n        \n        # 4. Remove duplicates\n        seen_ids = set()\n        unique_stories = []\n        for story in all_stories:\n            story_id = story.get('id')\n            if story_id and story_id not in seen_ids:\n                seen_ids.add(story_id)\n                unique_stories.append(story)\n        \n        print(f\"📊 Unique stories after deduplication: {len(unique_stories)}\")\n        \n        # 5. Filter recent stories (last 48 hours for HN)\n        now = datetime.now()\n        cutoff = now - timedelta(hours=48)\n        \n        recent_stories = []\n        for story in unique_stories:\n            story_time = datetime.fromtimestamp(story.get('time', 0))\n            if story_time >= cutoff:\n                recent_stories.append(story)\n        \n        print(f\"📊 Recent stories (last 48h): {len(recent_stories)}\")\n        \n        if not recent_stories:\n            print(\"\\n⚠️  No recent stories found, using all unique stories\")\n            recent_stories = unique_stories\n        \n        # 6. Rank by engagement\n        recent_stories.sort(\n            key=lambda s: self.calculate_engagement_score(s),\n            reverse=True\n        )\n        \n        # 7. Take top 20 and format\n        top_stories = recent_stories[:20]\n        \n        trends = []\n        print(\"\\n🔍 Processing top stories...\")\n        for story in top_stories:\n            title = story.get('title', '')\n            url = story.get('url', '')\n            \n            # Build HN discussion URL\n            hn_url = f\"https://news.ycombinator.com/item?id={story.get('id')}\"\n            \n            trend_item = {\n                'source': f\"Hacker News - {story.get('source_name', 'HN')}\",\n                'title': title,\n                'url': url if url and not url.startswith('https://news.ycombinator') else hn_url,\n                'hn_discussion': hn_url,\n                'author': f\"@{story.get('by', 'unknown')}\",\n                'posted': self.get_story_age_str(story.get('time', 0)),\n                'engagement': {\n                    'points': story.get('score', 0),\n                    'comments': story.get('descendants', 0),\n                },\n                'category': self.classify_category(title, url, story.get('text', '')),\n                'top_comments': story.get('top_comments', []),\n                'has_external_url': bool(url and not url.startswith('https://news.ycombinator')),\n            }\n            trends.append(trend_item)\n        \n        # 8. Create output JSON with metadata\n        output = {\n            'date': datetime.now().strftime('%Y-%m-%d'),\n            'generated_at': datetime.now().isoformat(),\n            'platform': 'Hacker News',\n            'total_analyzed': len(all_stories),\n            'unique_stories': len(unique_stories),\n            'recent_stories': len(recent_stories),\n            'trends_found': len(trends),\n            'trends': trends,\n            'note': 'Data collected from Hacker News API (completely free, no auth required!)'\n        }\n        \n        # 9. Save to file\n        with open(output_file, 'w', encoding='utf-8') as f:\n            json.dump(output, f, indent=2, ensure_ascii=False)\n        \n        print(f\"\\n✅ Crawl complete! Generated {len(trends)} trend items\")\n        print(f\"📄 Output saved to: {output_file}\")\n        \n        return output\n\n\ndef main():\n    \"\"\"Run the crawler\"\"\"\n    try:\n        crawler = AITrendCrawler()\n        result = crawler.crawl_daily_trends()\n        \n        # Print sample results\n        print(\"\\n\" + \"=\"*60)\n        print(\"📊 SAMPLE RESULTS\")\n        print(\"=\"*60)\n        \n        if result.get('trends'):\n            print(f\"\\nFound {len(result['trends'])} trends\\n\")\n            \n            # Show first 3 trends\n            for i, trend in enumerate(result['trends'][:3], 1):\n                print(f\"\\n--- Trend #{i} ---\")\n                print(f\"Title: {trend['title']}\")\n                print(f\"Category: {trend['category']}\")\n                print(f\"Author: {trend['author']}\")\n                print(f\"Posted: {trend['posted']}\")\n                print(f\"Engagement: {trend['engagement']['points']} points, {trend['engagement']['comments']} comments\")\n                print(f\"HN Discussion: {trend['hn_discussion']}\")\n                if trend['has_external_url']:\n                    print(f\"External URL: {trend['url']}\")\n                if trend['top_comments']:\n                    print(f\"Top Comment by @{trend['top_comments'][0]['author']}: {trend['top_comments'][0]['text'][:100]}...\")\n        else:\n            print(\"No trends found.\")\n            print(f\"Error: {result.get('error', 'Unknown error')}\")\n        \n    except Exception as e:\n        print(f\"❌ Error: {e}\")\n        import traceback\n        traceback.print_exc()\n        return 1\n    \n    return 0\n\n\nif __name__ == '__main__':\n    import sys\n    sys.exit(main())",
      "size": 16460,
      "language": "python"
    },
    "src/config/config.py": {
      "content": "\"\"\"\nConfiguration settings for Twitter Sentiment Crawler\n\"\"\"\nfrom typing import List, Dict, Any\nfrom pydantic import HttpUrl\nfrom pydantic_settings import BaseSettings\n\nclass Settings(BaseSettings):\n    # Twitter API credentials\n    TWITTER_BEARER_TOKEN: str\n    \n    # API request settings\n    MAX_RETRIES: int = 3\n    REQUEST_TIMEOUT: int = 10\n    RATE_LIMIT_DELAY: float = 2.0\n    \n    # Data collection limits\n    MAX_TWEETS_PER_HASHTAG: int = 100\n    MAX_TRENDING_TOPICS: int = 10\n    MAX_NEWS_ITEMS: int = 10\n    \n    # Caching\n    SENTIMENT_CACHE_SIZE: int = 1000\n    CACHE_TTL_HOURS: int = 24\n    \n    # Sentiment analysis\n    SENTIMENT_MODEL: str = \"distilbert-base-uncased-finetuned-sst-2-english\"\n    SENTIMENT_THRESHOLD: float = 0.7  # Confidence threshold for sentiment classification\n    \n    # Output\n    OUTPUT_DIR: str = \"output\"\n    LOG_LEVEL: str = \"INFO\"\n    \n    class Config:\n        env_file = \".env\"\n        case_sensitive = True\n\n# Create settings instance\nsettings = Settings()\n",
      "size": 1004,
      "language": "python"
    },
    "src/agents/social_sentiment_agent.py": {
      "content": "\"\"\"\nSocial Sentiment Analysis Agent\n\nThis agent analyzes social media sentiment using the Twitter API and a pre-trained\nsentiment analysis model. It demonstrates:\n1. Tool usage (Twitter API, Sentiment Analysis)\n2. Asynchronous operations\n3. State management\n4. Error handling and retries\n\"\"\"\nimport asyncio\nimport logging\nfrom typing import Dict, Any, List, Optional\nfrom datetime import datetime\nfrom pathlib import Path\n\nfrom src.agents.agent import Agent\nfrom src.agents.tools import ToolManager, tool\n\n# Import the necessary components from social_proved.py\nfrom src.social.social_proved import TwitterAPIClient, SentimentAnalyzer\n\nlogger = logging.getLogger(__name__)\n\nclass SocialSentimentAgent(Agent):\n    \"\"\"Agent for analyzing social media sentiment.\n    \n    This agent demonstrates:\n    - Integration with external APIs (Twitter)\n    - Using pre-trained ML models\n    - Asynchronous operations\n    - Tool usage and composition\n    \"\"\"\n    \n    def __init__(self, agent_id: str = None, config: Optional[Dict] = None):\n        \"\"\"Initialize the SocialSentimentAgent.\n        \n        Args:\n            agent_id: Unique identifier for the agent\n            config: Configuration dictionary with API keys and settings\n        \"\"\"\n        super().__init__(agent_id=agent_id or \"social_sentiment_agent\")\n        self.config = config or {}\n        self.tool_manager = ToolManager()\n        self.twitter_client = None\n        self.sentiment_analyzer = None\n        \n        # Initialize components\n        self._initialize_components()\n        \n        # Register tools\n        self._register_tools()\n    \n    def _initialize_components(self) -> None:\n        \"\"\"Initialize Twitter client and sentiment analyzer.\"\"\"\n        # Initialize Twitter client if API key is provided\n        if \"twitter_bearer_token\" in self.config:\n            self.twitter_client = TwitterAPIClient(\n                bearer_token=self.config[\"twitter_bearer_token\"]\n            )\n        \n        # Initialize sentiment analyzer\n        self.sentiment_analyzer = SentimentAnalyzer()\n        \n        logger.info(f\"{self.agent_id} initialized with config: {self.config}\")\n    \n    def _register_tools(self) -> None:\n        \"\"\"Register available tools with the tool manager.\"\"\"\n        if self.twitter_client:\n            self.tool_manager.register_function(\n                self.get_trending_topics,\n                name=\"get_trending_topics\",\n                description=\"Get currently trending topics on Twitter\"\n            )\n            self.tool_manager.register_function(\n                self.analyze_hashtag_sentiment,\n                name=\"analyze_hashtag_sentiment\",\n                description=\"Analyze sentiment of tweets with a specific hashtag\"\n            )\n        \n        # Add general sentiment analysis tool\n        self.tool_manager.register_function(\n            self.analyze_text_sentiment,\n            name=\"analyze_text_sentiment\",\n            description=\"Analyze sentiment of arbitrary text\"\n        )\n    \n    async def _process(self, input_data: Dict[str, Any], context: Dict) -> Dict:\n        \"\"\"Process incoming requests and route to appropriate handler.\n        \n        Args:\n            input_data: Dictionary containing the request data\n            context: Execution context and shared state\n            \n        Returns:\n            Dictionary containing the analysis results\n        \"\"\"\n        action = input_data.get(\"action\")\n        \n        try:\n            if action == \"analyze_hashtag\":\n                hashtag = input_data.get(\"hashtag\", \"\")\n                if not hashtag.startswith(\"#\"):\n                    hashtag = f\"#{hashtag}\"\n                return await self.analyze_hashtag_sentiment(hashtag)\n                \n            elif action == \"get_trending\":\n                woeid = input_data.get(\"woeid\", 1)  # Default to worldwide\n                return await self.get_trending_topics(woeid)\n                \n            elif action == \"analyze_text\":\n                text = input_data.get(\"text\", \"\")\n                return await self.analyze_text_sentiment(text)\n                \n            else:\n                return {\n                    \"status\": \"error\",\n                    \"message\": f\"Unknown action: {action}\",\n                    \"available_actions\": [\"analyze_hashtag\", \"get_trending\", \"analyze_text\"]\n                }\n                \n        except Exception as e:\n            logger.error(f\"Error processing request: {str(e)}\", exc_info=True)\n            return {\n                \"status\": \"error\",\n                \"message\": str(e)\n            }\n    \n    # --- Tool Methods ---\n    \n    async def get_trending_topics(self, woeid: int = 1) -> Dict:\n        \"\"\"Get currently trending topics on Twitter.\n        \n        Args:\n            woeid: Where On Earth ID (location for trends)\n            \n        Returns:\n            Dictionary containing trending topics\n        \"\"\"\n        if not self.twitter_client:\n            return {\"error\": \"Twitter client not configured\"}\n            \n        try:\n            # Initialize Twitter client session if needed\n            if not hasattr(self.twitter_client, 'session') or self.twitter_client.session.closed:\n                self.twitter_client.session = aiohttp.ClientSession()\n                \n            trends = await self.twitter_client.get_trending_topics(woeid)\n            return {\n                \"status\": \"success\",\n                \"woeid\": woeid,\n                \"trends\": trends,\n                \"timestamp\": datetime.utcnow().isoformat()\n            }\n            \n        except Exception as e:\n            logger.error(f\"Error getting trending topics: {str(e)}\")\n            return {\n                \"status\": \"error\",\n                \"message\": f\"Failed to get trending topics: {str(e)}\"\n            }\n    \n    async def analyze_hashtag_sentiment(self, hashtag: str, max_tweets: int = 50) -> Dict:\n        \"\"\"Analyze sentiment of tweets with a specific hashtag.\n        \n        Args:\n            hashtag: The hashtag to analyze (with or without #)\n            max_tweets: Maximum number of tweets to analyze\n            \n        Returns:\n            Dictionary containing sentiment analysis results\n        \"\"\"\n        if not self.twitter_client:\n            return {\"error\": \"Twitter client not configured\"}\n            \n        try:\n            # Ensure hashtag starts with #\n            if not hashtag.startswith(\"#\"):\n                hashtag = f\"#{hashtag}\"\n                \n            logger.info(f\"Analyzing sentiment for hashtag: {hashtag}\")\n            \n            # Search for tweets with the hashtag\n            tweets = await self.twitter_client.search_tweets(hashtag, max_results=max_tweets)\n            \n            if not tweets:\n                return {\n                    \"status\": \"success\",\n                    \"hashtag\": hashtag,\n                    \"tweet_count\": 0,\n                    \"message\": \"No tweets found with this hashtag\"\n                }\n            \n            # Analyze sentiment for each tweet\n            sentiment_scores = []\n            sentiment_labels = []\n            \n            for tweet in tweets:\n                text = tweet.get(\"text\", \"\")\n                if text:\n                    result = self.sentiment_analyzer.analyze(text)\n                    sentiment_scores.append(result[\"score\"])\n                    sentiment_labels.append(result[\"label\"])\n            \n            # Calculate aggregate statistics\n            avg_score = sum(sentiment_scores) / len(sentiment_scores) if sentiment_scores else 0\n            positive_count = sum(1 for label in sentiment_labels if label == \"POSITIVE\")\n            negative_count = sum(1 for label in sentiment_labels if label == \"NEGATIVE\")\n            neutral_count = len(sentiment_labels) - positive_count - negative_count\n            \n            return {\n                \"status\": \"success\",\n                \"hashtag\": hashtag,\n                \"tweet_count\": len(tweets),\n                \"analyzed_count\": len(sentiment_scores),\n                \"average_sentiment_score\": avg_score,\n                \"positive_tweets\": positive_count,\n                \"negative_tweets\": negative_count,\n                \"neutral_tweets\": neutral_count,\n                \"sentiment_distribution\": {\n                    \"positive\": positive_count / len(sentiment_scores) if sentiment_scores else 0,\n                    \"negative\": negative_count / len(sentiment_scores) if sentiment_scores else 0,\n                    \"neutral\": neutral_count / len(sentiment_scores) if sentiment_scores else 0\n                },\n                \"timestamp\": datetime.utcnow().isoformat()\n            }\n            \n        except Exception as e:\n            logger.error(f\"Error analyzing hashtag sentiment: {str(e)}\", exc_info=True)\n            return {\n                \"status\": \"error\",\n                \"message\": f\"Failed to analyze hashtag sentiment: {str(e)}\"\n            }\n    \n    async def analyze_text_sentiment(self, text: str) -> Dict:\n        \"\"\"Analyze sentiment of a single text input.\n        \n        Args:\n            text: The text to analyze\n            \n        Returns:\n            Dictionary containing sentiment analysis results\n        \"\"\"\n        try:\n            if not text or not text.strip():\n                return {\n                    \"status\": \"error\",\n                    \"message\": \"No text provided for analysis\"\n                }\n                \n            result = self.sentiment_analyzer.analyze(text)\n            \n            return {\n                \"status\": \"success\",\n                \"text\": text,\n                \"sentiment\": {\n                    \"label\": result[\"label\"],\n                    \"score\": result[\"score\"],\n                    \"interpretation\": self._interpret_sentiment(result[\"score\"])\n                },\n                \"timestamp\": datetime.utcnow().isoformat()\n            }\n            \n        except Exception as e:\n            logger.error(f\"Error analyzing text sentiment: {str(e)}\", exc_info=True)\n            return {\n                \"status\": \"error\",\n                \"message\": f\"Failed to analyze text sentiment: {str(e)}\"\n            }\n    \n    def _interpret_sentiment(self, score: float) -> str:\n        \"\"\"Convert sentiment score to human-readable interpretation.\"\"\"\n        if score >= 0.6:\n            return \"Very Positive\"\n        elif score >= 0.2:\n            return \"Positive\"\n        elif score <= -0.6:\n            return \"Very Negative\"\n        elif score <= -0.2:\n            return \"Negative\"\n        else:\n            return \"Neutral\"\n",
      "size": 10611,
      "language": "python"
    },
    "src/agents/agent.py": {
      "content": "\"\"\"\nBase Agent class for the multi-agent system.\n\nThis module provides the foundational Agent class that all specialized agents should inherit from.\nIt includes core functionality for tool usage, state management, and execution control.\n\"\"\"\nfrom typing import Any, Dict, List, Optional, Callable, TypeVar, Generic\nfrom dataclasses import dataclass, field\nfrom enum import Enum\nimport time\nimport uuid\nimport logging\n\n# Type variable for agent state\nT = TypeVar('T')\n\nclass AgentStatus(Enum):\n    \"\"\"Possible states of an agent.\"\"\"\n    IDLE = \"idle\"\n    PROCESSING = \"processing\"\n    WAITING_FOR_TOOL = \"waiting_for_tool\"\n    COMPLETED = \"completed\"\n    FAILED = \"failed\"\n\n@dataclass\nclass ToolResult:\n    \"\"\"Container for tool execution results.\"\"\"\n    success: bool\n    output: Any\n    error: Optional[str] = None\n\nclass Tool:\n    \"\"\"Base class for agent tools.\n    \n    Attributes:\n        name: Unique identifier for the tool\n        description: Human-readable description of the tool's purpose\n        parameters: JSON schema defining the tool's input parameters\n    \"\"\"\n    def __init__(self, name: str, description: str, parameters: dict):\n        self.name = name\n        self.description = description\n        self.parameters = parameters\n    \n    def execute(self, **kwargs) -> ToolResult:\n        \"\"\"Execute the tool with the given arguments.\n        \n        Args:\n            **kwargs: Tool-specific parameters\n            \n        Returns:\n            ToolResult containing the execution result\n        \"\"\"\n        raise NotImplementedError(\"Subclasses must implement execute()\")\n\nclass Agent(Generic[T]):\n    \"\"\"Base class for all agents in the multi-agent system.\n    \n    This class provides core functionality for agent operation including tool usage,\n    state management, and execution control.\n    \"\"\"\n    \n    def __init__(self, agent_id: Optional[str] = None):\n        \"\"\"Initialize the agent with a unique ID and default state.\n        \n        Args:\n            agent_id: Optional custom ID for the agent. If not provided, a UUID will be generated.\n        \"\"\"\n        self.agent_id = agent_id or f\"agent_{uuid.uuid4().hex[:8]}\"\n        self.status = AgentStatus.IDLE\n        self.state: Optional[T] = None\n        self.tools: Dict[str, Tool] = {}\n        self.logger = logging.getLogger(f\"agent.{self.agent_id}\")\n        self.execution_start_time: Optional[float] = None\n        self.tool_call_count: int = 0\n        self.max_tool_calls: int = 10  # Safety limit to prevent infinite loops\n        \n    def register_tool(self, tool: Tool) -> None:\n        \"\"\"Register a tool with the agent.\n        \n        Args:\n            tool: The Tool instance to register\n            \n        Raises:\n            ValueError: If a tool with the same name is already registered\n        \"\"\"\n        if tool.name in self.tools:\n            raise ValueError(f\"Tool with name '{tool.name}' already registered\")\n        self.tools[tool.name] = tool\n        self.logger.debug(f\"Registered tool: {tool.name}\")\n    \n    async def execute(self, input_data: Any, context: Optional[Dict] = None) -> Any:\n        \"\"\"Execute the agent's main logic.\n        \n        This method should be implemented by subclasses to define the agent's behavior.\n        \n        Args:\n            input_data: The input data to process\n            context: Optional context dictionary for additional parameters\n            \n        Returns:\n            The result of the agent's execution\n            \n        Raises:\n            RuntimeError: If the agent is already processing another request\n        \"\"\"\n        if self.status == AgentStatus.PROCESSING:\n            raise RuntimeError(\"Agent is already processing a request\")\n            \n        self.status = AgentStatus.PROCESSING\n        self.execution_start_time = time.time()\n        self.tool_call_count = 0\n        self.logger.info(f\"Starting execution with input: {input_data}\")\n        \n        try:\n            result = await self._process(input_data, context or {})\n            self.status = AgentStatus.COMPLETED\n            return result\n        except Exception as e:\n            self.status = AgentStatus.FAILED\n            self.logger.error(f\"Execution failed: {str(e)}\", exc_info=True)\n            raise\n        finally:\n            execution_time = time.time() - (self.execution_start_time or 0)\n            self.logger.info(f\"Execution completed in {execution_time:.2f} seconds\")\n    \n    async def _process(self, input_data: Any, context: Dict) -> Any:\n        \"\"\"Internal method to process the input data.\n        \n        Subclasses should override this method to implement their specific logic.\n        \"\"\"\n        raise NotImplementedError(\"Subclasses must implement _process()\")\n    \n    async def call_tool(self, tool_name: str, **kwargs) -> Any:\n        \"\"\"Call a registered tool and return its result.\n        \n        Args:\n            tool_name: Name of the tool to call\n            **kwargs: Arguments to pass to the tool\n            \n        Returns:\n            The tool's output if successful\n            \n        Raises:\n            ValueError: If the tool is not found or the maximum call limit is reached\n            RuntimeError: If the tool execution fails\n        \"\"\"\n        if self.tool_call_count >= self.max_tool_calls:\n            raise ValueError(f\"Maximum tool call limit reached ({self.max_tool_calls})\")\n            \n        if tool_name not in self.tools:\n            raise ValueError(f\"Tool '{tool_name}' not found\")\n            \n        tool = self.tools[tool_name]\n        self.status = AgentStatus.WAITING_FOR_TOOL\n        self.tool_call_count += 1\n        \n        self.logger.debug(f\"Calling tool: {tool_name} with args: {kwargs}\")\n        try:\n            result = tool.execute(**kwargs)\n            if not result.success:\n                raise RuntimeError(f\"Tool '{tool_name}' failed: {result.error}\")\n            return result.output\n        except Exception as e:\n            self.logger.error(f\"Tool '{tool_name}' execution failed: {str(e)}\", exc_info=True)\n            raise RuntimeError(f\"Tool execution failed: {str(e)}\")\n        finally:\n            self.status = AgentStatus.PROCESSING\n\n# Example tool implementation\nclass CalculatorTool(Tool):\n    \"\"\"Example tool that performs basic arithmetic operations.\"\"\"\n    \n    def __init__(self):\n        super().__init__(\n            name=\"calculator\",\n            description=\"Performs basic arithmetic operations\",\n            parameters={\n                \"type\": \"object\",\n                \"properties\": {\n                    \"operation\": {\"type\": \"string\", \"enum\": [\"add\", \"subtract\", \"multiply\", \"divide\"]},\n                    \"a\": {\"type\": \"number\"},\n                    \"b\": {\"type\": \"number\"}\n                },\n                \"required\": [\"operation\", \"a\", \"b\"]\n            }\n        )\n    \n    def execute(self, operation: str, a: float, b: float) -> ToolResult:\n        \"\"\"Execute a calculation.\n        \n        Args:\n            operation: The arithmetic operation to perform\n            a: First operand\n            b: Second operand\n            \n        Returns:\n            ToolResult containing the calculation result\n        \"\"\"\n        try:\n            if operation == \"add\":\n                result = a + b\n            elif operation == \"subtract\":\n                result = a - b\n            elif operation == \"multiply\":\n                result = a * b\n            elif operation == \"divide\":\n                if b == 0:\n                    return ToolResult(False, None, \"Division by zero\")\n                result = a / b\n            else:\n                return ToolResult(False, None, f\"Unknown operation: {operation}\")\n                \n            return ToolResult(True, result)\n        except Exception as e:\n            return ToolResult(False, None, str(e))\n",
      "size": 7833,
      "language": "python"
    },
    "src/agents/state.py": {
      "content": "\"\"\"\nState management for the agent framework.\n\nThis module provides utilities for managing shared state between agents,\nincluding versioning, conflict resolution, and persistence.\n\"\"\"\nfrom typing import Any, Dict, Optional, Type, TypeVar, Generic, List\nfrom dataclasses import dataclass, field, asdict\nimport json\nimport time\nimport uuid\nimport logging\nfrom datetime import datetime\nfrom enum import Enum\n\nT = TypeVar('T')\n\nclass StateOperation(Enum):\n    \"\"\"Types of state operations.\"\"\"\n    CREATE = \"create\"\n    UPDATE = \"update\"\n    DELETE = \"delete\"\n\n@dataclass\nclass StateChange:\n    \"\"\"Represents a single change to the state.\"\"\"\n    operation: StateOperation\n    path: str  # Dot-notation path to the changed value\n    old_value: Any = None\n    new_value: Any = None\n    timestamp: float = field(default_factory=time.time)\n    author: str = \"system\"\n\nclass StateManager(Generic[T]):\n    \"\"\"Manages shared state with versioning and conflict resolution.\n    \n    This class provides thread-safe state management with the following features:\n    - Atomic updates with optimistic concurrency control\n    - Change history and rollback capabilities\n    - Automatic conflict detection and resolution\n    - JSON serialization/deserialization\n    \"\"\"\n    \n    def __init__(self, initial_state: Optional[T] = None):\n        \"\"\"Initialize the state manager with an optional initial state.\n        \n        Args:\n            initial_state: Initial state object (must be JSON-serializable)\n        \"\"\"\n        self._state = initial_state or {}\n        self._version = 0\n        self._changes: List[StateChange] = []\n        self._lock = asyncio.Lock()\n        self.logger = logging.getLogger(\"state_manager\")\n    \n    @property\n    def version(self) -> int:\n        \"\"\"Get the current version number of the state.\"\"\"\n        return self._version\n    \n    @property\n    def state(self) -> T:\n        \"\"\"Get a deep copy of the current state.\"\"\"\n        return self._deep_copy(self._state)\n    \n    def get_changes_since(self, version: int) -> List[StateChange]:\n        \"\"\"Get all changes since a specific version.\n        \n        Args:\n            version: The version number to get changes since\n            \n        Returns:\n            List of StateChange objects representing the changes\n        \"\"\"\n        return self._deep_copy(self._changes[version:])\n    \n    async def update(self, \n                    updates: Dict[str, Any], \n                    author: str = \"system\",\n                    expected_version: Optional[int] = None) -> bool:\n        \"\"\"Atomically update the state.\n        \n        Args:\n            updates: Dictionary of updates to apply (dot notation for nested fields)\n            author: Identifier for who is making the change\n            expected_version: If provided, the update will fail if the current version \n                            doesn't match this value\n                            \n        Returns:\n            bool: True if the update was successful, False if there was a conflict\n        \"\"\"\n        async with self._lock:\n            if expected_version is not None and self._version != expected_version:\n                self.logger.warning(\n                    f\"Version conflict: expected {expected_version}, got {self._version}\"\n                )\n                return False\n                \n            changes = []\n            new_state = self._deep_copy(self._state)\n            \n            try:\n                for path, new_value in updates.items():\n                    old_value = self._get_by_path(new_state, path)\n                    \n                    # Skip if the value hasn't changed\n                    if old_value == new_value:\n                        continue\n                        \n                    # Record the change\n                    op = StateOperation.UPDATE\n                    if old_value is None:\n                        op = StateOperation.CREATE\n                    elif new_value is None:\n                        op = StateOperation.DELETE\n                        \n                    changes.append(StateChange(\n                        operation=op,\n                        path=path,\n                        old_value=old_value,\n                        new_value=new_value,\n                        author=author,\n                        timestamp=time.time()\n                    ))\n                    \n                    # Apply the change\n                    if new_value is None:\n                        self._delete_by_path(new_state, path)\n                    else:\n                        self._set_by_path(new_state, path, new_value)\n                \n                # If we have changes, update the state\n                if changes:\n                    self._state = new_state\n                    self._changes.extend(changes)\n                    self._version += 1\n                    self.logger.debug(f\"State updated to version {self._version}\")\n                    \n                return True\n                \n            except Exception as e:\n                self.logger.error(f\"State update failed: {str(e)}\", exc_info=True)\n                return False\n    \n    def rollback(self, version: int) -> bool:\n        \"\"\"Rollback the state to a previous version.\n        \n        Args:\n            version: The version number to rollback to\n            \n        Returns:\n            bool: True if the rollback was successful, False otherwise\n        \"\"\"\n        if version < 0 or version >= self._version:\n            return False\n            \n        # TODO: Implement rollback logic\n        self.logger.warning(f\"Rollback to version {version} not yet implemented\")\n        return False\n    \n    def to_dict(self) -> Dict[str, Any]:\n        \"\"\"Convert the state to a dictionary.\"\"\"\n        return self._deep_copy(self._state)\n    \n    def to_json(self) -> str:\n        \"\"\"Serialize the state to a JSON string.\"\"\"\n        return json.dumps(self.to_dict(), default=self._json_serializer)\n    \n    @classmethod\n    def from_json(cls, json_str: str) -> 'StateManager':\n        \"\"\"Create a new StateManager from a JSON string.\"\"\"\n        data = json.loads(json_str)\n        return cls(initial_state=data)\n    \n    @staticmethod\n    def _deep_copy(obj: Any) -> Any:\n        \"\"\"Create a deep copy of an object using JSON serialization.\"\"\"\n        return json.loads(json.dumps(obj, default=StateManager._json_serializer))\n    \n    @staticmethod\n    def _json_serializer(obj: Any) -> Any:\n        \"\"\"Custom JSON serializer for non-standard types.\"\"\"\n        if hasattr(obj, 'to_dict'):\n            return obj.to_dict()\n        elif hasattr(obj, 'isoformat'):  # Handle datetime objects\n            return obj.isoformat()\n        raise TypeError(f\"Object of type {type(obj)} is not JSON serializable\")\n    \n    @staticmethod\n    def _get_by_path(data: Dict, path: str) -> Any:\n        \"\"\"Get a value from a nested dictionary using dot notation.\"\"\"\n        keys = path.split('.')\n        value = data\n        for key in keys:\n            if key not in value:\n                return None\n            value = value[key]\n        return value\n    \n    @staticmethod\n    def _set_by_path(data: Dict, path: str, value: Any) -> None:\n        \"\"\"Set a value in a nested dictionary using dot notation.\"\"\"\n        keys = path.split('.')\n        for key in keys[:-1]:\n            if key not in data or not isinstance(data[key], dict):\n                data[key] = {}\n            data = data[key]\n        data[keys[-1]] = value\n    \n    @staticmethod\n    def _delete_by_path(data: Dict, path: str) -> None:\n        \"\"\"Delete a value from a nested dictionary using dot notation.\"\"\"\n        keys = path.split('.')\n        for key in keys[:-1]:\n            if key not in data or not isinstance(data[key], dict):\n                return  # Path doesn't exist\n            data = data[key]\n        if keys[-1] in data:\n            del data[keys[-1]]\n",
      "size": 7923,
      "language": "python"
    },
    "src/agents/standalone_sentiment_agent.py": {
      "content": "\"\"\"\nStandalone Social Sentiment Agent\n\nThis is a simplified version that doesn't depend on social_proved.py\n\"\"\"\n\nimport logging\nfrom typing import Dict, Any, Optional\nfrom datetime import datetime\nfrom src.agents.agent import Agent\nfrom src.agents.tools import ToolManager\n\n# Configure logging\nlogger = logging.getLogger(__name__)\n\nclass StandaloneSentimentAgent(Agent):\n    \"\"\"A simplified sentiment analysis agent that doesn't depend on Twitter API.\"\"\"\n    \n    def __init__(self, agent_id: str = None, config: Optional[Dict] = None):\n        super().__init__(agent_id=agent_id or \"standalone_sentiment_agent\")\n        self.config = config or {}\n        self.tool_manager = ToolManager()\n        \n        # Register tools\n        self._register_tools()\n    \n    def _register_tools(self) -> None:\n        \"\"\"Register available tools.\"\"\"\n        self.tool_manager.register_function(\n            self.analyze_text_sentiment,\n            name=\"analyze_text_sentiment\",\n            description=\"Analyze sentiment of text (positive/negative/neutral)\"\n        )\n    \n    async def _process(self, input_data: Dict[str, Any], context: Dict) -> Dict:\n        \"\"\"Process incoming requests.\"\"\"\n        text = input_data.get(\"text\", \"\")\n        return await self.analyze_text_sentiment(text)\n    \n    async def analyze_text_sentiment(self, text: str) -> Dict:\n        \"\"\"Simple sentiment analysis without external dependencies.\"\"\"\n        if not text or not text.strip():\n            return {\n                \"status\": \"error\",\n                \"message\": \"No text provided for analysis\"\n            }\n        \n        # Simple rule-based sentiment analysis\n        text_lower = text.lower()\n        positive_words = {\"good\", \"great\", \"excellent\", \"awesome\", \"love\", \"like\", \"amazing\"}\n        negative_words = {\"bad\", \"terrible\", \"awful\", \"hate\", \"dislike\", \"worst\"}\n        \n        positive_count = sum(1 for word in text_lower.split() if word in positive_words)\n        negative_count = sum(1 for word in text_lower.split() if word in negative_words)\n        \n        if positive_count > negative_count:\n            sentiment = \"POSITIVE\"\n            score = min(1.0, 0.5 + (positive_count * 0.1))\n        elif negative_count > positive_count:\n            sentiment = \"NEGATIVE\"\n            score = max(-1.0, -0.5 - (negative_count * 0.1))\n        else:\n            sentiment = \"NEUTRAL\"\n            score = 0.0\n            \n        return {\n            \"status\": \"success\",\n            \"text\": text,\n            \"sentiment\": {\n                \"label\": sentiment,\n                \"score\": score,\n                \"interpretation\": self._interpret_sentiment(score)\n            },\n            \"timestamp\": datetime.utcnow().isoformat()\n        }\n    \n    def _interpret_sentiment(self, score: float) -> str:\n        \"\"\"Convert sentiment score to human-readable interpretation.\"\"\"\n        if score >= 0.6:\n            return \"Very Positive\"\n        elif score >= 0.2:\n            return \"Positive\"\n        elif score <= -0.6:\n            return \"Very Negative\"\n        elif score <= -0.2:\n            return \"Negative\"\n        else:\n            return \"Neutral\"",
      "size": 3150,
      "language": "python"
    },
    "src/utils/tools.py": {
      "content": "\"\"\"\nTool management for the agent framework.\n\nThis module provides functionality for managing tools that agents can use,\nincluding registration, validation, and execution.\n\"\"\"\nfrom typing import Any, Dict, List, Optional, Callable, Type, Union, get_type_hints\nfrom dataclasses import dataclass, field\nimport inspect\nimport json\nimport logging\nfrom enum import Enum\nfrom pydantic import BaseModel, create_model, ValidationError\n\nclass ToolType(Enum):\n    \"\"\"Types of tools that can be registered.\"\"\"\n    FUNCTION = \"function\"\n    CLASS = \"class\"\n    HTTP = \"http\"\n\n@dataclass\nclass ToolDefinition:\n    \"\"\"Definition of a tool that can be used by agents.\"\"\"\n    name: str\n    description: str\n    parameters: Dict[str, Any]\n    execute: Callable\n    tool_type: ToolType = ToolType.FUNCTION\n    \n    def to_dict(self) -> Dict[str, Any]:\n        \"\"\"Convert the tool definition to a dictionary.\"\"\"\n        return {\n            \"name\": self.name,\n            \"description\": self.description,\n            \"parameters\": self.parameters,\n            \"type\": self.tool_type.value\n        }\n\nclass ToolExecutionError(Exception):\n    \"\"\"Exception raised when a tool execution fails.\"\"\"\n    def __init__(self, tool_name: str, message: str):\n        self.tool_name = tool_name\n        self.message = message\n        super().__init__(f\"Tool '{tool_name}' execution failed: {message}\")\n\nclass ToolManager:\n    \"\"\"Manages the registration and execution of tools.\"\"\"\n    \n    def __init__(self):\n        \"\"\"Initialize the tool manager with an empty registry.\"\"\"\n        self._tools: Dict[str, ToolDefinition] = {}\n        self.logger = logging.getLogger(\"tool_manager\")\n    \n    def register_function(self, \n                        func: Callable, \n                        name: Optional[str] = None,\n                        description: Optional[str] = None) -> None:\n        \"\"\"Register a function as a tool.\n        \n        Args:\n            func: The function to register\n            name: Optional name for the tool (defaults to function name)\n            description: Optional description of the tool (defaults to function docstring)\n            \n        Raises:\n            ValueError: If a tool with the same name is already registered\n        \"\"\"\n        tool_name = name or func.__name__\n        if tool_name in self._tools:\n            raise ValueError(f\"Tool with name '{tool_name}' already registered\")\n        \n        # Get parameter schema from type hints\n        sig = inspect.signature(func)\n        parameters = {\n            \"type\": \"object\",\n            \"properties\": {},\n            \"required\": []\n        }\n        \n        # Parse function parameters\n        for param_name, param in sig.parameters.items():\n            if param_name == 'self':\n                continue\n                \n            param_info = {\"type\": \"string\"}  # Default type\n            \n            # Get type from annotation\n            if param.annotation != inspect.Parameter.empty:\n                type_name = self._get_type_name(param.annotation)\n                if type_name:\n                    param_info[\"type\"] = type_name\n            \n            # Handle default values\n            if param.default != inspect.Parameter.empty:\n                param_info[\"default\"] = param.default\n            else:\n                parameters[\"required\"].append(param_name)\n                \n            parameters[\"properties\"][param_name] = param_info\n        \n        # Get description from docstring if not provided\n        if not description and func.__doc__:\n            description = func.__doc__.strip().split('\\n')[0]\n        \n        self._tools[tool_name] = ToolDefinition(\n            name=tool_name,\n            description=description or \"\",\n            parameters=parameters,\n            execute=func,\n            tool_type=ToolType.FUNCTION\n        )\n        self.logger.debug(f\"Registered function tool: {tool_name}\")\n    \n    def register_class(self, \n                      cls: Type, \n                      name: Optional[str] = None,\n                      description: Optional[str] = None) -> None:\n        \"\"\"Register a class as a tool.\n        \n        The class should have an `execute` method that will be called when the tool is used.\n        \n        Args:\n            cls: The class to register\n            name: Optional name for the tool (defaults to class name)\n            description: Optional description of the tool (defaults to class docstring)\n            \n        Raises:\n            ValueError: If the class doesn't have an execute method or if a tool with the same name exists\n        \"\"\"\n        tool_name = name or cls.__name__\n        if tool_name in self._tools:\n            raise ValueError(f\"Tool with name '{tool_name}' already registered\")\n            \n        if not hasattr(cls, 'execute') or not callable(getattr(cls, 'execute')):\n            raise ValueError(\"Class must have an 'execute' method\")\n        \n        # Create a model for the class parameters\n        if hasattr(cls, 'Config') and hasattr(cls.Config, 'schema_extra'):\n            # Use Pydantic model schema if available\n            parameters = cls.Config.schema_extra.get('parameters', {\n                \"type\": \"object\",\n                \"properties\": {},\n                \"required\": []\n            })\n        else:\n            # Infer parameters from __init__ signature\n            sig = inspect.signature(cls.__init__)\n            parameters = {\n                \"type\": \"object\",\n                \"properties\": {},\n                \"required\": []\n            }\n            \n            for param_name, param in sig.parameters.items():\n                if param_name == 'self':\n                    continue\n                    \n                param_info = {\"type\": \"string\"}  # Default type\n                \n                # Get type from annotation\n                if param.annotation != inspect.Parameter.empty:\n                    type_name = self._get_type_name(param.annotation)\n                    if type_name:\n                        param_info[\"type\"] = type_name\n                \n                # Handle default values\n                if param.default != inspect.Parameter.empty:\n                    param_info[\"default\"] = param.default\n                else:\n                    parameters[\"required\"].append(param_name)\n                    \n                parameters[\"properties\"][param_name] = param_info\n        \n        # Get description from docstring if not provided\n        if not description and cls.__doc__:\n            description = cls.__doc__.strip().split('\\n')[0]\n        \n        self._tools[tool_name] = ToolDefinition(\n            name=tool_name,\n            description=description or \"\",\n            parameters=parameters,\n            execute=cls,\n            tool_type=ToolType.CLASS\n        )\n        self.logger.debug(f\"Registered class tool: {tool_name}\")\n    \n    async def execute(self, tool_name: str, **kwargs) -> Any:\n        \"\"\"Execute a tool with the given arguments.\n        \n        Args:\n            tool_name: Name of the tool to execute\n            **kwargs: Arguments to pass to the tool\n            \n        Returns:\n            The result of the tool execution\n            \n        Raises:\n            ToolExecutionError: If the tool execution fails\n            ValueError: If the tool is not found\n        \"\"\"\n        if tool_name not in self._tools:\n            raise ValueError(f\"Tool not found: {tool_name}\")\n            \n        tool = self._tools[tool_name]\n        self.logger.debug(f\"Executing tool: {tool_name} with args: {kwargs}\")\n        \n        try:\n            # Validate input parameters\n            self._validate_parameters(tool.parameters, kwargs)\n            \n            # Execute the tool\n            if tool.tool_type == ToolType.CLASS:\n                # For class tools, instantiate the class and call execute\n                instance = tool.execute(**kwargs)\n                if inspect.iscoroutinefunction(instance.execute):\n                    result = await instance.execute()\n                else:\n                    result = instance.execute()\n            else:\n                # For function tools, just call the function\n                if inspect.iscoroutinefunction(tool.execute):\n                    result = await tool.execute(**kwargs)\n                else:\n                    result = tool.execute(**kwargs)\n                    \n            return result\n            \n        except ValidationError as e:\n            error_msg = f\"Invalid parameters for tool '{tool_name}': {str(e)}\"\n            self.logger.error(error_msg)\n            raise ToolExecutionError(tool_name, error_msg) from e\n            \n        except Exception as e:\n            error_msg = f\"Error executing tool '{tool_name}': {str(e)}\"\n            self.logger.error(error_msg, exc_info=True)\n            raise ToolExecutionError(tool_name, error_msg) from e\n    \n    def get_tool_schema(self, tool_name: str) -> Dict[str, Any]:\n        \"\"\"Get the JSON schema for a tool.\n        \n        Args:\n            tool_name: Name of the tool\n            \n        Returns:\n            The tool's JSON schema\n            \n        Raises:\n            ValueError: If the tool is not found\n        \"\"\"\n        if tool_name not in self._tools:\n            raise ValueError(f\"Tool not found: {tool_name}\")\n        return self._tools[tool_name].to_dict()\n    \n    def list_tools(self) -> List[Dict[str, Any]]:\n        \"\"\"Get a list of all registered tools.\"\"\"\n        return [tool.to_dict() for tool in self._tools.values()]\n    \n    def _validate_parameters(self, schema: Dict[str, Any], params: Dict[str, Any]) -> None:\n        \"\"\"Validate parameters against a JSON schema.\n        \n        Args:\n            schema: The JSON schema to validate against\n            params: The parameters to validate\n            \n        Raises:\n            ValidationError: If the parameters don't match the schema\n        \"\"\"\n        # Create a Pydantic model from the schema\n        properties = schema.get(\"properties\", {})\n        required = schema.get(\"required\", [])\n        \n        # Convert schema to Pydantic model\n        fields = {}\n        for name, prop in properties.items():\n            field_type = self._get_python_type(prop.get(\"type\", \"string\"))\n            default = prop.get(\"default\", ... if name in required else None)\n            fields[name] = (field_type, default)\n        \n        model = create_model('Params', **fields)\n        \n        # Validate the parameters\n        model(**params)\n    \n    @staticmethod\n    def _get_type_name(python_type: Type) -> Optional[str]:\n        \"\"\"Convert a Python type to a JSON schema type name.\"\"\"\n        type_map = {\n            str: \"string\",\n            int: \"integer\",\n            float: \"number\",\n            bool: \"boolean\",\n            list: \"array\",\n            dict: \"object\"\n        }\n        \n        # Handle Optional and Union types\n        if hasattr(python_type, \"__origin__\"):\n            if python_type.__origin__ == Union:\n                # For Optional[Type], which is Union[Type, None]\n                args = [arg for arg in python_type.__args__ if arg is not type(None)]\n                if len(args) == 1:\n                    return ToolManager._get_type_name(args[0])\n            return None\n            \n        return type_map.get(python_type, \"string\")\n    \n    @staticmethod\n    def _get_python_type(type_name: str) -> Type:\n        \"\"\"Convert a JSON schema type name to a Python type.\"\"\"\n        type_map = {\n            \"string\": str,\n            \"integer\": int,\n            \"number\": float,\n            \"boolean\": bool,\n            \"array\": list,\n            \"object\": dict\n        }\n        return type_map.get(type_name, str)\n\n# Example tool decorator\ndef tool(name: Optional[str] = None, description: Optional[str] = None):\n    \"\"\"Decorator to register a function as a tool.\n    \n    Example:\n        @tool(name=\"add_numbers\", description=\"Add two numbers together\")\n        def add(a: int, b: int) -> int:\n            return a + b\n    \"\"\"\n    def decorator(func):\n        # The actual registration happens when the tool manager is created\n        func._is_tool = True\n        func._tool_name = name or func.__name__\n        func._tool_description = description or func.__doc__ or \"\"\n        return func\n    return decorator\n",
      "size": 12347,
      "language": "python"
    },
    "src/utils/orchestrator.py": {
      "content": "from typing import Dict, List, Optional, Any, Callable, Type, TypeVar, Generic\nimport asyncio\nimport logging\nimport time\nfrom dataclasses import dataclass, field\nfrom src.agents.agent import Agent, AgentStatus\n\n\"\"\"\nOrchestrator for managing multiple agents and their interactions.\n\nThis module provides the Orchestrator class that coordinates the execution of\nmultiple agents, handles communication between them, and manages shared state.\n\"\"\"\n\n\nT = TypeVar('T')\n\n@dataclass\nclass AgentConfig:\n    \n    \"\"\" Configuration for an agent in the orchestrator. \"\"\"\n    agent_class: Type[Agent]\n    config: Dict[str, Any] = field(default_factory=dict)\n    dependencies: List[str] = field(default_factory=list)\n\nclass Orchestrator(Generic[T]):\n    \"\"\"Manages a collection of agents and coordinates their execution.\"\"\"\n    \n    def __init__(self):\n        \"\"\"Initialize the orchestrator with empty agent registry and state.\"\"\"\n        self.agents: Dict[str, Agent] = {}\n        self.agent_configs: Dict[str, AgentConfig] = {}\n        self.state: T = None  # Shared state for all agents\n        self.logger = logging.getLogger(\"orchestrator\")\n        self._execution_lock = asyncio.Lock()\n        \n    def register_agent(self, \n                      agent_id: str, \n                      agent_class: Type[Agent], \n                      config: Optional[Dict] = None,\n                      dependencies: Optional[List[str]] = None) -> None:\n        \"\"\"Register an agent with the orchestrator.\n        \n        Args:\n            agent_id: Unique identifier for the agent\n            agent_class: The Agent class to instantiate\n            config: Configuration dictionary for the agent\n            dependencies: List of agent IDs this agent depends on\n            \n        Raises:\n            ValueError: If an agent with the same ID is already registered\n        \"\"\"\n        if agent_id in self.agent_configs:\n            raise ValueError(f\"Agent with ID '{agent_id}' already registered\")\n            \n        self.agent_configs[agent_id] = AgentConfig(\n            agent_class=agent_class,\n            config=config or {},\n            dependencies=dependencies or []\n        )\n        self.logger.debug(f\"Registered agent: {agent_id}\")\n    \n    async def initialize(self) -> None:\n        \"\"\"Initialize all registered agents and their dependencies.\"\"\"\n        self.logger.info(\"Initializing orchestrator...\")\n        \n        # Create agent instances in dependency order\n        initialized = set()\n        \n        def get_agent(agent_id: str) -> Agent:\n            \"\"\"Get or create an agent instance, initializing dependencies first.\"\"\"\n            if agent_id in self.agents:\n                return self.agents[agent_id]\n                \n            if agent_id not in self.agent_configs:\n                raise ValueError(f\"No configuration found for agent: {agent_id}\")\n                \n            # Initialize dependencies first\n            config = self.agent_configs[agent_id]\n            for dep_id in config.dependencies:\n                if dep_id not in initialized:\n                    get_agent(dep_id)\n            \n            # Create the agent instance\n            agent = config.agent_class(agent_id=agent_id, **config.config)\n            self.agents[agent_id] = agent\n            initialized.add(agent_id)\n            self.logger.info(f\"Initialized agent: {agent_id}\")\n            return agent\n        \n        # Initialize all agents\n        for agent_id in self.agent_configs:\n            if agent_id not in initialized:\n                get_agent(agent_id)\n        \n        self.logger.info(f\"Orchestrator initialized with {len(self.agents)} agents\")\n    \n    async def execute_workflow(self, \n                             start_agent_id: str, \n                             input_data: Any,\n                             context: Optional[Dict] = None) -> Any:\n        \"\"\"Execute a workflow starting from the specified agent.\n        \n        Args:\n            start_agent_id: ID of the agent to start execution with\n            input_data: Input data for the workflow\n            context: Optional context dictionary\n            \n        Returns:\n            The result of the workflow execution\n            \n        Raises:\n            ValueError: If the start agent is not found\n            RuntimeError: If there's an error during execution\n        \"\"\"\n        if start_agent_id not in self.agents:\n            raise ValueError(f\"Agent not found: {start_agent_id}\")\n            \n        async with self._execution_lock:\n            self.logger.info(f\"Starting workflow from agent: {start_agent_id}\")\n            context = context or {}\n            context['orchestrator'] = self\n            context['start_time'] = time.time()\n            \n            try:\n                result = await self._execute_agent(start_agent_id, input_data, context)\n                self.logger.info(\"Workflow completed successfully\")\n                return result\n            except Exception as e:\n                self.logger.error(f\"Workflow failed: {str(e)}\", exc_info=True)\n                raise RuntimeError(f\"Workflow execution failed: {str(e)}\")\n    \n    async def _execute_agent(self, \n                           agent_id: str, \n                           input_data: Any,\n                           context: Dict) -> Any:\n        \"\"\"Execute an agent and handle its output.\n        \n        This method handles the execution of a single agent and can be extended\n        to add instrumentation, monitoring, or other cross-cutting concerns.\n        \"\"\"\n        agent = self.agents[agent_id]\n        self.logger.debug(f\"Executing agent: {agent_id} with input: {input_data}\")\n        \n        try:\n            # Pass the shared state to the agent\n            context['shared_state'] = self.state\n            \n            # Execute the agent\n            result = await agent.execute(input_data, context)\n            \n            # Update the shared state if the agent modified it\n            if 'shared_state' in context:\n                self.state = context['shared_state']\n                \n            return result\n            \n        except Exception as e:\n            self.logger.error(f\"Agent '{agent_id}' execution failed: {str(e)}\", exc_info=True)\n            raise\n    \n    def get_agent_status(self, agent_id: str) -> AgentStatus:\n        \"\"\"Get the current status of an agent.\n        \n        Args:\n            agent_id: ID of the agent\n            \n        Returns:\n            The current status of the agent\n            \n        Raises:\n            ValueError: If the agent is not found\n        \"\"\"\n        if agent_id not in self.agents:\n            raise ValueError(f\"Agent not found: {agent_id}\")\n        return self.agents[agent_id].status\n    \n    def get_agent(self, agent_id: str) -> Agent:\n        \"\"\"Get an agent instance by ID.\n        \n        Args:\n            agent_id: ID of the agent to retrieve\n            \n        Returns:\n            The agent instance\n            \n        Raises:\n            ValueError: If the agent is not found\n        \"\"\"\n        if agent_id not in self.agents:\n            raise ValueError(f\"Agent not found: {agent_id}\")\n        return self.agents[agent_id]\n",
      "size": 7213,
      "language": "python"
    },
    "src/utils/exp.py": {
      "content": "\"\"\"\nExample usage of StandaloneSentimentAgent\n\"\"\"\n\nimport asyncio\nfrom src.agents.standalone_sentiment_agent import StandaloneSentimentAgent\nfrom src.agents.orchestrator import Orchestrator\n\nasync def main():\n    # Create the orchestrator\n    orchestrator = Orchestrator()\n    \n    # Register the StandaloneSentimentAgent\n    orchestrator.register_agent(\n        \"sentiment_agent\",\n        StandaloneSentimentAgent\n    )\n    \n    # Initialize the orchestrator\n    await orchestrator.initialize()\n    \n    # Example: Analyze text sentiment\n    print(\"\\n--- Example: Analyze Text Sentiment ---\")\n    test_texts = [\n        \"I love this new AI framework! It's amazing how easy it is to use.\",\n        \"I hate when things don't work as expected. This is terrible!\",\n        \"The weather is neither good nor bad today.\"\n    ]\n    \n    for text in test_texts:\n        result = await orchestrator.execute_workflow(\n            \"sentiment_agent\",\n            {\"text\": text}\n        )\n        \n        print(f\"\\nText: {text}\")\n        print(f\"Sentiment: {result['sentiment']['label']} (Score: {result['sentiment']['score']:.2f})\")\n        print(f\"Interpretation: {result['sentiment']['interpretation']}\")\n\nif __name__ == \"__main__\":\n    asyncio.run(main())",
      "size": 1247,
      "language": "python"
    },
    "src/social/social.py": {
      "content": "\"\"\"\nSocial Sentiment Crawler - Twitter Edition\nCrawls: Trending hashtags + Top tweets + News sentiment\nSaves to local JSON file\n\"\"\"\n\nimport requests\nimport json\nfrom datetime import datetime, timedelta\nfrom typing import List, Dict, Optional\nimport os\nimport time\n\nclass TwitterSentimentCrawler:\n    def __init__(self, bearer_token: str):\n        \"\"\"\n        Initialize Twitter API v2 client\n        Get token from: https://developer.twitter.com/en/portal/dashboard\n        \"\"\"\n        self.bearer_token = bearer_token\n        self.base_url = \"https://api.twitter.com/2\"\n        self.headers = {\n            \"Authorization\": f\"Bearer {self.bearer_token}\"\n        }\n        \n    def get_trending_topics(self, woeid: int = 1) -> List[Dict]:\n        \"\"\"\n        Get Twitter trending topics\n        WOEID: 1=Global, 2459115=New York, 23424977=USA\n        \"\"\"\n        url = \"https://api.twitter.com/1.1/trends/place.json\"\n        params = {\"id\": woeid}\n        \n        try:\n            response = requests.get(url, headers=self.headers, params=params)\n            \n            if response.status_code == 200:\n                data = response.json()\n                \n                if data and len(data) > 0:\n                    trends = []\n                    for trend in data[0].get('trends', [])[:20]:  # Top 20\n                        trends.append({\n                            'name': trend['name'],\n                            'url': trend.get('url', ''),\n                            'tweet_volume': trend.get('tweet_volume', 0) or 0,\n                            'is_hashtag': trend['name'].startswith('#')\n                        })\n                    return trends\n            else:\n                print(f\"Error getting trends: {response.status_code} - {response.text}\")\n                \n        except Exception as e:\n            print(f\"Error fetching trending topics: {e}\")\n        \n        return []\n    \n    def search_tweets_by_hashtag(self, hashtag: str, max_results: int = 100) -> List[Dict]:\n        \"\"\"\n        Search recent tweets for a specific hashtag\n        \"\"\"\n        url = f\"{self.base_url}/tweets/search/recent\"\n        \n        # Clean hashtag\n        tag = hashtag.replace('#', '')\n        \n        params = {\n            \"query\": f\"#{tag} -is:retweet lang:en\",\n            \"max_results\": max_results,\n            \"tweet.fields\": \"public_metrics,created_at,author_id,entities\",\n            \"expansions\": \"author_id\",\n            \"user.fields\": \"username,verified,public_metrics\"\n        }\n        \n        try:\n            response = requests.get(url, headers=self.headers, params=params)\n            \n            if response.status_code == 200:\n                data = response.json()\n                \n                tweets = []\n                users_map = {}\n                \n                # Build user lookup map\n                if 'includes' in data and 'users' in data['includes']:\n                    for user in data['includes']['users']:\n                        users_map[user['id']] = user\n                \n                # Parse tweets\n                if 'data' in data:\n                    for tweet in data['data']:\n                        author = users_map.get(tweet['author_id'], {})\n                        \n                        tweets.append({\n                            'text': tweet['text'],\n                            'created_at': tweet['created_at'],\n                            'author': author.get('username', 'unknown'),\n                            'author_verified': author.get('verified', False),\n                            'author_followers': author.get('public_metrics', {}).get('followers_count', 0),\n                            'likes': tweet['public_metrics']['like_count'],\n                            'retweets': tweet['public_metrics']['retweet_count'],\n                            'replies': tweet['public_metrics']['reply_count'],\n                            'engagement_score': (\n                                tweet['public_metrics']['like_count'] * 1 +\n                                tweet['public_metrics']['retweet_count'] * 2 +\n                                tweet['public_metrics']['reply_count'] * 1.5\n                            )\n                        })\n                \n                return tweets\n            else:\n                print(f\"Error searching tweets: {response.status_code} - {response.text}\")\n                \n        except Exception as e:\n            print(f\"Error searching tweets for {hashtag}: {e}\")\n        \n        return []\n    \n    def get_top_tweet(self, tweets: List[Dict]) -> Optional[Dict]:\n        \"\"\"\n        Get the top tweet by engagement score\n        \"\"\"\n        if not tweets:\n            return None\n        \n        # Sort by engagement score\n        sorted_tweets = sorted(tweets, key=lambda x: x['engagement_score'], reverse=True)\n        \n        return sorted_tweets[0]\n    \n    def classify_sentiment(self, text: str) -> str:\n        \"\"\"\n        Simple sentiment classification based on keywords\n        \"\"\"\n        text_lower = text.lower()\n        \n        # Positive indicators\n        positive_words = ['great', 'awesome', 'amazing', 'love', 'excellent', 'best', 'wow', \n                         'incredible', 'fantastic', 'perfect', '🔥', '❤️', '😍', '🎉', '✨']\n        \n        # Negative indicators\n        negative_words = ['bad', 'terrible', 'awful', 'hate', 'worst', 'disappointing',\n                         'useless', 'broken', 'failed', 'disappointed', '😡', '😤', '💔']\n        \n        # Neutral/skeptical indicators\n        skeptical_words = ['maybe', 'doubt', 'skeptical', 'unsure', 'questionable',\n                          'allegedly', 'supposedly', 'claims', 'hmm', '🤔']\n        \n        positive_count = sum(1 for word in positive_words if word in text_lower)\n        negative_count = sum(1 for word in negative_words if word in text_lower)\n        skeptical_count = sum(1 for word in skeptical_words if word in text_lower)\n        \n        if positive_count > negative_count and positive_count > skeptical_count:\n            return 'Positive'\n        elif negative_count > positive_count:\n            return 'Negative'\n        elif skeptical_count > 0:\n            return 'Skeptical'\n        else:\n            return 'Neutral'\n    \n    def search_ai_news(self, keywords: List[str] = None) -> List[Dict]:\n        \"\"\"\n        Search for AI-related news tweets\n        \"\"\"\n        if keywords is None:\n            keywords = ['AI', 'OpenAI', 'ChatGPT', 'Claude', 'Gemini', 'artificial intelligence']\n        \n        url = f\"{self.base_url}/tweets/search/recent\"\n        \n        all_news = []\n        \n        for keyword in keywords:\n            params = {\n                \"query\": f\"{keyword} -is:retweet lang:en\",\n                \"max_results\": 20,\n                \"tweet.fields\": \"public_metrics,created_at,author_id\",\n                \"expansions\": \"author_id\",\n                \"user.fields\": \"username,verified,public_metrics\"\n            }\n            \n            try:\n                response = requests.get(url, headers=self.headers, params=params)\n                \n                if response.status_code == 200:\n                    data = response.json()\n                    \n                    users_map = {}\n                    if 'includes' in data and 'users' in data['includes']:\n                        for user in data['includes']['users']:\n                            users_map[user['id']] = user\n                    \n                    if 'data' in data:\n                        for tweet in data['data']:\n                            author = users_map.get(tweet['author_id'], {})\n                            \n                            # Filter for news-like content (verified users or high engagement)\n                            if author.get('verified') or tweet['public_metrics']['like_count'] > 50:\n                                all_news.append({\n                                    'keyword': keyword,\n                                    'text': tweet['text'],\n                                    'author': author.get('username', 'unknown'),\n                                    'verified': author.get('verified', False),\n                                    'created_at': tweet['created_at'],\n                                    'likes': tweet['public_metrics']['like_count'],\n                                    'retweets': tweet['public_metrics']['retweet_count']\n                                })\n                \n                time.sleep(1)  # Rate limiting\n                \n            except Exception as e:\n                print(f\"Error searching news for {keyword}: {e}\")\n        \n        # Sort by engagement\n        all_news.sort(key=lambda x: x['likes'] + x['retweets'] * 2, reverse=True)\n        \n        return all_news[:10]  # Top 10 news tweets\n    \n    def crawl_all(self) -> Dict:\n        \"\"\"\n        Main crawl function - gets everything\n        \"\"\"\n        print(f\"Starting Twitter sentiment crawl at {datetime.now()}\")\n        \n        # Step 1: Get trending topics\n        print(\"\\n[1/3] Fetching trending topics...\")\n        trending = self.get_trending_topics()\n        print(f\"  Found {len(trending)} trending topics\")\n        \n        # Step 2: For each trending hashtag, get top tweet + sentiment\n        print(\"\\n[2/3] Analyzing trending hashtags...\")\n        hashtag_analysis = []\n        \n        for trend in trending[:10]:  # Top 10 trends only\n            if trend['is_hashtag']:\n                print(f\"  Processing {trend['name']}...\")\n                \n                # Search tweets for this hashtag\n                tweets = self.search_tweets_by_hashtag(trend['name'], max_results=50)\n                \n                if tweets:\n                    # Get top tweet\n                    top_tweet = self.get_top_tweet(tweets)\n                    \n                    # Analyze sentiment across all tweets\n                    sentiments = [self.classify_sentiment(t['text']) for t in tweets[:10]]\n                    sentiment_counts = {\n                        'Positive': sentiments.count('Positive'),\n                        'Negative': sentiments.count('Negative'),\n                        'Neutral': sentiments.count('Neutral'),\n                        'Skeptical': sentiments.count('Skeptical')\n                    }\n                    dominant_sentiment = max(sentiment_counts, key=sentiment_counts.get)\n                    \n                    hashtag_analysis.append({\n                        'hashtag': trend['name'],\n                        'tweet_volume': trend['tweet_volume'],\n                        'top_tweet': {\n                            'text': top_tweet['text'],\n                            'author': f\"@{top_tweet['author']}\",\n                            'verified': top_tweet['author_verified'],\n                            'likes': top_tweet['likes'],\n                            'retweets': top_tweet['retweets'],\n                            'engagement_score': int(top_tweet['engagement_score'])\n                        },\n                        'sentiment_breakdown': sentiment_counts,\n                        'dominant_sentiment': dominant_sentiment,\n                        'total_tweets_analyzed': len(tweets)\n                    })\n                \n                time.sleep(2)  # Rate limiting\n        \n        # Step 3: Get AI news\n        print(\"\\n[3/3] Fetching AI news...\")\n        ai_news = self.search_ai_news()\n        print(f\"  Found {len(ai_news)} AI news tweets\")\n        \n        # Build final output\n        output = {\n            'date': datetime.now().strftime('%Y-%m-%d'),\n            'generated_at': datetime.now().isoformat(),\n            'data_source': 'Twitter API v2',\n            'trending_topics': {\n                'total': len(trending),\n                'list': trending\n            },\n            'hashtag_sentiment_analysis': hashtag_analysis,\n            'ai_news': ai_news,\n            'summary': {\n                'total_trends': len(trending),\n                'hashtags_analyzed': len(hashtag_analysis),\n                'news_tweets': len(ai_news),\n                'overall_sentiment': self._calculate_overall_sentiment(hashtag_analysis)\n            }\n        }\n        \n        return output\n    \n    def _calculate_overall_sentiment(self, analysis: List[Dict]) -> str:\n        \"\"\"\n        Calculate overall sentiment from all hashtags\n        \"\"\"\n        if not analysis:\n            return 'Neutral'\n        \n        all_sentiments = [item['dominant_sentiment'] for item in analysis]\n        \n        sentiment_score = {\n            'Positive': all_sentiments.count('Positive'),\n            'Negative': all_sentiments.count('Negative'),\n            'Neutral': all_sentiments.count('Neutral'),\n            'Skeptical': all_sentiments.count('Skeptical')\n        }\n        \n        return max(sentiment_score, key=sentiment_score.get)\n    \n    def save_to_file(self, data: Dict, filepath: str = 'twitter_sentiment.json'):\n        \"\"\"\n        Save sentiment data to local JSON file\n        \"\"\"\n        with open(filepath, 'w', encoding='utf-8') as f:\n            json.dump(data, f, indent=2, ensure_ascii=False)\n        \n        print(f\"\\n✓ Saved to {filepath}\")\n        \n        # Also save a readable summary\n        summary_file = filepath.replace('.json', '_summary.txt')\n        with open(summary_file, 'w', encoding='utf-8') as f:\n            f.write(f\"TWITTER SENTIMENT REPORT - {data['date']}\\n\")\n            f.write(\"=\" * 60 + \"\\n\\n\")\n            \n            f.write(f\"Overall Sentiment: {data['summary']['overall_sentiment']}\\n\")\n            f.write(f\"Trends Analyzed: {data['summary']['hashtags_analyzed']}\\n\\n\")\n            \n            f.write(\"TOP TRENDING HASHTAGS:\\n\")\n            f.write(\"-\" * 60 + \"\\n\")\n            \n            for item in data['hashtag_sentiment_analysis']:\n                f.write(f\"\\n{item['hashtag']}\\n\")\n                f.write(f\"  Volume: {item['tweet_volume']:,} tweets\\n\")\n                f.write(f\"  Sentiment: {item['dominant_sentiment']}\\n\")\n                f.write(f\"  Top Tweet: {item['top_tweet']['text'][:100]}...\\n\")\n                f.write(f\"  By: {item['top_tweet']['author']} ({item['top_tweet']['likes']:,} likes)\\n\")\n        \n        print(f\"✓ Summary saved to {summary_file}\")\n\n\ndef main():\n    \"\"\"\n    Main execution\n    \"\"\"\n    bearer_token = os.getenv('TWITTER_BEARER_TOKEN')\n    \n    if not bearer_token:\n        print(\"❌ ERROR: TWITTER_BEARER_TOKEN not set!\")\n        print(\"\\nGet your token from: https://developer.twitter.com/en/portal/dashboard\")\n        print(\"\\nThen run:\")\n        print(\"  export TWITTER_BEARER_TOKEN='your_token_here'\")\n        return\n    \n    # Initialize crawler\n    crawler = TwitterSentimentCrawler(bearer_token)\n    \n    # Crawl everything\n    sentiment_data = crawler.crawl_all()\n    \n    # Save to local files\n    crawler.save_to_file(sentiment_data, 'twitter_sentiment.json')\n    \n    # Print summary\n    print(\"\\n\" + \"=\" * 60)\n    print(\"CRAWL COMPLETE!\")\n    print(\"=\" * 60)\n    print(f\"Overall Sentiment: {sentiment_data['summary']['overall_sentiment']}\")\n    print(f\"Trending Topics: {sentiment_data['summary']['total_trends']}\")\n    print(f\"Hashtags Analyzed: {sentiment_data['summary']['hashtags_analyzed']}\")\n    print(f\"AI News: {sentiment_data['summary']['news_tweets']}\")\n    print(\"\\nFiles saved:\")\n    print(\"  - twitter_sentiment.json (Full data)\")\n    print(\"  - twitter_sentiment_summary.txt (Human readable)\")\n    print(\"=\" * 60)\n\n\nif __name__ == '__main__':\n    main()",
      "size": 15562,
      "language": "python"
    },
    "src/social/Mastodon.py": {
      "content": "\"\"\"\nAI Trend Crawler - Mastodon Edition\nFetches daily trending AI topics from Mastodon for Hongjie's Coze Bot\n\nRequirements:\npip install Mastodon.py python-dotenv\n\nSetup:\n1. No app registration needed for public timeline access!\n2. Optional: Create app at your preferred instance (e.g., mastodon.social)\n   - Go to Settings > Development > New Application\n   - Copy access token to .env as MASTODON_ACCESS_TOKEN (optional)\n3. Create .env file (optional):\n   MASTODON_ACCESS_TOKEN=your_access_token_here\n   MASTODON_INSTANCE=https://mastodon.social\n\"\"\"\n\nimport json\nfrom datetime import datetime, timedelta\nfrom mastodon import Mastodon\nimport os\nfrom dotenv import load_dotenv\nimport time\n\n# Load environment variables\nload_dotenv()\n\nclass AITrendCrawler:\n    def __init__(self):\n        \"\"\"Initialize Mastodon API client\"\"\"\n        \n        # Mastodon instance to use (largest instances for AI content)\n        self.instance = os.getenv('MASTODON_INSTANCE', 'https://mastodon.social')\n        access_token = os.getenv('MASTODON_ACCESS_TOKEN')\n        \n        # Initialize Mastodon client\n        # Note: Can work without access token for public timelines!\n        if access_token:\n            self.mastodon = Mastodon(\n                access_token=access_token,\n                api_base_url=self.instance\n            )\n            print(f\"✓ Connected to {self.instance} with authentication\")\n        else:\n            self.mastodon = Mastodon(\n                api_base_url=self.instance\n            )\n            print(f\"✓ Connected to {self.instance} (public access only)\")\n        \n        # Popular Mastodon instances with AI content\n        self.instances_to_check = [\n            'https://mastodon.social',      # Largest general instance\n            'https://fosstodon.org',        # FOSS/Tech focused\n            'https://hachyderm.io',         # Tech community\n            'https://sigmoid.social',       # AI/ML specific\n        ]\n        \n        # AI-focused hashtags on Mastodon\n        self.ai_hashtags = [\n            'AI',\n            'ArtificialIntelligence',\n            'MachineLearning',\n            'DeepLearning',\n            'LLM',\n            'ChatGPT',\n            'OpenAI',\n            'Claude',\n            'GenerativeAI',\n            'AGI',\n            'NeuralNetworks',\n            'GPT4',\n            'StableDiffusion',\n            'Anthropic',\n        ]\n        \n        # AI influencers on Mastodon (format: instance, username)\n        self.ai_accounts = [\n            # Add known AI researchers/accounts here\n            # Format: ('instance.url', 'username')\n        ]\n    \n    def search_hashtag(self, hashtag, limit=20):\n        \"\"\"Search for posts with a specific hashtag\"\"\"\n        try:\n            print(f\"   🔍 Searching #{hashtag}...\")\n            \n            # Search for the hashtag\n            results = self.mastodon.timeline_hashtag(\n                hashtag,\n                limit=limit\n            )\n            \n            # Filter recent posts (last 24 hours)\n            recent_posts = []\n            now = datetime.now(results[0]['created_at'].tzinfo) if results else datetime.now()\n            cutoff_time = now - timedelta(hours=24)\n            \n            for post in results:\n                if post['created_at'] >= cutoff_time:\n                    recent_posts.append(post)\n            \n            print(f\"   ✓ #{hashtag}: Found {len(recent_posts)} recent posts\")\n            return recent_posts\n            \n        except Exception as e:\n            print(f\"   ✗ Error searching #{hashtag}: {str(e)[:100]}\")\n            return []\n    \n    def get_trending_tags(self, limit=10):\n        \"\"\"Get trending hashtags from the instance\"\"\"\n        try:\n            print(\"   🔥 Fetching trending tags...\")\n            trends = self.mastodon.trends(limit=limit)\n            \n            # Filter for AI-related trends\n            ai_trends = []\n            ai_keywords = ['ai', 'ml', 'gpt', 'llm', 'bot', 'neural', 'deep', 'learning', 'data']\n            \n            for trend in trends:\n                tag_name = trend['name'].lower()\n                if any(keyword in tag_name for keyword in ai_keywords):\n                    ai_trends.append(trend['name'])\n            \n            print(f\"   ✓ Found {len(ai_trends)} AI-related trending tags\")\n            return ai_trends\n            \n        except Exception as e:\n            print(f\"   ✗ Error fetching trends: {str(e)[:100]}\")\n            return []\n    \n    def get_public_timeline(self, limit=40):\n        \"\"\"Get posts from public timeline\"\"\"\n        try:\n            print(\"   🌍 Fetching public timeline...\")\n            \n            timeline = self.mastodon.timeline_public(limit=limit)\n            \n            # Filter for AI-related content\n            ai_posts = []\n            ai_keywords = ['ai', 'llm', 'gpt', 'chatgpt', 'claude', 'machine learning', \n                          'deep learning', 'neural', 'openai', 'anthropic', 'ml']\n            \n            for post in timeline:\n                content_lower = post['content'].lower()\n                if any(keyword in content_lower for keyword in ai_keywords):\n                    ai_posts.append(post)\n            \n            print(f\"   ✓ Found {len(ai_posts)} AI-related posts in public timeline\")\n            return ai_posts\n            \n        except Exception as e:\n            print(f\"   ✗ Error fetching public timeline: {str(e)[:100]}\")\n            return []\n    \n    def get_account_posts(self, instance_url, username, limit=10):\n        \"\"\"Get recent posts from a specific account\"\"\"\n        try:\n            print(f\"   🔍 Fetching @{username}@{instance_url}...\")\n            \n            # Search for the account\n            accounts = self.mastodon.account_search(username, limit=5)\n            \n            target_account = None\n            for account in accounts:\n                if username.lower() in account['username'].lower():\n                    target_account = account\n                    break\n            \n            if not target_account:\n                print(f\"   ✗ Account not found: {username}\")\n                return []\n            \n            # Get account's posts\n            posts = self.mastodon.account_statuses(\n                target_account['id'],\n                limit=limit\n            )\n            \n            print(f\"   ✓ @{username}: Found {len(posts)} posts\")\n            return posts\n            \n        except Exception as e:\n            print(f\"   ✗ Error fetching @{username}: {str(e)[:100]}\")\n            return []\n    \n    def calculate_engagement_score(self, post):\n        \"\"\"Calculate engagement score for ranking\"\"\"\n        # Mastodon scoring: boosts (reblogs) + favorites + replies\n        return (\n            post['reblogs_count'] * 3 +\n            post['favourites_count'] * 2 +\n            post['replies_count'] * 1\n        )\n    \n    def classify_sentiment(self, text):\n        \"\"\"Simple sentiment classification\"\"\"\n        text_lower = text.lower()\n        \n        if any(word in text_lower for word in ['scary', 'dangerous', 'terrifying', 'fear', 'worried', 'concerning']):\n            return 'Fear/Concern'\n        elif any(word in text_lower for word in ['lol', 'haha', 'funny', 'hilarious']):\n            return 'Humor'\n        elif any(word in text_lower for word in ['hype', 'scam', 'skeptical', 'doubt', 'overrated']):\n            return 'Skepticism'\n        elif any(word in text_lower for word in ['amazing', 'incredible', 'wow', 'breakthrough', 'revolutionary']):\n            return 'Excitement'\n        elif any(word in text_lower for word in ['tutorial', 'guide', 'how', 'useful', 'practical']):\n            return 'Practical/Educational'\n        elif any(word in text_lower for word in ['research', 'paper', 'study', 'arxiv']):\n            return 'Research'\n        else:\n            return 'Discussion'\n    \n    def strip_html(self, html_content):\n        \"\"\"Remove HTML tags from content\"\"\"\n        from html.parser import HTMLParser\n        \n        class MLStripper(HTMLParser):\n            def __init__(self):\n                super().__init__()\n                self.reset()\n                self.strict = False\n                self.convert_charrefs = True\n                self.text = []\n            \n            def handle_data(self, d):\n                self.text.append(d)\n            \n            def get_data(self):\n                return ''.join(self.text)\n        \n        s = MLStripper()\n        s.feed(html_content)\n        return s.get_data()\n    \n    def crawl_daily_trends(self, output_file='daily_trends_mastodon.json'):\n        \"\"\"Main crawler: fetch and process daily AI trends from Mastodon\"\"\"\n        print(f\"🕷️  Starting Mastodon AI Trend Crawl at {datetime.now()}\")\n        print(f\"📡 Instance: {self.instance}\\n\")\n        \n        all_posts = []\n        \n        # 1. Get trending AI hashtags and search them\n        print(\"\\n🔥 Fetching trending tags...\")\n        trending_tags = self.get_trending_tags(limit=10)\n        \n        # Combine trending tags with our predefined AI hashtags\n        hashtags_to_search = list(set(trending_tags + self.ai_hashtags[:8]))\n        \n        print(f\"\\n📱 Searching {len(hashtags_to_search)} hashtags...\")\n        for hashtag in hashtags_to_search:\n            posts = self.search_hashtag(hashtag, limit=10)\n            for post in posts:\n                post['source_type'] = 'hashtag'\n                post['source_name'] = f'#{hashtag}'\n                all_posts.append(post)\n            time.sleep(0.5)  # Be respectful to the API\n        \n        # 2. Get AI content from public timeline\n        print(\"\\n🌍 Searching public timeline...\")\n        public_posts = self.get_public_timeline(limit=40)\n        for post in public_posts:\n            post['source_type'] = 'public'\n            post['source_name'] = 'Public Timeline'\n            all_posts.append(post)\n        \n        # 3. Get posts from specific AI accounts (if configured)\n        if self.ai_accounts:\n            print(\"\\n👥 Fetching from AI accounts...\")\n            for instance_url, username in self.ai_accounts:\n                posts = self.get_account_posts(instance_url, username, limit=5)\n                for post in posts:\n                    post['source_type'] = 'account'\n                    post['source_name'] = f'@{username}'\n                    all_posts.append(post)\n                time.sleep(1)\n        \n        print(f\"\\n📊 Total posts collected: {len(all_posts)}\")\n        \n        if not all_posts:\n            print(\"\\n⚠️  No posts found to analyze\")\n            return {'error': 'No posts found', 'trends': []}\n        \n        # 4. Remove duplicates\n        seen_ids = set()\n        unique_posts = []\n        for post in all_posts:\n            post_id = post['id']\n            if post_id not in seen_ids:\n                seen_ids.add(post_id)\n                unique_posts.append(post)\n        \n        print(f\"📊 Unique posts after deduplication: {len(unique_posts)}\")\n        \n        # 5. Rank by engagement\n        unique_posts.sort(\n            key=lambda p: self.calculate_engagement_score(p),\n            reverse=True\n        )\n        \n        # 6. Take top 15 and format\n        top_posts = unique_posts[:15]\n        \n        trends = []\n        print(\"\\n🔍 Processing top posts...\")\n        for post in top_posts:\n            # Extract clean text from HTML content\n            content_text = self.strip_html(post['content'])\n            \n            # Get account info\n            account = post['account']\n            username = f\"@{account['username']}@{account['acct'].split('@')[-1] if '@' in account['acct'] else self.instance.replace('https://', '')}\"\n            \n            # Calculate post age\n            post_age = datetime.now(post['created_at'].tzinfo) - post['created_at']\n            age_str = f\"{int(post_age.total_seconds() / 3600)}h ago\" if post_age.total_seconds() < 86400 else f\"{int(post_age.days)}d ago\"\n            \n            # Get media info if present\n            has_media = len(post.get('media_attachments', [])) > 0\n            media_types = [m['type'] for m in post.get('media_attachments', [])]\n            \n            trend_item = {\n                'source': f\"Mastodon - {post.get('source_name', 'Fediverse')}\",\n                'title': (content_text[:97] + '...') if len(content_text) > 100 else content_text,\n                'url': post['url'],\n                'full_content': content_text,\n                'author': username,\n                'author_display_name': account.get('display_name', account['username']),\n                'posted': age_str,\n                'engagement': {\n                    'boosts': post['reblogs_count'],\n                    'favorites': post['favourites_count'],\n                    'replies': post['replies_count'],\n                },\n                'sentiment_tag': self.classify_sentiment(content_text),\n                'has_media': has_media,\n                'media_types': media_types,\n                'language': post.get('language', 'unknown'),\n                'visibility': post['visibility'],\n            }\n            trends.append(trend_item)\n        \n        # 7. Create output JSON with metadata\n        output = {\n            'date': datetime.now().strftime('%Y-%m-%d'),\n            'generated_at': datetime.now().isoformat(),\n            'instance': self.instance,\n            'total_analyzed': len(all_posts),\n            'unique_posts': len(unique_posts),\n            'hashtags_searched': len(hashtags_to_search),\n            'trends_found': len(trends),\n            'trends': trends,\n            'note': 'Data collected from Mastodon via official API. Free and open source!'\n        }\n        \n        # 8. Save to file\n        with open(output_file, 'w', encoding='utf-8') as f:\n            json.dump(output, f, indent=2, ensure_ascii=False)\n        \n        print(f\"\\n✅ Crawl complete! Generated {len(trends)} trend items\")\n        print(f\"📄 Output saved to: {output_file}\")\n        \n        return output\n\n\ndef main():\n    \"\"\"Run the crawler\"\"\"\n    try:\n        crawler = AITrendCrawler()\n        result = crawler.crawl_daily_trends()\n        \n        # Print sample results\n        print(\"\\n\" + \"=\"*60)\n        print(\"📊 SAMPLE RESULTS\")\n        print(\"=\"*60)\n        \n        if result.get('trends'):\n            print(f\"\\nFound {len(result['trends'])} trends\\n\")\n            \n            # Show first 3 trends\n            for i, trend in enumerate(result['trends'][:3], 1):\n                print(f\"\\n--- Trend #{i} ---\")\n                print(f\"Title: {trend['title']}\")\n                print(f\"Author: {trend['author']} ({trend['author_display_name']})\")\n                print(f\"Posted: {trend['posted']}\")\n                print(f\"Engagement: {trend['engagement']['favorites']} favs, {trend['engagement']['boosts']} boosts, {trend['engagement']['replies']} replies\")\n                print(f\"Sentiment: {trend['sentiment_tag']}\")\n                print(f\"URL: {trend['url']}\")\n                if trend['has_media']:\n                    print(f\"Media: {', '.join(trend['media_types'])}\")\n        else:\n            print(\"No trends found.\")\n        \n    except Exception as e:\n        print(f\"❌ Error: {e}\")\n        import traceback\n        traceback.print_exc()\n        return 1\n    \n    return 0\n\n\nif __name__ == '__main__':\n    import sys\n    sys.exit(main())",
      "size": 15366,
      "language": "python"
    },
    "src/social/social_proved.py": {
      "content": "\"\"\"\nTwitter Sentiment Crawler v2.0\n-----------------------------\nAdvanced Twitter sentiment analysis with:\n- Robust error handling and retries\n- Caching for performance\n- Better sentiment analysis with RoBERTa\n- Async support\n- Comprehensive logging\n\"\"\"\n\nimport os\nimport json\nimport time\nimport logging\nfrom datetime import datetime, timedelta\nfrom pathlib import Path\nfrom typing import List, Dict, Optional, Any, Tuple, Set\nfrom functools import lru_cache\nimport asyncio\nimport aiohttp\nfrom aiohttp import ClientSession, ClientTimeout, ClientError\n\nfrom src.config.config import settings\n\n# Configure logging\nlogging.basicConfig(\n    level=getattr(logging, settings.LOG_LEVEL),\n    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',\n    handlers=[\n        logging.FileHandler('crawler.log'),\n        logging.StreamHandler()\n    ]\n)\nlogger = logging.getLogger(__name__)\n\nclass TwitterAPIClient:\n    \"\"\"Handles Twitter API v1.1 interactions with retries and rate limiting\"\"\"\n    \n    def __init__(self, bearer_token: str):\n        self.base_url = \"https://api.twitter.com/1.1\"\n        self.bearer_token = bearer_token\n        self.headers = {\n            \"Authorization\": f\"Bearer {self.bearer_token}\",\n            \"User-Agent\": \"v2UserLookupPython\"\n        }\n        self.session = None\n        self.rate_limit_remaining = 15  # Start with default rate limit\n        self.rate_limit_reset = 0\n        \n    async def __aenter__(self):\n        self.session = aiohttp.ClientSession(\n            headers={\n                \"Authorization\": f\"Bearer {self.bearer_token}\",\n                \"User-Agent\": \"TwitterSentimentCrawler/2.0\"\n            },\n            timeout=ClientTimeout(total=settings.REQUEST_TIMEOUT)\n        )\n        return self\n        \n    async def __aexit__(self, exc_type, exc, tb):\n        if self.session:\n            await self.session.close()\n            \n    async def _handle_rate_limit(self):\n        \"\"\"Handle rate limiting with exponential backoff\"\"\"\n        now = time.time()\n        if self.rate_limit_remaining <= 1:\n            sleep_time = max(self.rate_limit_reset - now, 0) + 1\n            logger.warning(f\"Approaching rate limit. Sleeping for {sleep_time:.1f} seconds\")\n            await asyncio.sleep(sleep_time)\n            \n    async def get(self, endpoint: str, params: Optional[Dict] = None) -> Dict:\n        \"\"\"Make a GET request to the Twitter API with retries\"\"\"\n        url = f\"{self.base_url}/{endpoint.lstrip('/')}\"\n        \n        for attempt in range(settings.MAX_RETRIES + 1):\n            try:\n                await self._handle_rate_limit()\n                \n                async with self.session.get(url, params=params) as response:\n                    # Update rate limit info\n                    self.rate_limit_remaining = int(\n                        response.headers.get('x-rate-limit-remaining', 15)\n                    )\n                    self.rate_limit_reset = int(\n                        response.headers.get('x-rate-limit-reset', time.time() + 60)\n                    )\n                    \n                    if response.status == 200:\n                        return await response.json()\n                    elif response.status == 429:  # Rate limited\n                        reset_time = int(response.headers.get('x-rate-limit-reset', 0))\n                        sleep_time = max(reset_time - time.time(), 0) + 1\n                        logger.warning(f\"Rate limited. Sleeping for {sleep_time:.1f} seconds\")\n                        await asyncio.sleep(sleep_time)\n                        continue\n                    else:\n                        error_text = await response.text()\n                        logger.error(f\"API Error {response.status}: {error_text}\")\n                        response.raise_for_status()\n                        \n            except (ClientError, asyncio.TimeoutError) as e:\n                if attempt == settings.MAX_RETRIES:\n                    logger.error(f\"Request failed after {settings.MAX_RETRIES} attempts: {e}\")\n                    raise\n                backoff = 2 ** attempt  # Exponential backoff\n                await asyncio.sleep(backoff)\n                \n        return {}\n\nclass SentimentAnalyzer:\n    \"\"\"Handles sentiment analysis using pre-trained models\"\"\"\n    \n    def __init__(self):\n        self.model = None\n        self.tokenizer = None\n        self.device = \"cuda\" if self._has_gpu() else \"cpu\"\n        self._load_model()\n        \n    def _has_gpu(self) -> bool:\n        \"\"\"Check if GPU is available\"\"\"\n        try:\n            import torch\n            return torch.cuda.is_available()\n        except ImportError:\n            return False\n            \n    def _load_model(self):\n        \"\"\"Load the sentiment analysis model\"\"\"\n        try:\n            from transformers import AutoModelForSequenceClassification, AutoTokenizer\n            import torch\n            \n            logger.info(f\"Loading sentiment model: {settings.SENTIMENT_MODEL}\")\n            self.tokenizer = AutoTokenizer.from_pretrained(settings.SENTIMENT_MODEL)\n            self.model = AutoModelForSequenceClassification.from_pretrained(\n                settings.SENTIMENT_MODEL\n            ).to(self.device)\n            \n            if self.device == \"cuda\":\n                self.model = self.model.half()  # Use half precision for faster inference\n                \n            self.model.eval()  # Set to evaluation mode\n            logger.info(\"Sentiment model loaded successfully\")\n            \n        except Exception as e:\n            logger.error(f\"Failed to load sentiment model: {e}\")\n            raise\n            \n    @lru_cache(maxsize=settings.SENTIMENT_CACHE_SIZE)\n    def analyze(self, text: str) -> Dict[str, Any]:\n        \"\"\"Analyze sentiment of the given text with caching\"\"\"\n        if not text.strip():\n            return {\"label\": \"neutral\", \"score\": 0.5}\n            \n        try:\n            import torch\n            from transformers import pipeline\n            \n            # Use pipeline for simpler inference\n            sentiment_pipeline = pipeline(\n                \"sentiment-analysis\",\n                model=self.model,\n                tokenizer=self.tokenizer,\n                device=0 if self.device == \"cuda\" else -1,\n                truncation=True,\n                max_length=512\n            )\n            \n            result = sentiment_pipeline(text[:512])[0]  # Truncate to model max length\n            return {\n                \"label\": result[\"label\"].lower(),\n                \"score\": float(result[\"score\"])\n            }\n            \n        except Exception as e:\n            logger.error(f\"Sentiment analysis failed: {e}\")\n            return {\"label\": \"neutral\", \"score\": 0.5}\n\nclass TwitterSentimentCrawler:\n    \"\"\"Main crawler class with improved functionality\"\"\"\n    \n    def __init__(self):\n        self.bearer_token = settings.TWITTER_BEARER_TOKEN\n        self.sentiment_analyzer = SentimentAnalyzer()\n        self.output_dir = Path(settings.OUTPUT_DIR)\n        self.output_dir.mkdir(exist_ok=True)\n        \n    async def get_trending_topics(self, woeid: int = 1) -> List[Dict]:\n        \"\"\"Get trending topics with caching\"\"\"\n        cache_file = self.output_dir / f\"trends_{woeid}_{datetime.utcnow().strftime('%Y%m%d%H')}.json\"\n        \n        try:\n            # First, get the location ID from the WOEID\n            async with aiohttp.ClientSession(\n                headers={\"Authorization\": f\"Bearer {self.bearer_token}\", \"User-Agent\": \"TwitterSentimentCrawler/2.0\"}\n            ) as session:\n                # Get trends available locations\n                locations_url = \"https://api.twitter.com/1.1/trends/available.json\"\n                async with session.get(locations_url) as resp:\n                    if resp.status != 200:\n                        error_text = await resp.text()\n                        logger.error(f\"Error getting locations: {resp.status} - {error_text}\")\n                        return []\n                    locations = await resp.json()\n                \n                # Find the location by WOEID\n                location = next((loc for loc in locations if loc['woeid'] == woeid), None)\n                if not location:\n                    logger.warning(f\"Location with WOEID {woeid} not found\")\n                    return []\n                \n                # Get trending topics for the location\n                trends_url = f\"https://api.twitter.com/1.1/trends/place.json\"\n                params = {\"id\": woeid}\n                \n                async with session.get(trends_url, params=params) as resp:\n                    if resp.status != 200:\n                        error_text = await resp.text()\n                        logger.error(f\"Error getting trends: {resp.status} - {error_text}\")\n                        return []\n                    \n                    response = await resp.json()\n                    \n                    if not response or not isinstance(response, list) or not response[0].get('trends'):\n                        logger.warning(f\"No trending topics found for WOEID: {woeid}\")\n                        return []\n                    \n                    # Process and return trending topics\n                    trends = response[0]['trends']\n                    return [\n                        {\n                            'name': trend['name'],\n                            'url': trend.get('url', ''),\n                            'tweet_volume': trend.get('tweet_volume', 0) or 0,\n                            'is_hashtag': trend['name'].startswith('#')\n                        }\n                        for trend in trends[:20]  # Limit to top 20 trends\n                        if trend.get('name')\n                    ]\n                \n        except Exception as e:\n            logger.error(f\"Error in get_trending_topics: {str(e)}\", exc_info=True)\n            return []\n        \n            logger.error(f\"Error in get_trending_topics: {e}\")\n            \n        return []\n\n    async def search_tweets(self, query: str, max_results: int = 100) -> List[Dict]:\n        \"\"\"Search for tweets with the given query\"\"\"\n        async with TwitterAPIClient(self.bearer_token) as client:\n            params = {\n                \"query\": f\"{query} -is:retweet lang:en\",\n                \"max_results\": min(max_results, 100),  # Max 100 per request\n                \"tweet.fields\": \"public_metrics,created_at,author_id,context_annotations\",\n                \"user.fields\": \"username,verified,public_metrics\",\n                \"expansions\": \"author_id\"\n            }\n            \n            results = []\n            next_token = None\n            \n            while len(results) < max_results:\n                if next_token:\n                    params[\"next_token\"] = next_token\n                    \n                data = await client.get(\"tweets/search/recent\", params)\n                \n                if \"data\" not in data:\n                    break\n                    \n                # Process tweets\n                users = {u[\"id\"]: u for u in data.get(\"includes\", {}).get(\"users\", [])}\n                \n                for tweet in data[\"data\"]:\n                    author = users.get(tweet[\"author_id\"], {})\n                    sentiment = self.sentiment_analyzer.analyze(tweet[\"text\"])\n                    \n                    results.append({\n                        \"id\": tweet[\"id\"],\n                        \"text\": tweet[\"text\"],\n                        \"created_at\": tweet[\"created_at\"],\n                        \"author\": {\n                            \"username\": author.get(\"username\", \"unknown\"),\n                            \"verified\": author.get(\"verified\", False),\n                            \"followers\": author.get(\"public_metrics\", {}).get(\"followers_count\", 0)\n                        },\n                        \"metrics\": tweet[\"public_metrics\"],\n                        \"sentiment\": sentiment,\n                        \"url\": f\"https://twitter.com/{author.get('username', '')}/status/{tweet['id']}\"\n                    })\n                \n                # Check if there are more results\n                next_token = data.get(\"meta\", {}).get(\"next_token\")\n                if not next_token or len(results) >= max_results:\n                    break\n                    \n            return results[:max_results]\n            \n    async def analyze_hashtag(self, hashtag: str) -> Dict:\n        \"\"\"Analyze sentiment for a specific hashtag\"\"\"\n        logger.info(f\"Analyzing hashtag: {hashtag}\")\n        \n        # Get tweets for the hashtag\n        tweets = await self.search_tweets(f\"#{hashtag}\", settings.MAX_TWEETS_PER_HASHTAG)\n        \n        if not tweets:\n            return {\n                \"hashtag\": hashtag,\n                \"tweet_count\": 0,\n                \"error\": \"No tweets found\"\n            }\n            \n        # Calculate overall sentiment\n        sentiment_scores = [t[\"sentiment\"][\"score\"] for t in tweets]\n        sentiment_labels = [t[\"sentiment\"][\"label\"] for t in tweets]\n        \n        sentiment_summary = {\n            \"positive\": sum(1 for l in sentiment_labels if l == \"positive\"),\n            \"negative\": sum(1 for l in sentiment_labels if l == \"negative\"),\n            \"neutral\": sum(1 for l in sentiment_labels if l == \"neutral\"),\n            \"average_score\": sum(sentiment_scores) / len(sentiment_scores)\n        }\n        \n        # Get top tweet by engagement\n        top_tweet = max(\n            tweets,\n            key=lambda x: (\n                x[\"metrics\"][\"like_count\"] + \n                x[\"metrics\"][\"retweet_count\"] * 2\n            )\n        )\n        \n        return {\n            \"hashtag\": hashtag,\n            \"tweet_count\": len(tweets),\n            \"sentiment_summary\": sentiment_summary,\n            \"top_tweet\": {\n                \"text\": top_tweet[\"text\"],\n                \"author\": top_tweet[\"author\"][\"username\"],\n                \"url\": top_tweet[\"url\"],\n                \"likes\": top_tweet[\"metrics\"][\"like_count\"],\n                \"retweets\": top_tweet[\"metrics\"][\"retweet_count\"]\n            },\n            \"sample_size\": len(tweets)\n        }\n        \n    async def crawl_trending_topics(self) -> List[Dict]:\n        \"\"\"Crawl and analyze trending topics\"\"\"\n        trending = await self.get_trending_topics()\n        hashtags = [t[\"name\"] for t in trending if t[\"is_hashtag\"]]\n        \n        logger.info(f\"Found {len(hashtags)} trending hashtags to analyze\")\n        \n        # Analyze hashtags concurrently\n        tasks = [self.analyze_hashtag(tag.strip(\"#\")) for tag in hashtags]\n        results = await asyncio.gather(*tasks, return_exceptions=True)\n        \n        # Filter out any failed analyses\n        valid_results = [\n            r for r in results \n            if isinstance(r, dict) and \"error\" not in r\n        ]\n        \n        return valid_results\n        \n    async def generate_report(self, analysis: List[Dict]) -> Dict:\n        \"\"\"Generate a summary report from the analysis\"\"\"\n        if not analysis:\n            return {\"error\": \"No data to generate report\"}\n            \n        # Calculate overall sentiment\n        total_tweets = sum(a.get(\"tweet_count\", 0) for a in analysis)\n        avg_sentiment = sum(\n            a.get(\"sentiment_summary\", {}).get(\"average_score\", 0) \n            for a in analysis\n        ) / len(analysis)\n        \n        overall_sentiment = \"positive\" if avg_sentiment > 0.6 else (\n            \"negative\" if avg_sentiment < 0.4 else \"neutral\"\n        )\n        \n        return {\n            \"generated_at\": datetime.utcnow().isoformat(),\n            \"overall_sentiment\": overall_sentiment,\n            \"avg_sentiment_score\": avg_sentiment,\n            \"total_hashtags_analyzed\": len(analysis),\n            \"total_tweets_analyzed\": total_tweets,\n            \"top_hashtags\": sorted(\n                analysis,\n                key=lambda x: x.get(\"tweet_count\", 0),\n                reverse=True\n            )[:10]  # Top 10 by tweet volume\n        }\n        \n    def save_results(self, data: Dict, filename: str) -> str:\n        \"\"\"Save results to a JSON file\"\"\"\n        output_file = self.output_dir / filename\n        with open(output_file, \"w\") as f:\n            json.dump(data, f, indent=2)\n        return str(output_file)\n        \n    async def run(self):\n        \"\"\"Main execution method\"\"\"\n        start_time = time.time()\n        \n        try:\n            # Crawl trending topics and analyze\n            logger.info(\"Starting Twitter sentiment analysis...\")\n            analysis = await self.crawl_trending_topics()\n            \n            # Generate report\n            report = await self.generate_report(analysis)\n            \n            # Save results\n            timestamp = datetime.utcnow().strftime(\"%Y%m%d_%H%M%S\")\n            report_file = self.save_results(\n                report, \n                f\"twitter_sentiment_report_{timestamp}.json\"\n            )\n            \n            # Print summary\n            duration = time.time() - start_time\n            logger.info(f\"Analysis complete in {duration:.1f} seconds\")\n            logger.info(f\"Report saved to: {report_file}\")\n            \n            return report\n            \n        except Exception as e:\n            logger.critical(f\"Fatal error: {e}\", exc_info=True)\n            raise\n\nasync def main():\n    \"\"\"Main async entry point\"\"\"\n    try:\n        crawler = TwitterSentimentCrawler()\n        await crawler.run()\n    except KeyboardInterrupt:\n        logger.info(\"Analysis interrupted by user\")\n    except Exception as e:\n        logger.critical(f\"Unhandled exception: {e}\", exc_info=True)\n        raise\n\nif __name__ == \"__main__\":\n    asyncio.run(main())",
      "size": 17689,
      "language": "python"
    },
    "src/social/nitter.py": {
      "content": "\"\"\"\nAI Trend Crawler - Nitter Edition (No API Key Required!)\nScrapes Twitter via Nitter instances for daily trending AI topics\n\nRequirements:\npip install requests beautifulsoup4 python-dotenv\n\nNo Twitter API credentials needed! Uses public Nitter instances.\n\"\"\"\n\nimport requests\nfrom bs4 import BeautifulSoup\nimport json\nfrom datetime import datetime\nimport time\nimport re\nfrom urllib.parse import urljoin\n\nclass AITrendCrawler:\n    def __init__(self):\n        \"\"\"Initialize Nitter scraper (no auth needed!)\"\"\"\n        # Public Nitter instances (updated with known working instances)\n        self.nitter_instances = [\n            'https://nitter.privacydev.net',\n            'https://nitter.poast.org',\n            'https://nitter.fdn.fr',\n            'https://nitter.nixnet.services',\n            'https://nitter.1d4.us',\n            'https://nitter.kavin.rocks',\n            'https://nitter.moomoo.me',\n            'https://nitter.it.army',\n            'https://nitter.ir',\n            'https://nitter.weiler.rocks'\n        ]\n        \n        self.current_instance = self.nitter_instances[0]\n        \n        # AI-focused accounts to monitor\n        self.ai_accounts = [\n            'sama',             # Sam Altman\n            'karpathy',         # Andrej Karpathy\n            'emollick',         # Ethan Mollick\n            'OpenAI',           # OpenAI\n            'anthropicai',      # Anthropic\n            'GoogleDeepMind',   # DeepMind\n            'ylecun',           # Yann LeCun\n            'goodside',         # Riley Goodside\n            'bindureddy',       # Aravind Srinivas (Perplexity)\n        ]\n        \n        # Search keywords\n        self.ai_keywords = [\n            'ChatGPT', 'GPT-4', 'Claude', 'LLM',\n            'AI', 'machine learning', 'AGI'\n        ]\n        \n        # Headers to avoid blocking - updated with more realistic browser headers\n        self.headers = {\n            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',\n            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',\n            'Accept-Language': 'en-US,en;q=0.5',\n            'DNT': '1',\n            'Connection': 'keep-alive',\n            'Upgrade-Insecure-Requests': '1',\n            'Sec-Fetch-Dest': 'document',\n            'Sec-Fetch-Mode': 'navigate',\n            'Sec-Fetch-Site': 'none',\n            'Sec-Fetch-User': '?1',\n            'Cache-Control': 'max-age=0'\n        }\n    \n    def test_nitter_instance(self, instance):\n        \"\"\"Test if a Nitter instance is working\"\"\"\n        try:\n            # Test with a popular account that's likely to exist\n            url = f\"{instance}/jack\"\n            response = requests.get(\n                url, \n                headers=self.headers, \n                timeout=15,\n                allow_redirects=True,\n                verify=False\n            )\n            \n            # Check if we got a valid response\n            if response.status_code != 200:\n                print(f\"   ⚠️ Instance {instance} returned status code {response.status_code}\")\n                return False\n                \n            # Check if the response looks like a Nitter page\n            response_text = response.text.lower()\n            \n            # Check for signs of a working Nitter page\n            is_nitter = ('nitter' in response_text or \n                        'twitter' in response_text or\n                        'tweet' in response_text or\n                        'timeline' in response_text)\n            \n            # Check for signs of error pages or captchas\n            is_error_page = any(term in response_text for term in [\n                'error', 'unavailable', 'captcha', 'rate limit', 'too many requests',\n                'jeans', 'lavare'  # Terms from the Italian error page we saw\n            ])\n            \n            # Check rate limiting\n            is_rate_limited = ('rate limit' in response_text or \n                             'too many requests' in response_text or\n                             response.status_code == 429)\n            \n            if is_rate_limited:\n                print(f\"   ⚠️ Rate limited on {instance}, trying next instance...\")\n                return False\n                \n            if is_error_page:\n                print(f\"   ⚠️ Instance {instance} returned an error page\")\n                return False\n                \n            return is_nitter and not is_error_page\n            \n        except requests.exceptions.SSLError:\n            # Try with verify=False for SSL issues\n            try:\n                response = requests.get(\n                    f\"{instance}/jack\",\n                    headers=self.headers,\n                    timeout=10,\n                    verify=False\n                )\n                return response.status_code == 200 and 'twitter' in response.text.lower()\n            except:\n                return False\n                \n        except Exception as e:\n            return False\n    \n    def get_working_instance(self, max_retries=3):\n        \"\"\"Find a working Nitter instance with retries\"\"\"\n        import random\n        import time\n        \n        # First try the instances in order\n        for attempt in range(max_retries):\n            print(f\"\\n🔍 Attempt {attempt + 1}/{max_retries} to find working Nitter instance...\")\n            \n            # Try instances in a random order\n            instances = self.nitter_instances.copy()\n            random.shuffle(instances)\n            \n            for instance in instances:\n                try:\n                    print(f\"   Testing instance: {instance}\")\n                    if self.test_nitter_instance(instance):\n                        print(f\"   ✓ Found working Nitter instance: {instance}\")\n                        return instance\n                    else:\n                        print(f\"   ⚠️ Instance {instance} failed validation\")\n                except Exception as e:\n                    print(f\"   ⚠️ Error with {instance}: {str(e)[:100]}...\")\n                    continue\n            \n            if attempt < max_retries - 1:\n                wait_time = (attempt + 1) * 5  # Exponential backoff\n                print(f\"\\n🔄 No working instances found. Retrying in {wait_time} seconds...\")\n                time.sleep(wait_time)\n        \n        # If we get here, all attempts failed with the current instances list\n        # Try to fetch a fresh list from the Nitter instances API\n        try:\n            print(\"\\n🔄 No working instances in the default list. Trying to fetch fresh instances...\")\n            response = requests.get(\n                'https://raw.githubusercontent.com/zedeus/nitter/master/status/instances.json',\n                headers=self.headers,\n                timeout=10\n            )\n            \n            if response.status_code == 200:\n                instances_data = response.json()\n                # Get the best instances (sorted by uptime)\n                fresh_instances = [\n                    f\"https://{instance['name']}\" \n                    for instance in instances_data.get('instances', []) \n                    if instance.get('status') == 'up' and not instance.get('is_tor', False)\n                ][:10]  # Try top 10 most reliable non-Tor instances\n                \n                print(f\"   Fetched {len(fresh_instances)} fresh instances, testing...\")\n                \n                for instance in fresh_instances:\n                    try:\n                        if self.test_nitter_instance(instance):\n                            print(f\"   ✓ Found working Nitter instance: {instance}\")\n                            # Add this instance to our list for future use\n                            if instance not in self.nitter_instances:\n                                self.nitter_instances.insert(0, instance)\n                            return instance\n                    except Exception as e:\n                        print(f\"   ⚠️ Error testing fresh instance {instance}: {str(e)[:100]}...\")\n                        continue\n        except Exception as e:\n            print(f\"   ⚠️ Could not fetch fresh instances: {str(e)[:100]}...\")\n        \n        # If we get here, all attempts failed\n        raise Exception(\n            \"No working Nitter instances found after multiple attempts. \"\n            \"This could be due to rate limiting or temporary outages. \"\n            \"Please try again later or check https://github.com/zedeus/nitter/wiki/Instances \"\n            \"for updated instance list.\"\n        )\n    \n    def fetch_account_tweets(self, username, limit=5, retry_count=0, max_retries=3):\n        \"\"\"Scrape recent tweets from a user\"\"\"\n        try:\n            # Get a working instance if we don't have one\n            if not hasattr(self, 'current_instance') or not self.current_instance:\n                self.current_instance = self.get_working_instance()\n            \n            print(f\"\\n   🔍 Fetching @{username}...\")\n            print(f\"   🌐 URL: {self.current_instance}/{username}\")\n            \n            url = f\"{self.current_instance}/{username}\"\n            start_time = time.time()\n            response = requests.get(url, headers=self.headers, timeout=15)\n            elapsed = time.time() - start_time\n            \n            print(f\"   ⏱️  Response time: {elapsed:.2f}s\")\n            print(f\"   📡 Status code: {response.status_code}\")\n            \n            if response.status_code != 200:\n                print(f\"   ❌ Failed to fetch @{username}\")\n                print(f\"   📄 Response text: {response.text[:200]}...\" if response.text else \"   📄 No response content\")\n                \n                # If we get a non-200 status, try with a different instance\n                if retry_count < max_retries:\n                    print(f\"   🔄 Retrying with a different instance (attempt {retry_count + 1}/{max_retries})...\")\n                    # Remove the current instance from the list\n                    if self.current_instance in self.nitter_instances:\n                        self.nitter_instances.remove(self.current_instance)\n                    self.current_instance = None\n                    return self.fetch_account_tweets(username, limit, retry_count + 1, max_retries)\n                return []\n                \n            print(f\"   ✅ Successfully fetched @{username}'s page\")\n            \n            # Check if this looks like a valid Nitter page\n            soup = BeautifulSoup(response.text, 'html.parser')\n            page_title = soup.title.string.lower() if soup.title else ''\n            \n            if 'twitter' not in page_title and 'nitter' not in page_title:\n                print(f\"⚠️  Warning: Doesn't look like a Twitter/Nitter page. Title: {page_title}\")\n                if retry_count < max_retries:\n                    print(f\"   🔄 Retrying with a different instance (attempt {retry_count + 1}/{max_retries})...\")\n                    # Remove the current instance from the list\n                    if self.current_instance in self.nitter_instances:\n                        self.nitter_instances.remove(self.current_instance)\n                    self.current_instance = None\n                    return self.fetch_account_tweets(username, limit, retry_count + 1, max_retries)\n                return []\n            \n            soup = BeautifulSoup(response.text, 'html.parser')\n            # Add this right after line 184\n            print(\"=== DEBUG: Analyzing HTML Structure ===\")\n\n            # 1. Check if we got a valid Twitter/Nitter page\n            page_title = soup.title.string.lower() if soup.title else ''\n            if 'twitter' not in page_title and 'nitter' not in page_title:\n                print(f\"⚠️  Warning: Doesn't look like a Twitter/Nitter page. Title: {page_title}\")\n                print(\"   The instance might be down or redirecting to a different page.\")\n                print(\"   Trying next Nitter instance...\")\n                return []\n\n            # 2. Print the first 500 characters of the HTML to see what we're working with\n            print(\"\\n=== First 500 chars of HTML ===\")\n            print(response.text[:500])\n\n            # 3. Print all div classes in the document\n            print(\"\\n=== All div classes found ===\")\n            div_classes = set()\n            for div in soup.find_all('div'):\n                if div.get('class'):\n                    div_classes.update(div.get('class'))\n            print(div_classes)\n\n            # 4. Look for any tweet-like elements using multiple possible selectors\n            print(\"\\n=== Potential tweet containers ===\")\n            tweet_selectors = [\n                'div.timeline-item', \n                'div.tweet', \n                'article.tweet', \n                'div.timeline-item-content',\n                'div.timeline',\n                'div.timeline-item',\n                'div.thread',\n                'div.timeline-item:not(.show-more)'\n            ]\n            \n            tweet_containers = []\n            for selector in tweet_selectors:\n                elements = soup.select(selector)\n                print(f\"Found {len(elements)} elements with selector '{selector}'\")\n                if elements:\n                    print(f\"First element classes: {elements[0].get('class', [])}\")\n                    print(f\"First 100 chars of text: {elements[0].get_text()[:100]}...\\n\")\n                    tweet_containers.extend(elements)\n\n            # 5. Print all article tags (common for tweets)\n            print(\"\\n=== All article tags ===\")\n            articles = soup.find_all('article')\n            print(f\"Found {len(articles)} article tags\")\n            for i, article in enumerate(articles[:3]):  # Only show first 3 to avoid too much output\n                print(f\"\\nArticle {i+1} classes: {article.get('class', [])}\")\n                print(f\"Text preview: {article.get_text()[:200]}...\\n\")\n                tweet_containers.append(article)\n\n            # 6. Print all divs with class containing 'tweet' or 'timeline'\n            print(\"\\n=== Divs with tweet or timeline in class ===\")\n            for div in soup.find_all('div', class_=lambda c: c and any(x in str(c) for x in ['tweet', 'timeline', 'thread'])):\n                classes = div.get('class', [])\n                print(f\"Div classes: {classes}\")\n                print(f\"Text preview: {div.get_text()[:100]}...\\n\")\n                tweet_containers.append(div)\n\n            print(\"=== END DEBUG ===\\n\")\n            tweets = []\n            \n            # If we found potential tweet containers, use them\n            if tweet_containers:\n                tweet_items = tweet_containers[:limit*2]  # Get more items in case some fail parsing\n            else:\n                print(\"⚠️  No tweet containers found. The page structure might have changed.\")\n                print(\"   Trying to find any content that looks like a tweet...\")\n                \n                # Fallback: look for any div with content that looks like a tweet\n                tweet_items = []\n                for div in soup.find_all('div'):\n                    text = div.get_text(strip=True)\n                    # More strict checks for tweet-like content\n                    if (50 < len(text) < 500 and \n                        any(c in text for c in ['@', '#']) and  # Likely contains @mentions or #hashtags\n                        not any(word in text.lower() for word in ['cookie', 'privacy', 'terms', '©'])  # Skip footers\n                    ):\n                        print(f\"Found potential tweet-like content: {text[:100]}...\")\n                        tweet_items.append(div)\n                        if len(tweet_items) >= limit*2:\n                            break\n                \n                if not tweet_items:\n                    print(\"❌ No tweet-like content found. The page might be showing a captcha or error message.\")\n                    print(\"   Try a different Nitter instance or check if the instance is working in a web browser.\")\n                    return []\n            \n            for item in tweet_items:\n                try:\n                    # Try multiple selectors for tweet content\n                    tweet_content = None\n                    content_selectors = [\n                        'div.tweet-content', \n                        'div.timeline-item-content',\n                        'div.tweet-body',\n                        'div.content',\n                        'div.entry-content',\n                        'div.text'\n                    ]\n                    \n                    for selector in content_selectors:\n                        tweet_content = item.select_one(selector)\n                        if tweet_content:\n                            break\n                    \n                    # If no content found, try to get text directly\n                    if not tweet_content:\n                        text = item.get_text(separator=' ', strip=True)\n                    else:\n                        text = tweet_content.get_text(separator=' ', strip=True)\n                    \n                    # Skip if text is too short or too long to be a tweet\n                    if len(text) < 20 or len(text) > 500:\n                        continue\n                    \n                    # Initialize metrics\n                    comments = retweets = likes = 0\n                    \n                    # Try multiple selectors for stats\n                    stats_selectors = [\n                        'div.tweet-stats', \n                        'div.timeline-item-stats',\n                        'div.stats',\n                        'div.engagement',\n                        'div.tweet-actions'\n                    ]\n                    \n                    for selector in stats_selectors:\n                        stats = item.select(selector)\n                        if stats:\n                            stats_text = ' '.join(stat.get_text(' ', strip=True) for stat in stats)\n                            \n                            # Parse engagement numbers with more flexible patterns\n                            comment_match = re.search(r'(\\d+)\\s*(comment|reply|replies|💬|💭|🗨️)', stats_text, re.IGNORECASE | re.UNICODE)\n                            retweet_match = re.search(r'(\\d+)\\s*(retweet|rt|🔁|♻️)', stats_text, re.IGNORECASE | re.UNICODE)\n                            like_match = re.search(r'(\\d+)\\s*(like|❤️|♥️|👍|♡)', stats_text, re.IGNORECASE | re.UNICODE)\n                            \n                            if comment_match:\n                                comments = int(comment_match.group(1))\n                            if retweet_match:\n                                retweets = int(retweet_match.group(1))\n                            if like_match:\n                                likes = int(like_match.group(1))\n                            break\n                    \n                    # Try to find username and tweet ID\n                    username = 'Unknown'\n                    tweet_id = None\n                    \n                    # Try multiple link selectors\n                    link_selectors = [\n                        'a.tweet-link', \n                        'a.timeline-item-link',\n                        'a.permalink',\n                        'a.time',\n                        'a[href*=\"/status/\"]'\n                    ]\n                    \n                    for selector in link_selectors:\n                        tweet_link = item.select_one(selector)\n                        if tweet_link and 'href' in tweet_link.attrs:\n                            href = tweet_link['href']\n                            parts = [p for p in href.split('/') if p]\n                            if len(parts) >= 2:\n                                username = parts[0] if not parts[0].startswith('@') else parts[0][1:]\n                                # Look for status ID in the URL\n                                for part in parts:\n                                    if part.isdigit() and len(part) > 5:  # Likely a tweet ID\n                                        tweet_id = part\n                                        break\n                                if tweet_id:\n                                    break\n                    \n                    # Try to find timestamp\n                    timestamp = 'Unknown'\n                    time_selectors = [\n                        'span.tweet-date', \n                        'span.timeline-item-time',\n                        'time',\n                        'span.time',\n                        'span.timestamp'\n                    ]\n                    \n                    for selector in time_selectors:\n                        timestamp_elem = item.select_one(selector)\n                        if timestamp_elem and timestamp_elem.get_text(strip=True):\n                            timestamp = timestamp_elem.get_text(strip=True)\n                            break\n                    \n                    # Add the tweet if we found enough information\n                    if text and (username != 'Unknown' or tweet_id):\n                        tweets.append({\n                            'id': tweet_id or f'local_{len(tweets)}',\n                            'text': text,\n                            'author': username,\n                            'timestamp': timestamp,\n                            'public_metrics': {\n                                'reply_count': comments,\n                                'retweet_count': retweets,\n                                'like_count': likes,\n                            }\n                        })\n                    \n                except Exception as e:\n                    continue\n            \n            print(f\"   ✅ @{username}: Successfully parsed {len(tweets)} tweets\")\n            time.sleep(1)  # Be respectful to the server\n            return tweets\n            \n        except Exception as e:\n            import traceback\n            print(f\"\\n❌ Error fetching @{username}:\")\n            print(f\"   🔍 Error type: {type(e).__name__}\")\n            print(f\"   📝 Error details: {str(e)}\")\n            print(\"   🧵 Stack trace:\")\n            traceback.print_exc(limit=1)\n            return []\n    \n    def search_nitter(self, query, limit=20):\n        \"\"\"Search Nitter for tweets\"\"\"\n        try:\n            print(f\"   🔍 Searching: {query}...\")\n            \n            # URL encode the query\n            search_url = f\"{self.current_instance}/search?f=tweets&q={query.replace(' ', '+')}\"\n            response = requests.get(search_url, headers=self.headers, timeout=10)\n            \n            if response.status_code != 200:\n                print(f\"   ✗ Search failed for: {query}\")\n                return []\n            \n            soup = BeautifulSoup(response.text, 'html.parser')\n            tweets = []\n            \n            # Find tweet containers - updated for new Nitter structure\n            tweet_items = soup.select('div.timeline-item, div.timeline-item:not(.show-more)')[:limit]\n            \n            for item in tweet_items:\n                try:\n                    # Extract username - updated for new Nitter structure\n                    username_elem = item.select_one('a.username, a.timeline-item-username')\n                    username = username_elem.get_text(strip=True).replace('@', '') if username_elem else 'Unknown'\n                    \n                    # Extract tweet text - updated for new Nitter structure\n                    tweet_content = item.select_one('div.tweet-content, div.timeline-item-content')\n                    if not tweet_content:\n                        continue\n                    \n                    text = tweet_content.get_text(separator=' ', strip=True)\n                    \n                    # Extract stats - updated for new Nitter structure\n                    stats = item.select('div.tweet-stats, div.timeline-item-stats')\n                    comments = retweets = likes = 0\n                    \n                    if stats:\n                        stats_text = ' '.join(stat.get_text(' ', strip=True) for stat in stats)\n                        \n                        # Parse engagement numbers\n                        comment_match = re.search(r'(\\d+)\\s*(comment|reply|replies)', stats_text)\n                        retweet_match = re.search(r'(\\d+)\\s*retweet', stats_text, re.IGNORECASE)\n                        like_match = re.search(r'(\\d+)\\s*like', stats_text, re.IGNORECASE)\n                        \n                        if comment_match:\n                            comments = int(comment_match.group(1))\n                        if retweet_match:\n                            retweets = int(retweet_match.group(1))\n                        if like_match:\n                            likes = int(like_match.group(1))\n                    \n                    # Extract tweet ID - updated for new Nitter structure\n                    tweet_link = item.select_one('a.tweet-link, a.timeline-item-link')\n                    tweet_id = None\n                    if tweet_link and 'href' in tweet_link.attrs:\n                        href = tweet_link['href']\n                        parts = [p for p in href.split('/') if p]\n                        if parts:\n                            tweet_id = parts[-1].split('#')[0]\n                    \n                    tweets.append({\n                        'id': tweet_id,\n                        'text': text,\n                        'author': username,\n                        'public_metrics': {\n                            'reply_count': comments,\n                            'retweet_count': retweets,\n                            'like_count': likes,\n                        }\n                    })\n                    \n                except Exception as e:\n                    continue\n            \n            print(f\"   ✓ Found {len(tweets)} tweets for: {query}\")\n            time.sleep(2)  # Be respectful\n            return tweets\n            \n        except Exception as e:\n            print(f\"   ✗ Search error: {str(e)[:100]}\")\n            return []\n    \n    def calculate_engagement_score(self, metrics):\n        \"\"\"Calculate engagement score for ranking\"\"\"\n        return (\n            metrics.get('retweet_count', 0) * 3 +\n            metrics.get('like_count', 0) * 2 +\n            metrics.get('reply_count', 0) * 1\n        )\n    \n    def classify_sentiment(self, text):\n        \"\"\"Simple sentiment classification\"\"\"\n        text_lower = text.lower()\n        \n        if any(word in text_lower for word in ['scary', 'dangerous', 'terrifying', 'fear', 'worried']):\n            return 'Fear/Concern'\n        elif any(word in text_lower for word in ['lol', 'haha', 'funny', '😂', '🤣', 'hilarious']):\n            return 'Humor'\n        elif any(word in text_lower for word in ['overhype', 'scam', 'skeptical', 'doubt']):\n            return 'Skepticism'\n        elif any(word in text_lower for word in ['amazing', 'incredible', 'wow', 'mind-blowing', '🤯']):\n            return 'Excitement'\n        elif any(word in text_lower for word in ['finally', 'useful', 'practical', 'works']):\n            return 'Practical Interest'\n        else:\n            return 'Discussion'\n    \n    def crawl_daily_trends(self, output_file='daily_trends_nitter.json'):\n        \"\"\"Main crawler using Nitter\"\"\"\n        print(f\"🕷️  Starting Nitter AI Trend Crawl at {datetime.now()}\")\n        print(\"ℹ️  No API key needed! Using public Nitter instances.\\n\")\n        \n        # Find working Nitter instance\n        try:\n            self.current_instance = self.get_working_instance()\n        except Exception as e:\n            print(f\"❌ {e}\")\n            return {'error': str(e), 'trends': []}\n        \n        all_tweets = []\n        \n        # 1. Fetch from AI influencers\n        print(\"\\n📱 Fetching from AI influencer accounts...\")\n        for username in self.ai_accounts:\n            tweets = self.fetch_account_tweets(username, limit=3)\n            for tweet in tweets:\n                tweet['source_type'] = 'account'\n                tweet['source_name'] = f'@{username}'\n                all_tweets.append(tweet)\n        \n        # 2. Search for trending AI topics\n        print(\"\\n🔥 Searching trending AI topics...\")\n        for keyword in self.ai_keywords[:3]:  # Limit searches to avoid rate limits\n            tweets = self.search_nitter(keyword, limit=5)\n            for tweet in tweets:\n                tweet['source_type'] = 'search'\n                tweet['source_name'] = f'Search: {keyword}'\n                all_tweets.append(tweet)\n        \n        # 3. Remove duplicates\n        seen_ids = set()\n        unique_tweets = []\n        for tweet in all_tweets:\n            tweet_id = tweet.get('id')\n            if tweet_id and tweet_id not in seen_ids:\n                seen_ids.add(tweet_id)\n                unique_tweets.append(tweet)\n        \n        print(f\"\\n📊 Total unique tweets: {len(unique_tweets)}\")\n        \n        if not unique_tweets:\n            print(\"⚠️  No tweets found\")\n            return {'error': 'No tweets found', 'trends': []}\n        \n        # 4. Rank by engagement\n        unique_tweets.sort(\n            key=lambda t: self.calculate_engagement_score(t.get('public_metrics', {})),\n            reverse=True\n        )\n        \n        # 5. Format top 15\n        top_tweets = unique_tweets[:15]\n        \n        trends = []\n        for tweet in top_tweets:\n            text = tweet.get('text', '')\n            metrics = tweet.get('public_metrics', {})\n            \n            trend_item = {\n                'source': f\"Twitter - {tweet.get('source_name', 'AI Community')}\",\n                'title': (text[:97] + '...') if len(text) > 100 else text,\n                'url': f\"https://twitter.com/{tweet.get('author')}/status/{tweet.get('id')}\" if tweet.get('id') else '#',\n                'full_text': text,\n                'author': f\"@{tweet.get('author', 'Unknown')}\",\n                'timestamp': tweet.get('timestamp', 'Unknown'),\n                'engagement': {\n                    'likes': metrics.get('like_count', 0),\n                    'retweets': metrics.get('retweet_count', 0),\n                    'replies': metrics.get('reply_count', 0),\n                },\n                'sentiment_tag': self.classify_sentiment(text)\n            }\n            trends.append(trend_item)\n        \n        # 6. Create output\n        output = {\n            'date': datetime.now().strftime('%Y-%m-%d'),\n            'generated_at': datetime.now().isoformat(),\n            'method': 'Nitter Scraping (No API Key)',\n            'nitter_instance': self.current_instance,\n            'total_analyzed': len(all_tweets),\n            'unique_tweets': len(unique_tweets),\n            'accounts_processed': len(self.ai_accounts),\n            'trends_found': len(trends),\n            'trends': trends,\n            'note': 'Data scraped from public Nitter instances. No Twitter API key required.'\n        }\n        \n        # 7. Save to file\n        with open(output_file, 'w', encoding='utf-8') as f:\n            json.dump(output, f, indent=2, ensure_ascii=False)\n        \n        print(f\"\\n✅ Crawl complete! Generated {len(trends)} trend items\")\n        print(f\"📄 Output saved to: {output_file}\")\n        \n        return output\n\n\ndef main():\n    \"\"\"Run the crawler\"\"\"\n    try:\n        crawler = AITrendCrawler()\n        result = crawler.crawl_daily_trends()\n        \n        # Print sample\n        print(\"\\n\" + \"=\"*60)\n        print(\"📊 SAMPLE RESULTS\")\n        print(\"=\"*60)\n        \n        if result.get('trends'):\n            print(f\"\\nFound {len(result['trends'])} trends\\n\")\n            for i, trend in enumerate(result['trends'][:3], 1):\n                print(f\"\\n--- Trend #{i} ---\")\n                print(f\"Author: {trend['author']}\")\n                print(f\"Text: {trend['title']}\")\n                print(f\"Engagement: {trend['engagement']['likes']} likes, {trend['engagement']['retweets']} retweets\")\n                print(f\"Sentiment: {trend['sentiment_tag']}\")\n        else:\n            print(\"No trends found.\")\n        \n    except Exception as e:\n        print(f\"❌ Error: {e}\")\n        import traceback\n        traceback.print_exc()\n        return 1\n    \n    return 0\n\n\nif __name__ == '__main__':\n    import sys\n    sys.exit(main())",
      "size": 32379,
      "language": "python"
    },
    "src/social/reddit.py": {
      "content": "\"\"\"\nAI Trend Crawler - Reddit Edition\nFetches daily trending AI topics from Reddit for Hongjie's Coze Bot\n\nRequirements:\npip install praw python-dotenv\n\nSetup:\n1. Create a Reddit app at https://www.reddit.com/prefs/apps\n   - Click \"create another app\" or \"create app\"\n   - Choose \"script\" type\n   - Set redirect URI to http://localhost:8080\n2. Create .env file with:\n   REDDIT_CLIENT_ID=your_client_id\n   REDDIT_CLIENT_SECRET=your_client_secret\n   REDDIT_USER_AGENT=AITrendCrawler/1.0\n\"\"\"\n\nimport praw\nimport json\nfrom datetime import datetime, timedelta\nfrom collections import defaultdict\nimport os\nfrom dotenv import load_dotenv\n\n# Load environment variables\nload_dotenv()\n\nclass AITrendCrawler:\n    def __init__(self):\n        \"\"\"Initialize Reddit API client\"\"\"\n        client_id = os.getenv('REDDIT_CLIENT_ID')\n        client_secret = os.getenv('REDDIT_CLIENT_SECRET')\n        user_agent = os.getenv('REDDIT_USER_AGENT', 'AITrendCrawler/1.0')\n        \n        if not client_id or not client_secret:\n            raise ValueError(\"REDDIT_CLIENT_ID and REDDIT_CLIENT_SECRET must be set in .env file\")\n        \n        self.reddit = praw.Reddit(\n            client_id=client_id,\n            client_secret=client_secret,\n            user_agent=user_agent\n        )\n        \n        # AI-focused subreddits to monitor\n        self.ai_subreddits = [\n            'artificial',           # r/artificial - AI discussions\n            'MachineLearning',      # r/MachineLearning - ML research & news\n            'OpenAI',               # r/OpenAI - OpenAI products\n            'ClaudeAI',             # r/ClaudeAI - Claude discussions\n            'LocalLLaMA',           # r/LocalLLaMA - Open source LLMs\n            'singularity',          # r/singularity - AGI & future\n            'ChatGPT',              # r/ChatGPT - ChatGPT discussions\n            'StableDiffusion',      # r/StableDiffusion - AI art\n            'ArtificialInteligence',# r/ArtificialInteligence (yes, misspelled)\n            'deeplearning',         # r/deeplearning - Deep learning\n        ]\n        \n        # Time filter for \"hot\" posts (last 24 hours)\n        self.time_filter = 'day'\n    \n    def fetch_subreddit_posts(self, subreddit_name, limit=10, sort='hot'):\n        \"\"\"Fetch top posts from a specific subreddit\"\"\"\n        try:\n            print(f\"   🔍 Fetching r/{subreddit_name}...\")\n            subreddit = self.reddit.subreddit(subreddit_name)\n            \n            # Get posts based on sort method\n            if sort == 'hot':\n                posts = list(subreddit.hot(limit=limit))\n            elif sort == 'top':\n                posts = list(subreddit.top(time_filter=self.time_filter, limit=limit))\n            else:\n                posts = list(subreddit.new(limit=limit))\n            \n            print(f\"   ✓ r/{subreddit_name}: Found {len(posts)} posts\")\n            return posts\n            \n        except Exception as e:\n            print(f\"   ✗ Error fetching r/{subreddit_name}: {str(e)[:100]}\")\n            return []\n    \n    def search_reddit_ai(self, query='AI OR LLM OR ChatGPT OR Claude', limit=50):\n        \"\"\"Search across all Reddit for AI-related content\"\"\"\n        try:\n            print(f\"   🔍 Searching Reddit for: {query[:50]}...\")\n            \n            # Search with time filter\n            posts = list(self.reddit.subreddit('all').search(\n                query, \n                time_filter=self.time_filter,\n                limit=limit,\n                sort='relevance'\n            ))\n            \n            print(f\"   ✓ Found {len(posts)} posts from search\")\n            return posts\n            \n        except Exception as e:\n            print(f\"   ✗ Error searching Reddit: {str(e)[:100]}\")\n            return []\n    \n    def calculate_engagement_score(self, post):\n        \"\"\"Calculate engagement score for ranking\"\"\"\n        # Reddit scoring: upvotes + comments weighted\n        score = post.score * 2  # Upvotes weighted heavily\n        score += post.num_comments * 3  # Comments show engagement\n        score += (1 if post.stickied else 0) * 100  # Stickied posts are important\n        \n        # Boost recent posts (within last 12 hours)\n        post_age_hours = (datetime.utcnow() - datetime.utcfromtimestamp(post.created_utc)).total_seconds() / 3600\n        if post_age_hours < 12:\n            score *= 1.5\n        \n        return score\n    \n    def classify_sentiment(self, title, selftext=''):\n        \"\"\"Simple sentiment classification based on keywords\"\"\"\n        text = (title + ' ' + selftext).lower()\n        \n        # Keyword-based classification\n        if any(word in text for word in ['scary', 'dangerous', 'terrifying', 'fear', 'worried', 'concerning']):\n            return 'Fear/Concern'\n        elif any(word in text for word in ['lol', 'haha', 'funny', 'hilarious', 'meme']):\n            return 'Humor'\n        elif any(word in text for word in ['overhype', 'scam', 'skeptical', 'doubt', 'bs', 'disappointing']):\n            return 'Skepticism'\n        elif any(word in text for word in ['amazing', 'incredible', 'wow', 'mind-blowing', 'breakthrough', 'revolutionary']):\n            return 'Excitement'\n        elif any(word in text for word in ['tutorial', 'guide', 'how to', 'useful', 'practical', 'works']):\n            return 'Practical/Educational'\n        elif any(word in text for word in ['research', 'paper', 'study', 'analysis']):\n            return 'Research'\n        else:\n            return 'Discussion'\n    \n    def extract_top_comment(self, post, max_length=200):\n        \"\"\"Extract the most upvoted comment from a post\"\"\"\n        try:\n            # Refresh to get comments\n            post.comments.replace_more(limit=0)  # Don't fetch \"more comments\"\n            \n            if post.comments:\n                # Get top comment by score\n                top_comment = max(post.comments, key=lambda c: c.score)\n                comment_text = top_comment.body[:max_length]\n                return {\n                    'text': comment_text + ('...' if len(top_comment.body) > max_length else ''),\n                    'author': str(top_comment.author),\n                    'score': top_comment.score\n                }\n        except:\n            pass\n        \n        return None\n    \n    def crawl_daily_trends(self, output_file='daily_trends_reddit.json'):\n        \"\"\"Main crawler: fetch and process daily AI trends from Reddit\"\"\"\n        print(f\"🕷️  Starting Reddit AI Trend Crawl at {datetime.now()}\")\n        \n        all_posts = []\n        \n        # 1. Fetch from curated AI subreddits\n        print(\"\\n📱 Fetching from AI subreddits...\")\n        for subreddit_name in self.ai_subreddits:\n            posts = self.fetch_subreddit_posts(subreddit_name, limit=5, sort='hot')\n            for post in posts:\n                all_posts.append({\n                    'post': post,\n                    'source_type': 'subreddit',\n                    'source_name': f'r/{subreddit_name}'\n                })\n        \n        # 2. Search for trending AI topics across all Reddit\n        print(\"\\n🔥 Searching trending AI topics across Reddit...\")\n        search_posts = self.search_reddit_ai(limit=20)\n        for post in search_posts:\n            all_posts.append({\n                'post': post,\n                'source_type': 'search',\n                'source_name': 'Reddit Search'\n            })\n        \n        print(f\"\\n📊 Total posts collected: {len(all_posts)}\")\n        \n        if not all_posts:\n            print(\"\\n⚠️  No posts found to analyze\")\n            return {'error': 'No posts found', 'trends': []}\n        \n        # 3. Remove duplicates (same post from different sources)\n        seen_ids = set()\n        unique_posts = []\n        for item in all_posts:\n            post_id = item['post'].id\n            if post_id not in seen_ids:\n                seen_ids.add(post_id)\n                unique_posts.append(item)\n        \n        print(f\"📊 Unique posts after deduplication: {len(unique_posts)}\")\n        \n        # 4. Rank by engagement\n        unique_posts.sort(\n            key=lambda item: self.calculate_engagement_score(item['post']),\n            reverse=True\n        )\n        \n        # 5. Take top 15 and format\n        top_posts = unique_posts[:15]\n        \n        trends = []\n        print(\"\\n🔍 Processing top posts...\")\n        for item in top_posts:\n            post = item['post']\n            \n            # Get post details\n            title = post.title\n            selftext = post.selftext[:300] if post.selftext else ''\n            \n            # Extract top comment\n            top_comment = self.extract_top_comment(post)\n            \n            # Calculate post age\n            post_age = datetime.utcnow() - datetime.utcfromtimestamp(post.created_utc)\n            age_str = f\"{int(post_age.total_seconds() / 3600)}h ago\" if post_age.total_seconds() < 86400 else f\"{int(post_age.days)}d ago\"\n            \n            trend_item = {\n                'source': f\"Reddit - {item['source_name']}\",\n                'title': title,\n                'url': f\"https://reddit.com{post.permalink}\",\n                'description': selftext if selftext else '(Link post)',\n                'author': f\"u/{post.author}\" if post.author else \"u/[deleted]\",\n                'subreddit': f\"r/{post.subreddit.display_name}\",\n                'posted': age_str,\n                'engagement': {\n                    'upvotes': post.score,\n                    'upvote_ratio': round(post.upvote_ratio * 100, 1),\n                    'comments': post.num_comments,\n                },\n                'top_comment': top_comment,\n                'sentiment_tag': self.classify_sentiment(title, selftext),\n                'awards': post.total_awards_received,\n                'is_video': post.is_video,\n            }\n            trends.append(trend_item)\n        \n        # 6. Create output JSON with metadata\n        output = {\n            'date': datetime.now().strftime('%Y-%m-%d'),\n            'generated_at': datetime.now().isoformat(),\n            'total_analyzed': len(all_posts),\n            'unique_posts': len(unique_posts),\n            'subreddits_processed': len(self.ai_subreddits),\n            'trends_found': len(trends),\n            'trends': trends,\n            'note': 'Data collected from Reddit using PRAW (Python Reddit API Wrapper)'\n        }\n        \n        # 7. Save to file\n        with open(output_file, 'w', encoding='utf-8') as f:\n            json.dump(output, f, indent=2, ensure_ascii=False)\n        \n        print(f\"\\n✅ Crawl complete! Generated {len(trends)} trend items\")\n        print(f\"📄 Output saved to: {output_file}\")\n        \n        return output\n\n\ndef main():\n    \"\"\"Run the crawler\"\"\"\n    try:\n        crawler = AITrendCrawler()\n        result = crawler.crawl_daily_trends()\n        \n        # Print sample results\n        print(\"\\n\" + \"=\"*60)\n        print(\"📊 SAMPLE RESULTS\")\n        print(\"=\"*60)\n        \n        if result.get('trends'):\n            print(f\"\\nFound {len(result['trends'])} trends\\n\")\n            \n            # Show first 3 trends\n            for i, trend in enumerate(result['trends'][:3], 1):\n                print(f\"\\n--- Trend #{i} ---\")\n                print(f\"Title: {trend['title']}\")\n                print(f\"Source: {trend['source']}\")\n                print(f\"Subreddit: {trend['subreddit']}\")\n                print(f\"Engagement: {trend['engagement']['upvotes']} upvotes, {trend['engagement']['comments']} comments\")\n                print(f\"Sentiment: {trend['sentiment_tag']}\")\n                print(f\"URL: {trend['url']}\")\n                if trend.get('top_comment'):\n                    print(f\"Top Comment: {trend['top_comment']['text'][:100]}...\")\n        else:\n            print(\"No trends found.\")\n        \n    except Exception as e:\n        print(f\"❌ Error: {e}\")\n        import traceback\n        traceback.print_exc()\n        return 1\n    \n    return 0\n\n\nif __name__ == '__main__':\n    import sys\n    sys.exit(main())",
      "size": 11962,
      "language": "python"
    },
    "src/social/blue_sky.py": {
      "content": "\"\"\"\nAI Trend Crawler - Bluesky Edition\nFetches daily trending AI topics from Bluesky for Hongjie's Coze Bot\n\nRequirements:\npip install atproto python-dotenv\n\nSetup:\n1. Create Bluesky account at https://bsky.app\n2. Create .env file with your credentials:\n   BLUESKY_HANDLE=your.handle.bsky.social\n   BLUESKY_PASSWORD=your_app_password\n\nNote: You can create an app password at Settings > App Passwords\n\"\"\"\n\nfrom atproto import Client\nimport json\nfrom datetime import datetime, timedelta, timezone\nimport os\nfrom dotenv import load_dotenv\nimport time\n\n# Load environment variables\nload_dotenv()\n\nclass AITrendCrawler:\n    def __init__(self):\n        \"\"\"Initialize Bluesky API client\"\"\"\n        \n        handle = os.getenv('BLUESKY_HANDLE')\n        password = os.getenv('BLUESKY_PASSWORD')\n        \n        if not handle or not password:\n            raise ValueError(\n                \"BLUESKY_HANDLE and BLUESKY_PASSWORD must be set in .env file\\n\"\n                \"Create an app password at: https://bsky.app/settings/app-passwords\"\n            )\n        \n        # Initialize client\n        self.client = Client()\n        \n        try:\n            # Login to Bluesky\n            self.client.login(handle, password)\n            print(f\"✓ Logged in to Bluesky as @{handle}\")\n        except Exception as e:\n            raise ValueError(f\"Failed to login to Bluesky: {e}\")\n        \n        # AI-focused accounts to monitor on Bluesky\n        self.ai_accounts = [\n            'sama.bsky.social',          # Sam Altman\n            'karpathy.bsky.social',      # Andrej Karpathy\n            'emollick.bsky.social',      # Ethan Mollick\n            'simonw.bsky.social',        # Simon Willison\n            'yoheinakajima.bsky.social', # Yohei Nakajima\n        ]\n        \n        # AI-related search queries\n        self.ai_keywords = [\n            'ChatGPT',\n            'Claude',\n            'GPT-4',\n            'LLM',\n            'AI',\n            'OpenAI',\n            'Anthropic',\n            'machine learning',\n            'AGI',\n            'Gemini',\n        ]\n    \n    def fetch_author_feed(self, handle, limit=10):\n        \"\"\"Fetch recent posts from a specific author\"\"\"\n        try:\n            print(f\"   🔍 Fetching @{handle}...\")\n            \n            # Get author's feed\n            feed = self.client.get_author_feed(actor=handle, limit=limit)\n            \n            posts = []\n            for item in feed.feed:\n                post = item.post\n                posts.append({\n                    'uri': post.uri,\n                    'cid': post.cid,\n                    'text': post.record.text,\n                    'author': post.author.handle,\n                    'author_display': post.author.display_name,\n                    'created_at': post.record.created_at,\n                    'like_count': post.like_count or 0,\n                    'repost_count': post.repost_count or 0,\n                    'reply_count': post.reply_count or 0,\n                    'embed': post.embed if hasattr(post, 'embed') else None,\n                })\n            \n            print(f\"   ✓ @{handle}: Found {len(posts)} posts\")\n            return posts\n            \n        except Exception as e:\n            print(f\"   ✗ Error fetching @{handle}: {str(e)[:100]}\")\n            return []\n    \n    def search_posts(self, query, limit=25):\n        \"\"\"Search for posts containing specific keywords\"\"\"\n        try:\n            print(f\"   🔍 Searching: {query}...\")\n            \n            # Search posts\n            results = self.client.app.bsky.feed.search_posts(\n                params={\n                    'q': query,\n                    'limit': limit,\n                }\n            )\n            \n            posts = []\n            for item in results.posts:\n                posts.append({\n                    'uri': item.uri,\n                    'cid': item.cid,\n                    'text': item.record.text,\n                    'author': item.author.handle,\n                    'author_display': item.author.display_name,\n                    'created_at': item.record.created_at,\n                    'like_count': item.like_count or 0,\n                    'repost_count': item.repost_count or 0,\n                    'reply_count': item.reply_count or 0,\n                    'embed': item.embed if hasattr(item, 'embed') else None,\n                })\n            \n            print(f\"   ✓ Found {len(posts)} posts for: {query}\")\n            return posts\n            \n        except Exception as e:\n            print(f\"   ✗ Error searching '{query}': {str(e)[:100]}\")\n            return []\n    \n    def get_timeline(self, limit=50):\n        \"\"\"Get posts from your timeline (following feed)\"\"\"\n        try:\n            print(\"   🌊 Fetching timeline...\")\n            \n            timeline = self.client.get_timeline(limit=limit)\n            \n            posts = []\n            for item in timeline.feed:\n                post = item.post\n                \n                # Filter for AI-related content\n                text_lower = post.record.text.lower()\n                ai_keywords = ['ai', 'llm', 'gpt', 'chatgpt', 'claude', 'machine learning',\n                              'deep learning', 'neural', 'openai', 'anthropic']\n                \n                if any(keyword in text_lower for keyword in ai_keywords):\n                    posts.append({\n                        'uri': post.uri,\n                        'cid': post.cid,\n                        'text': post.record.text,\n                        'author': post.author.handle,\n                        'author_display': post.author.display_name,\n                        'created_at': post.record.created_at,\n                        'like_count': post.like_count or 0,\n                        'repost_count': post.repost_count or 0,\n                        'reply_count': post.reply_count or 0,\n                        'embed': post.embed if hasattr(post, 'embed') else None,\n                    })\n            \n            print(f\"   ✓ Found {len(posts)} AI-related posts in timeline\")\n            return posts\n            \n        except Exception as e:\n            print(f\"   ✗ Error fetching timeline: {str(e)[:100]}\")\n            return []\n    \n    def get_popular_feed(self, limit=50):\n        \"\"\"Get posts from the 'What's Hot' algorithmic feed\"\"\"\n        try:\n            print(\"   🔥 Fetching popular feed...\")\n            \n            # Get popular feed\n            feed_uri = \"at://did:plc:z72i7hdynmk6r22z27h6tvur/app.bsky.feed.generator/whats-hot\"\n            popular = self.client.get_feed(feed=feed_uri, limit=limit)\n            \n            posts = []\n            for item in popular.feed:\n                post = item.post\n                \n                # Filter for AI-related content\n                text_lower = post.record.text.lower()\n                ai_keywords = ['ai', 'llm', 'gpt', 'chatgpt', 'claude', 'machine learning',\n                              'deep learning', 'neural', 'openai', 'anthropic', 'ml']\n                \n                if any(keyword in text_lower for keyword in ai_keywords):\n                    posts.append({\n                        'uri': post.uri,\n                        'cid': post.cid,\n                        'text': post.record.text,\n                        'author': post.author.handle,\n                        'author_display': post.author.display_name,\n                        'created_at': post.record.created_at,\n                        'like_count': post.like_count or 0,\n                        'repost_count': post.repost_count or 0,\n                        'reply_count': post.reply_count or 0,\n                        'embed': post.embed if hasattr(post, 'embed') else None,\n                    })\n            \n            print(f\"   ✓ Found {len(posts)} AI-related posts in popular feed\")\n            return posts\n            \n        except Exception as e:\n            print(f\"   ✗ Error fetching popular feed: {str(e)[:100]}\")\n            return []\n    \n    def calculate_engagement_score(self, post):\n        \"\"\"Calculate engagement score for ranking\"\"\"\n        # Bluesky scoring: reposts > likes > replies\n        return (\n            post['repost_count'] * 3 +\n            post['like_count'] * 2 +\n            post['reply_count'] * 1\n        )\n    \n    def classify_sentiment(self, text):\n        \"\"\"Simple sentiment classification\"\"\"\n        text_lower = text.lower()\n        \n        if any(word in text_lower for word in ['scary', 'dangerous', 'terrifying', 'fear', 'worried', 'concerning']):\n            return 'Fear/Concern'\n        elif any(word in text_lower for word in ['lol', 'haha', 'funny', '😂', '🤣', 'hilarious']):\n            return 'Humor'\n        elif any(word in text_lower for word in ['overhype', 'scam', 'skeptical', 'doubt', 'bs']):\n            return 'Skepticism'\n        elif any(word in text_lower for word in ['amazing', 'incredible', 'wow', 'mind-blowing', '🤯', 'breakthrough']):\n            return 'Excitement'\n        elif any(word in text_lower for word in ['tutorial', 'guide', 'how to', 'useful', 'practical']):\n            return 'Practical/Educational'\n        elif any(word in text_lower for word in ['research', 'paper', 'study', 'arxiv']):\n            return 'Research'\n        else:\n            return 'Discussion'\n    \n    def parse_created_at(self, created_at_str):\n        \"\"\"Parse Bluesky's datetime string to datetime object\"\"\"\n        try:\n            # Bluesky uses ISO 8601 format\n            return datetime.fromisoformat(created_at_str.replace('Z', '+00:00'))\n        except:\n            return datetime.now(timezone.utc)\n    \n    def get_post_age_str(self, created_at_str):\n        \"\"\"Get human-readable post age\"\"\"\n        post_time = self.parse_created_at(created_at_str)\n        now = datetime.now(timezone.utc)\n        age = now - post_time\n        \n        if age.total_seconds() < 3600:\n            return f\"{int(age.total_seconds() / 60)}m ago\"\n        elif age.total_seconds() < 86400:\n            return f\"{int(age.total_seconds() / 3600)}h ago\"\n        else:\n            return f\"{int(age.days)}d ago\"\n    \n    def crawl_daily_trends(self, output_file='daily_trends_bluesky.json'):\n        \"\"\"Main crawler: fetch and process daily AI trends from Bluesky\"\"\"\n        print(f\"🕷️  Starting Bluesky AI Trend Crawl at {datetime.now()}\")\n        print(\"☁️  Bluesky (AT Protocol) - The decentralized Twitter alternative\\n\")\n        \n        all_posts = []\n        \n        # 1. Fetch from AI influencer accounts\n        print(\"\\n📱 Fetching from AI influencer accounts...\")\n        for handle in self.ai_accounts:\n            posts = self.fetch_author_feed(handle, limit=5)\n            for post in posts:\n                post['source_type'] = 'account'\n                post['source_name'] = f'@{handle}'\n                all_posts.append(post)\n            time.sleep(0.5)  # Be respectful to the API\n        \n        # 2. Search for AI keywords\n        print(\"\\n🔍 Searching for AI keywords...\")\n        for keyword in self.ai_keywords[:5]:  # Limit to avoid rate limits\n            posts = self.search_posts(keyword, limit=15)\n            for post in posts:\n                post['source_type'] = 'search'\n                post['source_name'] = f'Search: {keyword}'\n                all_posts.append(post)\n            time.sleep(0.5)\n        \n        # 3. Get AI posts from popular feed\n        print(\"\\n🔥 Fetching from 'What's Hot' feed...\")\n        popular_posts = self.get_popular_feed(limit=50)\n        for post in popular_posts:\n            post['source_type'] = 'popular'\n            post['source_name'] = \"What's Hot\"\n            all_posts.append(post)\n        \n        # 4. Get AI posts from timeline (if following AI accounts)\n        print(\"\\n🌊 Checking your timeline...\")\n        timeline_posts = self.get_timeline(limit=30)\n        for post in timeline_posts:\n            post['source_type'] = 'timeline'\n            post['source_name'] = 'Timeline'\n            all_posts.append(post)\n        \n        print(f\"\\n📊 Total posts collected: {len(all_posts)}\")\n        \n        if not all_posts:\n            print(\"\\n⚠️  No posts found to analyze\")\n            return {'error': 'No posts found', 'trends': []}\n        \n        # 5. Filter recent posts (last 24 hours)\n        now = datetime.now(timezone.utc)\n        cutoff = now - timedelta(hours=24)\n        \n        recent_posts = []\n        for post in all_posts:\n            post_time = self.parse_created_at(post['created_at'])\n            if post_time >= cutoff:\n                recent_posts.append(post)\n        \n        print(f\"📊 Recent posts (last 24h): {len(recent_posts)}\")\n        \n        # 6. Remove duplicates\n        seen_uris = set()\n        unique_posts = []\n        for post in recent_posts:\n            if post['uri'] not in seen_uris:\n                seen_uris.add(post['uri'])\n                unique_posts.append(post)\n        \n        print(f\"📊 Unique posts after deduplication: {len(unique_posts)}\")\n        \n        if not unique_posts:\n            print(\"\\n⚠️  No unique recent posts found\")\n            return {'error': 'No recent posts found', 'trends': []}\n        \n        # 7. Rank by engagement\n        unique_posts.sort(\n            key=lambda p: self.calculate_engagement_score(p),\n            reverse=True\n        )\n        \n        # 8. Take top 15 and format\n        top_posts = unique_posts[:15]\n        \n        trends = []\n        print(\"\\n🔍 Processing top posts...\")\n        for post in top_posts:\n            text = post['text']\n            \n            # Build post URL\n            # Format: https://bsky.app/profile/{handle}/post/{post_id}\n            post_id = post['uri'].split('/')[-1]\n            post_url = f\"https://bsky.app/profile/{post['author']}/post/{post_id}\"\n            \n            # Check for media\n            has_media = post.get('embed') is not None\n            media_type = None\n            if has_media and post['embed']:\n                if hasattr(post['embed'], '$type'):\n                    media_type = post['embed'].type.split('.')[-1]\n            \n            trend_item = {\n                'source': f\"Bluesky - {post.get('source_name', 'AT Protocol')}\",\n                'title': (text[:97] + '...') if len(text) > 100 else text,\n                'url': post_url,\n                'full_text': text,\n                'author': f\"@{post['author']}\",\n                'author_display': post.get('author_display', ''),\n                'posted': self.get_post_age_str(post['created_at']),\n                'engagement': {\n                    'likes': post['like_count'],\n                    'reposts': post['repost_count'],\n                    'replies': post['reply_count'],\n                },\n                'sentiment_tag': self.classify_sentiment(text),\n                'has_media': has_media,\n                'media_type': media_type,\n            }\n            trends.append(trend_item)\n        \n        # 9. Create output JSON with metadata\n        output = {\n            'date': datetime.now().strftime('%Y-%m-%d'),\n            'generated_at': datetime.now().isoformat(),\n            'platform': 'Bluesky (AT Protocol)',\n            'total_analyzed': len(all_posts),\n            'recent_posts': len(recent_posts),\n            'unique_posts': len(unique_posts),\n            'accounts_processed': len(self.ai_accounts),\n            'trends_found': len(trends),\n            'trends': trends,\n            'note': 'Data collected from Bluesky using AT Protocol. Decentralized social network!'\n        }\n        \n        # 10. Save to file\n        with open(output_file, 'w', encoding='utf-8') as f:\n            json.dump(output, f, indent=2, ensure_ascii=False)\n        \n        print(f\"\\n✅ Crawl complete! Generated {len(trends)} trend items\")\n        print(f\"📄 Output saved to: {output_file}\")\n        \n        return output\n\n\ndef main():\n    \"\"\"Run the crawler\"\"\"\n    try:\n        crawler = AITrendCrawler()\n        result = crawler.crawl_daily_trends()\n        \n        # Print sample results\n        print(\"\\n\" + \"=\"*60)\n        print(\"📊 SAMPLE RESULTS\")\n        print(\"=\"*60)\n        \n        if result.get('trends'):\n            print(f\"\\nFound {len(result['trends'])} trends\\n\")\n            \n            # Show first 3 trends\n            for i, trend in enumerate(result['trends'][:3], 1):\n                print(f\"\\n--- Trend #{i} ---\")\n                print(f\"Title: {trend['title']}\")\n                print(f\"Author: {trend['author']} ({trend['author_display']})\")\n                print(f\"Posted: {trend['posted']}\")\n                print(f\"Engagement: {trend['engagement']['likes']} likes, {trend['engagement']['reposts']} reposts\")\n                print(f\"Sentiment: {trend['sentiment_tag']}\")\n                print(f\"URL: {trend['url']}\")\n                if trend['has_media']:\n                    print(f\"Media: {trend['media_type']}\")\n        else:\n            print(\"No trends found.\")\n            print(f\"Error: {result.get('error', 'Unknown error')}\")\n        \n    except Exception as e:\n        print(f\"❌ Error: {e}\")\n        import traceback\n        traceback.print_exc()\n        return 1\n    \n    return 0\n\n\nif __name__ == '__main__':\n    import sys\n    sys.exit(main())",
      "size": 17194,
      "language": "python"
    },
    "src/social/bluesky/client.py": {
      "content": "\"\"\"Bluesky API client for fetching and processing posts.\"\"\"\nimport os\nimport json\nimport time\nfrom typing import Dict, List, Optional, Any\nfrom datetime import datetime, timezone, timedelta\n\nfrom atproto import Client\nfrom dotenv import load_dotenv\n\nfrom .models import BlueskyPost, TrendItem\nfrom .utils import BlueskyUtils\n\nclass BlueskyClient:\n    \"\"\"Client for interacting with the Bluesky API.\"\"\"\n    \n    def __init__(self):\n        \"\"\"Initialize Bluesky API client.\"\"\"\n        load_dotenv()\n        \n        handle = os.getenv('BLUESKY_HANDLE')\n        password = os.getenv('BLUESKY_PASSWORD')\n        \n        if not handle or not password:\n            raise ValueError(\n                \"BLUESKY_HANDLE and BLUESKY_PASSWORD must be set in .env file\\n\"\n                \"Create an app password at: https://bsky.app/settings/app-passwords\"\n            )\n        \n        self.client = Client()\n        self.utils = BlueskyUtils()\n        \n        try:\n            self.client.login(handle, password)\n            print(f\"✓ Logged in to Bluesky as @{handle}\")\n        except Exception as e:\n            raise ValueError(f\"Failed to login to Bluesky: {e}\")\n        \n        # AI-focused accounts to monitor on Bluesky\n        self.ai_accounts = [\n            'sama.bsky.social',          # Sam Altman\n            'karpathy.bsky.social',      # Andrej Karpathy\n            'emollick.bsky.social',      # Ethan Mollick\n            'simonw.bsky.social',        # Simon Willison\n            'yoheinakajima.bsky.social', # Yohei Nakajima\n        ]\n        \n        # AI-related search queries\n        self.ai_keywords = [\n            'ChatGPT', 'Claude', 'GPT-4', 'LLM', 'AI',\n            'OpenAI', 'Anthropic', 'machine learning', 'AGI', 'Gemini',\n        ]\n    \n    def fetch_author_feed(self, handle: str, limit: int = 10) -> List[Dict[str, Any]]:\n        \"\"\"Fetch recent posts from a specific author.\n        \n        Args:\n            handle: Bluesky handle (without @)\n            limit: Maximum number of posts to fetch\n            \n        Returns:\n            List of post dictionaries\n        \"\"\"\n        try:\n            print(f\"   🔍 Fetching @{handle}...\")\n            \n            feed = self.client.get_author_feed(actor=handle, limit=limit)\n            posts = []\n            \n            for item in feed.feed:\n                post = item.post\n                posts.append({\n                    'uri': post.uri,\n                    'cid': post.cid,\n                    'text': post.record.text,\n                    'author': post.author.handle,\n                    'author_display': post.author.display_name,\n                    'created_at': post.record.created_at,\n                    'like_count': post.like_count or 0,\n                    'repost_count': post.repost_count or 0,\n                    'reply_count': post.reply_count or 0,\n                    'embed': post.embed if hasattr(post, 'embed') else None,\n                })\n            \n            print(f\"   ✓ @{handle}: Found {len(posts)} posts\")\n            return posts\n            \n        except Exception as e:\n            print(f\"   ✗ Error fetching @{handle}: {str(e)[:100]}\")\n            return []\n    \n    def search_posts(self, query: str, limit: int = 25) -> List[Dict[str, Any]]:\n        \"\"\"Search for posts containing specific keywords.\n        \n        Args:\n            query: Search query string\n            limit: Maximum number of results to return\n            \n        Returns:\n            List of matching posts with metadata\n        \"\"\"\n        try:\n            print(f\"   🔍 Searching: {query}...\")\n            \n            results = self.client.app.bsky.feed.search_posts(\n                params={'q': query, 'limit': min(limit, 100)}  # API limit is 100\n            )\n            \n            posts = []\n            for item in results.posts:\n                posts.append({\n                    'uri': item.uri,\n                    'cid': item.cid,\n                    'text': item.record.text,\n                    'author': item.author.handle,\n                    'author_display': item.author.display_name,\n                    'created_at': item.record.created_at,\n                    'like_count': getattr(item, 'like_count', 0) or 0,\n                    'repost_count': getattr(item, 'repost_count', 0) or 0,\n                    'reply_count': getattr(item, 'reply_count', 0) or 0,\n                    'embed': getattr(item, 'embed', None)\n                })\n            \n            print(f\"   ✓ Found {len(posts)} posts for: {query}\")\n            return posts\n            \n        except Exception as e:\n            print(f\"   ✗ Error searching '{query}': {str(e)[:100]}\")\n            return []\n\n    def get_timeline(self, limit: int = 50) -> List[Dict[str, Any]]:\n        \"\"\"Get posts from your timeline (following feed).\n        \n        Args:\n            limit: Maximum number of posts to return\n            \n        Returns:\n            List of posts from your timeline\n        \"\"\"\n        try:\n            print(\"   🌊 Fetching timeline...\")\n            \n            # Get timeline and filter for AI-related content\n            timeline = self.client.get_timeline(limit=limit)\n            ai_keywords = ['ai', 'llm', 'gpt', 'chatgpt', 'claude', \n                          'machine learning', 'deep learning', 'neural', \n                          'openai', 'anthropic']\n            \n            posts = []\n            for item in timeline.feed:\n                post = item.post\n                text_lower = post.record.text.lower()\n                \n                if any(keyword in text_lower for keyword in ai_keywords):\n                    posts.append({\n                        'uri': post.uri,\n                        'cid': post.cid,\n                        'text': post.record.text,\n                        'author': post.author.handle,\n                        'author_display': post.author.display_name,\n                        'created_at': post.record.created_at,\n                        'like_count': getattr(post, 'like_count', 0) or 0,\n                        'repost_count': getattr(post, 'repost_count', 0) or 0,\n                        'reply_count': getattr(post, 'reply_count', 0) or 0,\n                        'embed': getattr(post, 'embed', None)\n                    })\n            \n            print(f\"   ✓ Found {len(posts)} AI-related posts in timeline\")\n            return posts\n            \n        except Exception as e:\n            print(f\"   ✗ Error fetching timeline: {str(e)[:100]}\")\n            return []\n\n    def get_popular_feed(self, limit: int = 50) -> List[Dict[str, Any]]:\n        \"\"\"Get posts from the 'What's Hot' algorithmic feed.\n        \n        Args:\n            limit: Maximum number of posts to return\n            \n        Returns:\n            List of popular posts\n        \"\"\"\n        try:\n            print(\"   🔥 Fetching popular feed...\")\n            \n            # Get popular feed\n            feed_uri = \"at://did:plc:z72i7hdynmk6r22z27h6tvur/app.bsky.feed.generator/whats-hot\"\n            popular = self.client.get_feed(feed=feed_uri, limit=min(limit, 100))  # API limit is 100\n            \n            # Filter for AI-related content\n            ai_keywords = ['ai', 'llm', 'gpt', 'chatgpt', 'claude', \n                          'machine learning', 'deep learning', 'neural', \n                          'openai', 'anthropic', 'ml']\n            \n            posts = []\n            for item in popular.feed:\n                post = item.post\n                text_lower = post.record.text.lower()\n                \n                if any(keyword in text_lower for keyword in ai_keywords):\n                    posts.append({\n                        'uri': post.uri,\n                        'cid': post.cid,\n                        'text': post.record.text,\n                        'author': post.author.handle,\n                        'author_display': post.author.display_name,\n                        'created_at': post.record.created_at,\n                        'like_count': getattr(post, 'like_count', 0) or 0,\n                        'repost_count': getattr(post, 'repost_count', 0) or 0,\n                        'reply_count': getattr(post, 'reply_count', 0) or 0,\n                        'embed': getattr(post, 'embed', None)\n                    })\n            \n            print(f\"   ✓ Found {len(posts)} AI-related posts in popular feed\")\n            return posts\n            \n        except Exception as e:\n            print(f\"   ✗ Error fetching popular feed: {str(e)[:100]}\")\n            return []\n\n    def calculate_engagement_score(self, post: Dict[str, Any]) -> int:\n        \"\"\"Calculate engagement score for ranking.\n        \n        Args:\n            post: Dictionary containing post data\n            \n        Returns:\n            int: Engagement score (reposts * 3 + likes * 2 + replies * 1)\n        \"\"\"\n        return (\n            post.get('repost_count', 0) * 3 +\n            post.get('like_count', 0) * 2 +\n            post.get('reply_count', 0) * 1\n        )\n\n    \n    def crawl_daily_trends(self, output_file: str = 'data/raw/daily_trends_bluesky.json') -> Dict[str, Any]:\n        \"\"\"Main method to crawl and process daily AI trends from Bluesky.\n        \n        Args:\n            output_file: Path to save the output JSON file\n            \n        Returns:\n            Dictionary containing the crawl results\n        \"\"\"\n        print(f\"🕷️  Starting Bluesky AI Trend Crawl at {datetime.now()}\")\n        print(\"☁️  Bluesky (AT Protocol) - The decentralized Twitter alternative\\n\")\n        \n        all_posts = []\n        \n        # 1. Fetch from AI influencer accounts\n        print(\"\\n📱 Fetching from AI influencer accounts...\")\n        for handle in self.ai_accounts:\n            posts = self.fetch_author_feed(handle, limit=5)\n            for post in posts:\n                post['source_type'] = 'account'\n                post['source_name'] = f'@{handle}'\n                all_posts.append(post)\n            time.sleep(0.5)  # Be respectful to the API\n        \n        # [Rest of the crawl_daily_trends implementation would go here]\n        # Let me know if you'd like to see the full implementation\n        \n        # Placeholder return for now\n        return {\n            'status': 'success',\n            'message': 'Crawl completed',\n            'output_file': output_file\n        }\n",
      "size": 10372,
      "language": "python"
    },
    "src/social/bluesky/utils/__init__.py": {
      "content": "\"\"\"Utility functions for the Bluesky module.\"\"\"\nfrom datetime import datetime, timezone, timedelta\nfrom typing import Dict, Any, List, Optional\n\nclass BlueskyUtils:\n    \"\"\"Utility class for Bluesky operations.\"\"\"\n    \n    @staticmethod\n    def calculate_engagement_score(post: Dict[str, Any]) -> int:\n        \"\"\"Calculate engagement score for ranking.\n        \n        Args:\n            post: Dictionary containing post data\n            \n        Returns:\n            int: Engagement score (reposts * 3 + likes * 2 + replies * 1)\n        \"\"\"\n        return (\n            post.get('repost_count', 0) * 3 +\n            post.get('like_count', 0) * 2 +\n            post.get('reply_count', 0) * 1\n        )\n    \n    @staticmethod\n    def classify_sentiment(text: str) -> str:\n        \"\"\"Simple sentiment classification.\n        \n        Args:\n            text: Text to analyze\n            \n        Returns:\n            str: Sentiment tag\n        \"\"\"\n        text_lower = text.lower()\n        \n        sentiment_keywords = {\n            'Fear/Concern': ['scary', 'dangerous', 'terrifying', 'fear', 'worried', 'concerning'],\n            'Humor': ['lol', 'haha', 'funny', '😂', '🤣', 'hilarious'],\n            'Skepticism': ['overhype', 'scam', 'skeptical', 'doubt', 'bs'],\n            'Excitement': ['amazing', 'incredible', 'wow', 'mind-blowing', '🤯', 'breakthrough'],\n            'Practical/Educational': ['tutorial', 'guide', 'how to', 'useful', 'practical'],\n            'Research': ['research', 'paper', 'study', 'arxiv']\n        }\n        \n        for sentiment, keywords in sentiment_keywords.items():\n            if any(keyword in text_lower for keyword in keywords):\n                return sentiment\n                \n        return 'Discussion'\n    \n    @staticmethod\n    def parse_created_at(created_at_str: str) -> datetime:\n        \"\"\"Parse Bluesky's datetime string to datetime object.\n        \n        Args:\n            created_at_str: ISO 8601 formatted datetime string\n            \n        Returns:\n            datetime: Parsed datetime object\n        \"\"\"\n        try:\n            # Handle both 'Z' and timezone offset formats\n            if created_at_str.endswith('Z'):\n                return datetime.fromisoformat(created_at_str.replace('Z', '+00:00'))\n            return datetime.fromisoformat(created_at_str)\n        except (ValueError, TypeError):\n            return datetime.now(timezone.utc)\n    \n    @staticmethod\n    def get_post_age_str(created_at_str: str) -> str:\n        \"\"\"Get human-readable post age.\n        \n        Args:\n            created_at_str: ISO 8601 formatted datetime string\n            \n        Returns:\n            str: Human-readable time difference (e.g., '2h ago', '3d ago')\n        \"\"\"\n        post_time = BlueskyUtils.parse_created_at(created_at_str)\n        now = datetime.now(timezone.utc)\n        age = now - post_time\n        \n        if age.total_seconds() < 3600:  # Less than 1 hour\n            minutes = int(age.total_seconds() / 60)\n            return f\"{minutes}m ago\"\n        elif age.total_seconds() < 86400:  # Less than 1 day\n            hours = int(age.total_seconds() / 3600)\n            return f\"{hours}h ago\"\n        else:  # 1 or more days\n            return f\"{age.days}d ago\"\n",
      "size": 3239,
      "language": "python"
    },
    "src/social/bluesky/models/__init__.py": {
      "content": "\"\"\"Data models for Bluesky API interactions.\"\"\"\nfrom dataclasses import dataclass\nfrom datetime import datetime\nfrom typing import Optional, Dict, Any, List\n\n@dataclass\nclass BlueskyPost:\n    \"\"\"Represents a post from Bluesky.\"\"\"\n    uri: str\n    cid: str\n    text: str\n    author: str\n    author_display: str\n    created_at: str\n    like_count: int\n    repost_count: int\n    reply_count: int\n    embed: Optional[Dict[str, Any]] = None\n    source_type: Optional[str] = None\n    source_name: Optional[str] = None\n\n@dataclass\nclass TrendItem:\n    \"\"\"Represents a trending topic from Bluesky.\"\"\"\n    source: str\n    title: str\n    url: str\n    full_text: str\n    author: str\n    author_display: str\n    posted: str\n    engagement: Dict[str, int]\n    sentiment_tag: str\n    has_media: bool\n    media_type: Optional[str]\n",
      "size": 816,
      "language": "python"
    }
  },
  "_cache_metadata": {
    "url": "https://github.com/ronelsolomon/crawlerx.git",
    "content_type": "github",
    "cached_at": "2026-03-02T22:49:13.773384",
    "cache_key": "695c3097bae6d039bfccf0a470613925"
  }
}