utils.py

import numpy as np
from typing import List, Dict, Any

def preprocess_text(text: str) -> str:
    """Basic text preprocessing"""
    # Add your custom preprocessing logic here
    text = text.strip()
    text = ' '.join(text.split())  # Remove extra whitespace
    return text

def batch_predict(texts: List[str], classifier, batch_size: int = 32):
    """Process predictions in batches"""
    results = []
    for i in range(0, len(texts), batch_size):
        batch = texts[i:i + batch_size]
        batch_results = classifier(batch)
        results.extend(batch_results)
    return results

def confidence_score(predictions: List[Dict]) -> List[Dict]:
    """Add confidence scores to predictions"""
    for pred in predictions:
        if 'score' in pred:
            pred['confidence'] = f"{pred['score']*100:.2f}%"
    return predictions