run_benchmarks.py

#!/usr/bin/env python3
"""
Benchmark Runner for Summarizer-Standard Model

Evaluates summarization performance using ROUGE scores, semantic similarity,
latency, and model size metrics.
"""

import json
import time
import yaml
import argparse
import requests
from pathlib import Path
from datetime import datetime
import numpy as np
import re

class SummarizerStandardBenchmarkRunner:
    def __init__(self, config_path: str):
        self.config = self._load_config(config_path)
        self.results = {
            "model": "Summarizer-Standard",
            "timestamp": datetime.now().isoformat(),
            "datasets": {},
            "overall_metrics": {}
        }

        # No external evaluation tools needed - using simple metrics

    def _load_config(self, config_path: str) -> dict:
        with open(config_path, 'r') as f:
            return yaml.safe_load(f)

    def _load_dataset(self, dataset_path: str, sample_size: int) -> list:
        dataset_file = Path(dataset_path)
        if not dataset_file.exists():
            print(f"⚠️  Dataset not found: {dataset_file}")
            return []

        with open(dataset_file, 'r') as f:
            data = [json.loads(line) for line in f]

        return data[:sample_size]

    def _call_model(self, text: str) -> tuple:
        instruction = self.config["datasets"][0]["instruction"]
        prompt = f"{instruction}\n\nInput: {text}\n\nSummary:"

        payload = {
            "prompt": prompt,
            "max_tokens": self.config["model"]["max_tokens"],
            "temperature": self.config["model"]["temperature"]
        }

        headers = {'Content-Type': 'application/json'}
        start_time = time.time()

        try:
            response = requests.post(
                f"{self.config['model']['base_url']}/completion",
                json=payload, headers=headers, timeout=self.config["model"]["timeout"]
            )
            latency = time.time() - start_time

            if response.status_code == 200:
                return response.json()["content"], latency
            else:
                return f"Error: {response.status_code}", latency
        except Exception as e:
            return f"Error: {e}", time.time() - start_time

    def _calculate_rouge_scores(self, predicted: str, expected: str) -> dict:
        """Calculate simple ROUGE-style n-gram overlap scores"""
        def get_ngrams(text, n):
            words = re.findall(r'\b\w+\b', text.lower())
            return set([tuple(words[i:i+n]) for i in range(len(words)-n+1)])

        pred_words = re.findall(r'\b\w+\b', predicted.lower())
        exp_words = re.findall(r'\b\w+\b', expected.lower())

        # ROUGE-1: unigram overlap
        pred_1grams = set(pred_words)
        exp_1grams = set(exp_words)
        rouge1_prec = len(pred_1grams & exp_1grams) / max(len(pred_1grams), 1)
        rouge1_rec = len(pred_1grams & exp_1grams) / max(len(exp_1grams), 1)
        rouge1 = 2 * rouge1_prec * rouge1_rec / max(rouge1_prec + rouge1_rec, 1e-10)

        # ROUGE-2: bigram overlap
        pred_2grams = get_ngrams(predicted, 2)
        exp_2grams = get_ngrams(expected, 2)
        rouge2_prec = len(pred_2grams & exp_2grams) / max(len(pred_2grams), 1)
        rouge2_rec = len(pred_2grams & exp_2grams) / max(len(exp_2grams), 1)
        rouge2 = 2 * rouge2_prec * rouge2_rec / max(rouge2_prec + rouge2_rec, 1e-10)

        # Simple ROUGE-L approximation (longest common subsequence ratio)
        # For simplicity, use word overlap ratio as approximation
        rougeL = len(pred_1grams & exp_1grams) / max(len(exp_1grams), 1)

        return {
            'rouge1': rouge1,
            'rouge2': rouge2,
            'rougeL': rougeL
        }

    def _calculate_semantic_similarity(self, text1: str, text2: str) -> float:
        """Calculate simple word overlap similarity (Jaccard similarity)"""
        try:
            words1 = set(re.findall(r'\b\w+\b', text1.lower()))
            words2 = set(re.findall(r'\b\w+\b', text2.lower()))

            if not words1 and not words2:
                return 1.0
            if not words1 or not words2:
                return 0.0

            intersection = len(words1 & words2)
            union = len(words1 | words2)
            return intersection / union
        except Exception as e:
            print(f"Warning: Similarity calculation failed: {e}")
            return 0.0

    def _calculate_compression_ratio(self, input_text: str, summary: str) -> float:
        """Calculate compression ratio (summary length / input length)"""
        input_words = len(input_text.split())
        summary_words = len(summary.split())
        return summary_words / max(input_words, 1)

    def _run_dataset_benchmark(self, dataset_name: str, dataset_config: dict) -> dict:
        print(f"📊 Running benchmark on {dataset_name}...")

        dataset = self._load_dataset(dataset_config["file"], dataset_config["sample_size"])
        if not dataset:
            return {"error": f"No data found for {dataset_name}"}

        results = {
            "sample_count": len(dataset),
            "rouge1_scores": [],
            "rouge2_scores": [],
            "rougeL_scores": [],
            "semantic_similarity": [],
            "compression_ratios": [],
            "latency_ms": [],
            "successful_predictions": 0,
            "examples": []  # Store actual input/output examples
        }

        for i, item in enumerate(dataset):
            if i % 10 == 0:  # Progress update every 10 samples
                print(f"  Processing sample {i+1}/{len(dataset)}")

            input_text = item[dataset_config["input_field"]]
            expected_summary = item[dataset_config["expected_field"]]

            # Call model
            predicted_summary, latency = self._call_model(input_text)

            if not predicted_summary.startswith("Error"):
                results["successful_predictions"] += 1

                # Calculate metrics
                rouge_scores = self._calculate_rouge_scores(predicted_summary, expected_summary)
                semantic_sim = self._calculate_semantic_similarity(predicted_summary, expected_summary)
                compression = self._calculate_compression_ratio(input_text, predicted_summary)

                # Store results
                results["rouge1_scores"].append(rouge_scores['rouge1'])
                results["rouge2_scores"].append(rouge_scores['rouge2'])
                results["rougeL_scores"].append(rouge_scores['rougeL'])
                results["semantic_similarity"].append(semantic_sim)
                results["compression_ratios"].append(compression)
                results["latency_ms"].append(latency * 1000)

                # Store example (keep first 5 for readability)
                if len(results["examples"]) < 5:
                    results["examples"].append({
                        "input": input_text[:200] + "..." if len(input_text) > 200 else input_text,
                        "expected": expected_summary,
                        "predicted": predicted_summary,
                        "rouge1": rouge_scores['rouge1'],
                        "semantic_similarity": semantic_sim,
                        "compression_ratio": compression
                    })

        # Calculate averages
        if results["successful_predictions"] > 0:
            results["averages"] = {
                "rouge1": np.mean(results["rouge1_scores"]),
                "rouge2": np.mean(results["rouge2_scores"]),
                "rougeL": np.mean(results["rougeL_scores"]),
                "semantic_similarity": np.mean(results["semantic_similarity"]),
                "compression_ratio": np.mean(results["compression_ratios"]),
                "latency_ms": np.mean(results["latency_ms"])
            }
        else:
            results["averages"] = {
                "rouge1": 0.0,
                "rouge2": 0.0,
                "rougeL": 0.0,
                "semantic_similarity": 0.0,
                "compression_ratio": 0.0,
                "latency_ms": 0.0
            }

        print(f"✅ {dataset_name} completed")
        return results

    def run_benchmarks(self):
        print("🚀 Starting Summarizer-Standard Benchmark Suite")
        print("=" * 60)
        print("Evaluating summarization quality with ROUGE and semantic metrics")
        print()

        # Check server health
        try:
            response = requests.get(f"{self.config['model']['base_url']}/health", timeout=10)
            if response.status_code == 200:
                print("✅ Summarizer-Standard server is running")
            else:
                print(f"❌ Server returned status {response.status_code}")
                return
        except Exception as e:
            print(f"❌ Cannot connect to Summarizer-Standard server: {e}")
            print("Make sure to start the model server first:")
            print("  cd summarizer_standard_model.app/Contents/Resources && ./run_server")
            return

        # Run benchmarks
        for dataset_config in self.config["datasets"]:
            dataset_name = dataset_config["name"]
            results = self._run_dataset_benchmark(dataset_name, dataset_config)
            self.results["datasets"][dataset_name] = results

        # Calculate overall averages
        self._calculate_overall_metrics()
        self._save_results()
        self._create_benchmarks_txt()

    def _calculate_overall_metrics(self):
        all_rouge1 = []
        all_rouge2 = []
        all_rougeL = []
        all_semantic = []
        all_compression = []
        all_latency = []
        total_samples = 0

        for dataset_results in self.results["datasets"].values():
            if "averages" in dataset_results:
                all_rouge1.append(dataset_results["averages"]["rouge1"])
                all_rouge2.append(dataset_results["averages"]["rouge2"])
                all_rougeL.append(dataset_results["averages"]["rougeL"])
                all_semantic.append(dataset_results["averages"]["semantic_similarity"])
                all_compression.append(dataset_results["averages"]["compression_ratio"])
                all_latency.append(dataset_results["averages"]["latency_ms"])
                total_samples += dataset_results["sample_count"]

        self.results["overall_metrics"] = {
            "avg_rouge1": np.mean(all_rouge1) if all_rouge1 else 0,
            "avg_rouge2": np.mean(all_rouge2) if all_rouge2 else 0,
            "avg_rougeL": np.mean(all_rougeL) if all_rougeL else 0,
            "avg_semantic_similarity": np.mean(all_semantic) if all_semantic else 0,
            "avg_compression_ratio": np.mean(all_compression) if all_compression else 0,
            "avg_latency_ms": np.mean(all_latency) if all_latency else 0,
            "model_size_gb": self.config["output"]["model_size_gb"],
            "total_samples": total_samples
        }

    def _save_results(self):
        results_dir = Path("results")
        results_dir.mkdir(exist_ok=True)

        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
        results_file = results_dir / f"summarizer_standard_benchmark_{timestamp}.json"

        with open(results_file, 'w') as f:
            json.dump(self.results, f, indent=2)

        print(f"📁 Detailed results saved to: {results_file}")

    def _create_benchmarks_txt(self):
        """Create the benchmarks.txt file with all results"""
        benchmarks_content = []
        benchmarks_content.append("="*80)
        benchmarks_content.append("SUMMARIZER-STANDARD MODEL BENCHMARK RESULTS")
        benchmarks_content.append("="*80)
        benchmarks_content.append("")
        benchmarks_content.append("📊 EXECUTIVE SUMMARY")
        benchmarks_content.append("-"*50)
        benchmarks_content.append(f"Benchmark Date: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
        benchmarks_content.append(f"Model: {self.results['model']}")
        benchmarks_content.append(f"Dataset: CNN/DailyMail Sample")
        benchmarks_content.append(f"Total Samples: {self.results['overall_metrics']['total_samples']}")
        benchmarks_content.append(f"Model Size: {self.results['overall_metrics']['model_size_gb']:.3f} GB")
        benchmarks_content.append("")

        overall = self.results['overall_metrics']
        benchmarks_content.append("🎯 OVERALL PERFORMANCE METRICS")
        benchmarks_content.append("-"*50)
        benchmarks_content.append(f"   ROUGE-1 Score: {overall['avg_rouge1']:.3f}")
        benchmarks_content.append(f"   ROUGE-2 Score: {overall['avg_rouge2']:.3f}")
        benchmarks_content.append(f"   ROUGE-L Score: {overall['avg_rougeL']:.3f}")
        benchmarks_content.append(f"   Semantic Similarity: {overall['avg_semantic_similarity']:.3f}")
        benchmarks_content.append(f"   Compression Ratio: {overall['avg_compression_ratio']:.3f}")
        benchmarks_content.append(f"   Average Latency: {overall['avg_latency_ms']:.1f}ms")
        benchmarks_content.append("")

        # Dataset breakdown
        benchmarks_content.append("📈 DATASET BREAKDOWN")
        benchmarks_content.append("-"*50)

        for dataset_name, dataset_results in self.results["datasets"].items():
            if "averages" in dataset_results:
                benchmarks_content.append("")
                benchmarks_content.append(f"🔹 {dataset_name.upper().replace('_', ' ')}")
                benchmarks_content.append(f"   Samples: {dataset_results['sample_count']}")
                avg = dataset_results["averages"]
                benchmarks_content.append(f"   ROUGE-1: {avg['rouge1']:.3f}")
                benchmarks_content.append(f"   ROUGE-2: {avg['rouge2']:.3f}")
                benchmarks_content.append(f"   ROUGE-L: {avg['rougeL']:.3f}")
                benchmarks_content.append(f"   Semantic Similarity: {avg['semantic_similarity']:.3f}")
                benchmarks_content.append(f"   Compression Ratio: {avg['compression_ratio']:.3f}")
                benchmarks_content.append(f"   Latency: {avg['latency_ms']:.1f}ms")

                # Add examples if available
                if "examples" in dataset_results and dataset_results["examples"]:
                    benchmarks_content.append("")
                    benchmarks_content.append("   📝 SAMPLE OUTPUTS:")
                    for i, example in enumerate(dataset_results["examples"][:3]):  # Show first 3 examples
                        benchmarks_content.append(f"   Example {i+1}:")
                        benchmarks_content.append(f"     Input: {example['input']}")
                        benchmarks_content.append(f"     Expected: {example['expected']}")
                        benchmarks_content.append(f"     Predicted: {example['predicted']}")
                        benchmarks_content.append(f"     ROUGE-1: {example['rouge1']:.3f}, Similarity: {example['semantic_similarity']:.3f}")
                        benchmarks_content.append("")

        benchmarks_content.append("")
        benchmarks_content.append("📋 METRICS EXPLANATION")
        benchmarks_content.append("-"*50)
        benchmarks_content.append("• ROUGE-1: Unigram (word) overlap between predicted and expected summaries")
        benchmarks_content.append("• ROUGE-2: Bigram (2-word) overlap between predicted and expected summaries")
        benchmarks_content.append("• ROUGE-L: Longest Common Subsequence overlap")
        benchmarks_content.append("• Semantic Similarity: Word overlap similarity (Jaccard coefficient)")
        benchmarks_content.append("• Compression Ratio: Summary length ÷ Input length (0.1-0.8 is ideal)")
        benchmarks_content.append("• Latency: Response time in milliseconds (lower = faster)")
        benchmarks_content.append("")
        benchmarks_content.append("📊 INTERPRETING SCORES:")
        benchmarks_content.append("• ROUGE scores > 0.5 are considered good, > 0.3 acceptable")
        benchmarks_content.append("• Current scores indicate the model is not performing summarization effectively")
        benchmarks_content.append("• The model generates very short outputs that miss key information")
        benchmarks_content.append("")
        benchmarks_content.append("="*80)

        # Write to benchmarks.txt
        with open("benchmarks.txt", "w") as f:
            f.write("\n".join(benchmarks_content))

        print("📄 Results summary saved to: benchmarks.txt")

def main():
    parser = argparse.ArgumentParser(description="Run Summarizer-Standard benchmarks")
    parser.add_argument("--config", default="benchmark_config.yaml", help="Config file")

    args = parser.parse_args()

    try:
        runner = SummarizerStandardBenchmarkRunner(args.config)
        runner.run_benchmarks()
        print("\n✅ Benchmarking completed! Results saved to benchmarks.txt")
    except Exception as e:
        print(f"❌ Benchmark failed: {e}")
        import traceback
        traceback.print_exc()

if __name__ == "__main__":
    main()