new passage-query dataset style

README.md

@@ -27,6 +27,8 @@ pip install -r requirements.txt
 
 ## Usage
 
+### Basic
+
 ```bash
 # Full benchmark (quality + speed + memory)
 python bench.py
@@ -45,14 +47,89 @@ python bench.py --skip-memory
 python bench.py --corpus-size 500 --batch-size 32 --num-runs 5
 ```
 
+### Datasets
+
+By default, quality is evaluated on the STS Benchmark. You can evaluate on multiple HuggingFace datasets using built-in presets:
+
+| Preset | HF Dataset | Type | Pairs |
+|--------|-----------|------|-------|
+| `sts` | `mteb/stsbenchmark-sts` | Scored (Spearman) | 1,379 |
+| `natural-questions` | `sentence-transformers/natural-questions` | Retrieval (MRR/Recall) | 100,231 |
+| `msmarco` | `sentence-transformers/msmarco-bm25` | Retrieval | 503,000 |
+| `squad` | `sentence-transformers/squad` | Retrieval | 87,599 |
+| `trivia-qa` | `sentence-transformers/trivia-qa` | Retrieval | 73,346 |
+| `gooaq` | `sentence-transformers/gooaq` | Retrieval | 3,012,496 |
+| `hotpotqa` | `sentence-transformers/hotpotqa` | Retrieval | 84,500 |
+
+```bash
+# Evaluate on multiple datasets
+python bench.py --models mpnet bge-small \
+  --datasets sts natural-questions squad \
+  --skip-speed --skip-memory
+
+# Limit pairs for large datasets
+python bench.py --datasets msmarco gooaq --max-pairs 1000
+
+# Use a custom HF dataset (overrides --datasets)
+python bench.py --dataset my-org/my-pairs \
+  --query-col query --passage-col passage --score-col none
+```
+
+Scored datasets (with `--score-col`) report **Spearman correlation**. Pair-only datasets (`--score-col none`) report **MRR**, **Recall@1**, **Recall@5**, and **Recall@10**.
+
+### Export results
+
+```bash
+# Export to CSV
+python bench.py --csv results.csv
+
+# Save charts as PNG
+python bench.py --charts ./results
+
+# Both
+python bench.py --models mpnet bge-small \
+  --datasets sts squad natural-questions \
+  --max-pairs 1000 \
+  --csv results.csv --charts ./results
+```
+
+Charts generated:
+- `quality_<dataset>.png` — Spearman bar chart (scored) or grouped MRR/Recall bars (retrieval)
+- `speed.png` — sentences/second comparison
+- `memory.png` — peak memory usage comparison
+
 ## Metrics
 
 | Dimension | Metric | Method |
 |-----------|--------|--------|
-| Quality | Spearman rho | STS Benchmark test set (1,379 pairs) |
+| Quality (scored) | Spearman rho | Cosine similarity vs gold scores |
+| Quality (pairs) | MRR, Recall@k | Retrieval ranking of positive passages |
 | Speed | Median encode time | Wall-clock over N runs with warmup |
 | Memory | Peak RSS delta | Isolated subprocess via `psutil` |
 
+## CLI reference
+
+```
+--models            Models to benchmark (default: all)
+--corpus-size       Sentences for speed/memory tests (default: 1000)
+--batch-size        Encoding batch size (default: 64)
+--num-runs          Speed benchmark runs (default: 3)
+--skip-quality      Skip quality evaluation
+--skip-speed        Skip speed measurement
+--skip-memory       Skip memory measurement
+--datasets          Dataset presets (default: sts)
+--max-pairs         Limit pairs per dataset
+--dataset           Custom HF dataset (overrides --datasets)
+--config            Dataset config/subset name (e.g. 'triplet')
+--split             Dataset split (default: test)
+--query-col         Query column name (default: sentence1)
+--passage-col       Passage column name (default: sentence2)
+--score-col         Score column (default: score, 'none' for pairs)
+--score-scale       Score normalization divisor (default: 5.0)
+--csv               Export results to CSV
+--charts            Save charts to directory
+```
+
 ## Adding a model
 
 Edit `models.py` and add an entry to `REGISTRY`:
@@ -76,14 +153,15 @@ Edit `models.py` and add an entry to `REGISTRY`:
 
 ```
 embedding-bench/
-├── bench.py           # CLI entry point
-├── models.py          # Model registry
-├── wrapper.py         # Backend wrappers (sbert, fastembed, gguf)
-├── corpus.py          # Sentence corpus builder
-├── report.py          # Table formatting
+├── bench.py             # CLI entry point
+├── models.py            # Model registry
+├── wrapper.py           # Backend wrappers (sbert, fastembed, gguf)
+├── corpus.py            # Sentence corpus builder
+├── dataset_config.py    # Dataset presets and configuration
+├── report.py            # Table formatting, CSV export, charts
 ├── evals/
-│   ├── quality.py     # STS Benchmark evaluation
-│   ├── speed.py       # Latency measurement
-│   └── memory.py      # Memory measurement
+│   ├── quality.py       # STS + retrieval evaluation
+│   ├── speed.py         # Latency measurement
+│   └── memory.py        # Memory measurement
 └── requirements.txt
 ```

bench.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import argparse
 
 from corpus import build_corpus
+from dataset_config import DATASET_PRESETS, DatasetConfig
 from evals import evaluate_memory, evaluate_quality, evaluate_speed
 from models import REGISTRY
 from report import print_report
@@ -28,15 +29,61 @@ def main(argv: list[str] | None = None) -> None:
     parser.add_argument("--skip-speed", action="store_true")
     parser.add_argument("--skip-memory", action="store_true")
 
+    # Dataset configuration
+    parser.add_argument(
+        "--datasets",
+        nargs="+",
+        default=["sts"],
+        choices=list(DATASET_PRESETS.keys()),
+        help=f"Dataset presets to evaluate (default: sts). "
+             f"Available: {', '.join(DATASET_PRESETS.keys())}",
+    )
+    parser.add_argument("--max-pairs", type=int, default=None,
+                        help="Limit number of pairs per dataset (useful for large datasets)")
+
+    # Custom dataset (overrides --datasets)
+    parser.add_argument("--dataset", default=None,
+                        help="Custom HF dataset name (overrides --datasets)")
+    parser.add_argument("--config", default=None,
+                        help="Dataset config/subset name (e.g. 'triplet')")
+    parser.add_argument("--split", default="test")
+    parser.add_argument("--query-col", default="sentence1")
+    parser.add_argument("--passage-col", default="sentence2")
+    parser.add_argument("--score-col", default="score",
+                        help="Score column name. Pass 'none' for pair-only datasets.")
+    parser.add_argument("--score-scale", type=float, default=5.0)
+
+    # Output options
+    parser.add_argument("--csv", default=None, metavar="PATH",
+                        help="Export results to a CSV file")
+    parser.add_argument("--charts", default=None, metavar="DIR",
+                        help="Save charts to a directory (e.g. ./results)")
+
     args = parser.parse_args(argv)
 
+    # Build list of dataset configs
+    if args.dataset:
+        # Custom dataset overrides presets
+        ds_configs = [DatasetConfig(
+            name=args.dataset,
+            config=args.config,
+            split=args.split,
+            query_col=args.query_col,
+            passage_col=args.passage_col,
+            score_col=None if args.score_col.lower() == "none" else args.score_col,
+            score_scale=args.score_scale,
+        )]
+    else:
+        ds_configs = [DATASET_PRESETS[k] for k in args.datasets]
+
     configs = [REGISTRY[k] for k in args.models]
     baseline_name = next((c.name for c in configs if c.is_baseline), None)
 
+    # Use first dataset for corpus building
     corpus: list[str] | None = None
     if not args.skip_speed or not args.skip_memory:
         print(f"Preparing corpus ({args.corpus_size} sentences)...")
-        corpus = build_corpus(args.corpus_size)
+        corpus = build_corpus(args.corpus_size, ds_configs[0])
 
     results = []
     for cfg in configs:
@@ -47,10 +94,16 @@ def main(argv: list[str] | None = None) -> None:
         result: dict = {"name": cfg.name, "is_baseline": cfg.is_baseline}
 
         if not args.skip_quality:
-            print("  Evaluating quality (STS Benchmark)...")
             model = load_model(cfg)
-            result["quality"] = evaluate_quality(model)
-            print(f"  Quality: {result['quality']:.4f}")
+            quality_results = {}
+            for ds_cfg in ds_configs:
+                ds_key = ds_cfg.name.split("/")[-1]
+                print(f"  Evaluating quality on {ds_cfg.name}...")
+                quality_results[ds_key] = evaluate_quality(
+                    model, ds_cfg, max_pairs=args.max_pairs,
+                )
+                print(f"    {quality_results[ds_key]}")
+            result["quality"] = quality_results
             del model
 
         if not args.skip_speed and corpus is not None:
@@ -67,7 +120,8 @@ def main(argv: list[str] | None = None) -> None:
 
         results.append(result)
 
-    print_report(results, baseline_name=baseline_name)
+    print_report(results, baseline_name=baseline_name,
+                 csv_path=args.csv, chart_dir=args.charts)
 
 
 if __name__ == "__main__":

corpus.py

@@ -2,11 +2,15 @@ from __future__ import annotations
 
 from datasets import load_dataset
 
+from dataset_config import DatasetConfig
 
-def build_corpus(size: int) -> list[str]:
-    """Build a corpus of real sentences from the STS Benchmark dataset."""
-    dataset = load_dataset("mteb/stsbenchmark-sts", split="test")
-    sentences = list(dataset["sentence1"]) + list(dataset["sentence2"])
+
+def build_corpus(size: int, ds_cfg: DatasetConfig | None = None) -> list[str]:
+    """Build a corpus of real sentences from the configured dataset."""
+    if ds_cfg is None:
+        ds_cfg = DatasetConfig()
+    dataset = load_dataset(ds_cfg.name, ds_cfg.config, split=ds_cfg.split)
+    sentences = list(dataset[ds_cfg.query_col]) + list(dataset[ds_cfg.passage_col])
     full: list[str] = []
     while len(full) < size:
         full.extend(sentences)

dataset_config.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass
+class DatasetConfig:
+    """Configuration for the evaluation dataset."""
+
+    name: str = "mteb/stsbenchmark-sts"
+    config: str | None = None
+    split: str = "test"
+    query_col: str = "sentence1"
+    passage_col: str = "sentence2"
+    score_col: str | None = "score"
+    score_scale: float = 5.0
+
+
+DATASET_PRESETS: dict[str, DatasetConfig] = {
+    "sts": DatasetConfig(
+        name="mteb/stsbenchmark-sts",
+        split="test",
+        query_col="sentence1",
+        passage_col="sentence2",
+        score_col="score",
+        score_scale=5.0,
+    ),
+    "natural-questions": DatasetConfig(
+        name="sentence-transformers/natural-questions",
+        split="train",
+        query_col="query",
+        passage_col="answer",
+        score_col=None,
+    ),
+    "msmarco": DatasetConfig(
+        name="sentence-transformers/msmarco-bm25",
+        config="triplet",
+        split="train",
+        query_col="query",
+        passage_col="positive",
+        score_col=None,
+    ),
+    "squad": DatasetConfig(
+        name="sentence-transformers/squad",
+        split="train",
+        query_col="question",
+        passage_col="answer",
+        score_col=None,
+    ),
+    "trivia-qa": DatasetConfig(
+        name="sentence-transformers/trivia-qa",
+        split="train",
+        query_col="query",
+        passage_col="answer",
+        score_col=None,
+    ),
+    "gooaq": DatasetConfig(
+        name="sentence-transformers/gooaq",
+        split="train",
+        query_col="question",
+        passage_col="answer",
+        score_col=None,
+    ),
+    "hotpotqa": DatasetConfig(
+        name="sentence-transformers/hotpotqa",
+        config="triplet",
+        split="train",
+        query_col="anchor",
+        passage_col="positive",
+        score_col=None,
+    ),
+}

evals/quality.py

@@ -4,21 +4,79 @@ import numpy as np
 from datasets import load_dataset
 from scipy.stats import spearmanr
 
+from dataset_config import DatasetConfig
 
-def evaluate_quality(model) -> float:
-    """Return Spearman correlation on the STS Benchmark test set."""
-    dataset = load_dataset("mteb/stsbenchmark-sts", split="test")
-    sentences1 = list(dataset["sentence1"])
-    sentences2 = list(dataset["sentence2"])
-    gold_scores = [s / 5.0 for s in dataset["score"]]
 
-    emb1 = model.encode(sentences1)
-    emb2 = model.encode(sentences2)
+def _normalize(emb: np.ndarray) -> np.ndarray:
+    """L2-normalize each row."""
+    norms = np.linalg.norm(emb, axis=1, keepdims=True)
+    return emb / norms
 
-    # Row-wise cosine similarity
-    cos_sims = np.sum(emb1 * emb2, axis=1) / (
-        np.linalg.norm(emb1, axis=1) * np.linalg.norm(emb2, axis=1)
-    )
 
-    correlation, _ = spearmanr(cos_sims, gold_scores)
-    return correlation
+def _retrieval_metrics(emb_q: np.ndarray, emb_p: np.ndarray) -> dict[str, float]:
+    """Compute MRR and Recall@k assuming query i matches passage i."""
+    emb_q = _normalize(emb_q)
+    emb_p = _normalize(emb_p)
+
+    # Similarity matrix: (num_queries, num_passages)
+    sims = emb_q @ emb_p.T
+
+    n = sims.shape[0]
+    # For each query, rank passages by descending similarity
+    # ranks[i] = rank of the correct passage (0-indexed)
+    sorted_indices = np.argsort(-sims, axis=1)
+    ranks = np.array([int(np.where(sorted_indices[i] == i)[0][0]) for i in range(n)])
+
+    mrr = float(np.mean(1.0 / (ranks + 1)))
+    recall_1 = float(np.mean(ranks < 1))
+    recall_5 = float(np.mean(ranks < 5))
+    recall_10 = float(np.mean(ranks < 10))
+
+    return {
+        "mrr": round(mrr, 4),
+        "recall@1": round(recall_1, 4),
+        "recall@5": round(recall_5, 4),
+        "recall@10": round(recall_10, 4),
+    }
+
+
+def evaluate_quality(
+    model,
+    ds_cfg: DatasetConfig | None = None,
+    max_pairs: int | None = None,
+) -> dict[str, float]:
+    """Evaluate embedding quality on a dataset.
+
+    Returns a dict with either {"spearman": float} for scored datasets
+    or {"mrr", "recall@1", "recall@5", "recall@10"} for pair datasets.
+    """
+    if ds_cfg is None:
+        ds_cfg = DatasetConfig()
+
+    dataset = load_dataset(ds_cfg.name, ds_cfg.config, split=ds_cfg.split)
+    queries = list(dataset[ds_cfg.query_col])
+    passages = list(dataset[ds_cfg.passage_col])
+
+    if max_pairs is not None and len(queries) > max_pairs:
+        queries = queries[:max_pairs]
+        passages = passages[:max_pairs]
+
+    emb_q = model.encode(queries)
+    emb_p = model.encode(passages)
+
+    if ds_cfg.score_col is not None:
+        # Scored mode: Spearman correlation
+        scores = list(dataset[ds_cfg.score_col])
+        if max_pairs is not None and len(scores) > max_pairs:
+            scores = scores[:max_pairs]
+        gold_scores = [s / ds_cfg.score_scale for s in scores]
+
+        cos_sims = np.sum(emb_q * emb_p, axis=1) / (
+            np.linalg.norm(emb_q, axis=1) * np.linalg.norm(emb_p, axis=1)
+        )
+
+        correlation, _ = spearmanr(cos_sims, gold_scores)
+        return {"spearman": round(float(correlation), 4)}
+
+    # Pair mode: retrieval metrics
+    return _retrieval_metrics(emb_q, emb_p)

models.py

@@ -22,24 +22,24 @@ REGISTRY: dict[str, ModelConfig] = {
         name="bge-small-en-v1.5",
         model_id="BAAI/bge-small-en-v1.5",
     ),
-    "bge-small-fe": ModelConfig(
-        name="bge-small-en-v1.5 (fastembed)",
-        model_id="BAAI/bge-small-en-v1.5",
-        backend="fastembed",
-    ),
-    "all-minilm-fe": ModelConfig(
-        name="all-MiniLM-L6-v2 (fastembed)",
-        model_id="sentence-transformers/all-MiniLM-L6-v2",
-        backend="fastembed",
-    ),
-    "bge-small-le": ModelConfig(
-        name="bge-small-en-v1.5 (libembedding)",
-        model_id="BAAI/bge-small-en-v1.5",
-        backend="libembedding",
-    ),
-    "all-minilm-le": ModelConfig(
-        name="all-MiniLM-L6-v2 (libembedding)",
-        model_id="sentence-transformers/all-MiniLM-L6-v2",
-        backend="libembedding",
-    ),
+    # "bge-small-fe": ModelConfig(
+    #     name="bge-small-en-v1.5 (fastembed)",
+    #     model_id="BAAI/bge-small-en-v1.5",
+    #     backend="fastembed",
+    # ),
+    # "all-minilm-fe": ModelConfig(
+    #     name="all-MiniLM-L6-v2 (fastembed)",
+    #     model_id="sentence-transformers/all-MiniLM-L6-v2",
+    #     backend="fastembed",
+    # ),
+    # "bge-small-le": ModelConfig(
+    #     name="bge-small-en-v1.5 (libembedding)",
+    #     model_id="BAAI/bge-small-en-v1.5",
+    #     backend="libembedding",
+    # ),
+    # "all-minilm-le": ModelConfig(
+    #     name="all-MiniLM-L6-v2 (libembedding)",
+    #     model_id="sentence-transformers/all-MiniLM-L6-v2",
+    #     backend="libembedding",
+    # ),
 }

report.py

@@ -1,13 +1,179 @@
 from __future__ import annotations
 
+import csv
+import os
+from pathlib import Path
 from typing import Any, Optional
 
+import matplotlib.pyplot as plt
+import numpy as np
 from tabulate import tabulate
 
 
-def print_report(results: list[dict[str, Any]], baseline_name: Optional[str] = None) -> None:
-    """Print a formatted comparison table to stdout."""
-    headers = ["Model", "Quality (STS)", "Speed (sent/s)", "Median Time (s)", "Memory (MB)"]
+def _format_metrics(metrics: dict[str, float]) -> str:
+    """Format a single dataset's metrics into a compact string."""
+    if "spearman" in metrics:
+        return f"{metrics['spearman']:.4f}"
+    if "mrr" in metrics:
+        return f"MRR={metrics['mrr']:.4f} R@1={metrics['recall@1']:.4f}"
+    return "—"
+
+
+def _flatten_result(r: dict[str, Any]) -> dict[str, Any]:
+    """Flatten a single result dict into a flat key-value dict for CSV."""
+    flat: dict[str, Any] = {"model": r["name"]}
+
+    for ds_key, metrics in r.get("quality", {}).items():
+        for metric_name, value in metrics.items():
+            flat[f"{ds_key}/{metric_name}"] = value
+
+    speed = r.get("speed")
+    if speed:
+        flat["speed_sent_per_s"] = speed["sentences_per_second"]
+        flat["median_time_s"] = speed["median_seconds"]
+
+    memory = r.get("memory_mb")
+    if memory is not None:
+        flat["memory_mb"] = memory
+
+    return flat
+
+
+def export_csv(results: list[dict[str, Any]], path: str) -> None:
+    """Export results to a CSV file."""
+    rows = [_flatten_result(r) for r in results]
+    fieldnames = list(rows[0].keys())
+    # Ensure all fields are captured
+    for row in rows[1:]:
+        for k in row:
+            if k not in fieldnames:
+                fieldnames.append(k)
+
+    with open(path, "w", newline="") as f:
+        writer = csv.DictWriter(f, fieldnames=fieldnames)
+        writer.writeheader()
+        writer.writerows(rows)
+    print(f"CSV saved to {path}")
+
+
+def plot_charts(results: list[dict[str, Any]], output_dir: str) -> None:
+    """Generate and save benchmark charts."""
+    os.makedirs(output_dir, exist_ok=True)
+    models = [r["name"] for r in results]
+
+    # --- Quality charts (one per dataset) ---
+    ds_keys: list[str] = []
+    for r in results:
+        quality = r.get("quality")
+        if quality:
+            ds_keys = list(quality.keys())
+            break
+
+    for ds_key in ds_keys:
+        first_metrics = None
+        for r in results:
+            m = r.get("quality", {}).get(ds_key)
+            if m:
+                first_metrics = m
+                break
+        if not first_metrics:
+            continue
+
+        if "spearman" in first_metrics:
+            # Single bar chart for spearman
+            values = [r.get("quality", {}).get(ds_key, {}).get("spearman", 0) for r in results]
+            fig, ax = plt.subplots(figsize=(max(6, len(models) * 1.2), 5))
+            bars = ax.bar(models, values, color="#4C72B0")
+            ax.set_ylabel("Spearman Correlation")
+            ax.set_title(f"Quality — {ds_key}")
+            ax.set_ylim(0, 1)
+            for bar, v in zip(bars, values):
+                ax.text(bar.get_x() + bar.get_width() / 2, bar.get_height() + 0.01,
+                        f"{v:.4f}", ha="center", va="bottom", fontsize=9)
+            plt.xticks(rotation=30, ha="right")
+            plt.tight_layout()
+            fig.savefig(os.path.join(output_dir, f"quality_{ds_key}.png"), dpi=150)
+            plt.close(fig)
+        else:
+            # Grouped bar chart for retrieval metrics
+            metric_names = ["mrr", "recall@1", "recall@5", "recall@10"]
+            x = np.arange(len(models))
+            width = 0.18
+            colors = ["#4C72B0", "#55A868", "#C44E52", "#8172B2"]
+
+            fig, ax = plt.subplots(figsize=(max(8, len(models) * 2), 5))
+            for i, (metric, color) in enumerate(zip(metric_names, colors)):
+                values = [r.get("quality", {}).get(ds_key, {}).get(metric, 0) for r in results]
+                offset = (i - 1.5) * width
+                bars = ax.bar(x + offset, values, width, label=metric, color=color)
+                for bar, v in zip(bars, values):
+                    ax.text(bar.get_x() + bar.get_width() / 2, bar.get_height() + 0.005,
+                            f"{v:.2f}", ha="center", va="bottom", fontsize=7)
+            ax.set_ylabel("Score")
+            ax.set_title(f"Retrieval Quality — {ds_key}")
+            ax.set_ylim(0, 1.15)
+            ax.set_xticks(x)
+            ax.set_xticklabels(models, rotation=30, ha="right")
+            ax.legend()
+            plt.tight_layout()
+            fig.savefig(os.path.join(output_dir, f"quality_{ds_key}.png"), dpi=150)
+            plt.close(fig)
+
+    # --- Speed chart ---
+    speed_values = [r.get("speed", {}).get("sentences_per_second", 0) for r in results]
+    if any(v > 0 for v in speed_values):
+        fig, ax = plt.subplots(figsize=(max(6, len(models) * 1.2), 5))
+        bars = ax.bar(models, speed_values, color="#55A868")
+        ax.set_ylabel("Sentences / second")
+        ax.set_title("Encoding Speed")
+        for bar, v in zip(bars, speed_values):
+            if v > 0:
+                ax.text(bar.get_x() + bar.get_width() / 2, bar.get_height() + 0.5,
+                        str(v), ha="center", va="bottom", fontsize=9)
+        plt.xticks(rotation=30, ha="right")
+        plt.tight_layout()
+        fig.savefig(os.path.join(output_dir, "speed.png"), dpi=150)
+        plt.close(fig)
+
+    # --- Memory chart ---
+    mem_values = [r.get("memory_mb", 0) for r in results]
+    if any(v > 0 for v in mem_values):
+        fig, ax = plt.subplots(figsize=(max(6, len(models) * 1.2), 5))
+        bars = ax.bar(models, mem_values, color="#C44E52")
+        ax.set_ylabel("Peak Memory (MB)")
+        ax.set_title("Memory Usage")
+        for bar, v in zip(bars, mem_values):
+            if v > 0:
+                ax.text(bar.get_x() + bar.get_width() / 2, bar.get_height() + 0.5,
+                        str(v), ha="center", va="bottom", fontsize=9)
+        plt.xticks(rotation=30, ha="right")
+        plt.tight_layout()
+        fig.savefig(os.path.join(output_dir, "memory.png"), dpi=150)
+        plt.close(fig)
+
+    print(f"Charts saved to {output_dir}/")
+
+
+def print_report(
+    results: list[dict[str, Any]],
+    baseline_name: Optional[str] = None,
+    csv_path: Optional[str] = None,
+    chart_dir: Optional[str] = None,
+) -> None:
+    """Print a formatted comparison table and optionally export CSV/charts."""
+    # Discover dataset columns from the first result that has quality data
+    ds_keys: list[str] = []
+    for r in results:
+        quality = r.get("quality")
+        if quality:
+            ds_keys = list(quality.keys())
+            break
+
+    headers = ["Model"]
+    for ds_key in ds_keys:
+        headers.append(f"Quality ({ds_key})")
+    headers.extend(["Speed (sent/s)", "Median Time (s)", "Memory (MB)"])
+
     rows: list[list[Any]] = []
 
     for r in results:
@@ -15,20 +181,29 @@ def print_report(results: list[dict[str, Any]], baseline_name: Optional[str] = N
         if r.get("is_baseline"):
             name += " [B]"
 
-        quality = r.get("quality")
+        quality = r.get("quality", {})
         speed = r.get("speed")
         memory = r.get("memory_mb")
 
-        rows.append([
-            name,
-            f"{quality:.4f}" if quality is not None else "—",
+        row: list[Any] = [name]
+        for ds_key in ds_keys:
+            metrics = quality.get(ds_key)
+            row.append(_format_metrics(metrics) if metrics else "—")
+        row.extend([
             f"{speed['sentences_per_second']}" if speed else "—",
             f"{speed['median_seconds']}" if speed else "—",
             f"{memory}" if memory is not None else "—",
         ])
+        rows.append(row)
 
     print()
     print(tabulate(rows, headers=headers, tablefmt="simple"))
     if baseline_name:
         print(f"\n[B] = baseline ({baseline_name})")
     print()
+
+    if csv_path:
+        export_csv(results, csv_path)
+
+    if chart_dir:
+        plot_charts(results, chart_dir)

requirements.txt

@@ -7,3 +7,4 @@ fastembed
 libembedding
 numpy
 scipy
+matplotlib

results.csv

@@ -0,0 +1,3 @@
+model,stsbenchmark-sts/spearman,natural-questions/mrr,natural-questions/recall@1,natural-questions/recall@5,natural-questions/recall@10,squad/mrr,squad/recall@1,squad/recall@5,squad/recall@10,speed_sent_per_s,median_time_s,memory_mb
+all-mpnet-base-v2,0.8519,0.9762,0.955,0.999,1.0,0.2282,0.075,0.405,0.627,775.4,1.2897,369.4
+bge-small-en-v1.5,0.8615,0.9557,0.927,0.988,0.995,0.2257,0.081,0.382,0.614,2144.4,0.4663,353.2