fix issue with multi-options answers, and with the samples that don't have binary score

.gitignore

@@ -1,2 +1,3 @@
 **/__pycache__/**
-.vscode/
+.vscode/
+scripts/*

backend/data_loader.py

@@ -69,6 +69,39 @@ for task_key, _, display in TASKS:
             bases.append(base)
 
 
+def _extract_base_metric_pairs(task_key: Any, metric_key: Any) -> List[tuple[str, str]]:
+    pairs: List[tuple[str, str]] = []
+
+    if isinstance(task_key, list):
+        if isinstance(metric_key, list):
+            for tk, mk in zip(task_key, metric_key):
+                if isinstance(mk, tuple):
+                    mk = mk[0]
+                pairs.extend(_extract_base_metric_pairs(tk, mk))
+        return pairs
+
+    if not isinstance(task_key, str) or not isinstance(metric_key, str):
+        return pairs
+
+    base = task_key.split(":", 1)[0].split("|", 1)[0].strip()
+    if base:
+        pairs.append((base, metric_key))
+    return pairs
+
+
+BENCHMARK_BASE_TO_METRICS: Dict[str, List[str]] = {}
+BENCHMARK_DISPLAY_TO_BASE_METRICS: Dict[str, Dict[str, List[str]]] = {}
+for task_key, metric_key, display in TASKS:
+    display_bucket = BENCHMARK_DISPLAY_TO_BASE_METRICS.setdefault(display, {})
+    for base, metric_name in _extract_base_metric_pairs(task_key, metric_key):
+        base_bucket = BENCHMARK_BASE_TO_METRICS.setdefault(base, [])
+        if metric_name and metric_name not in base_bucket:
+            base_bucket.append(metric_name)
+        display_metric_bucket = display_bucket.setdefault(base, [])
+        if metric_name and metric_name not in display_metric_bucket:
+            display_metric_bucket.append(metric_name)
+
+
 # -----------------------------------------------------------------------------
 # Utilities
 # -----------------------------------------------------------------------------
@@ -437,6 +470,91 @@ def _json_safe(value: Any) -> Any:
     return value
 
 
+def _to_float_scalar(value: Any) -> Optional[float]:
+    value = _py_scalar(value)
+    if isinstance(value, (int, float, np.integer, np.floating)):
+        return float(value)
+    return None
+
+
+def _normalize_indices(value: Any) -> List[int]:
+    indices: List[int] = []
+    for item in _as_list(value):
+        item = _py_scalar(item)
+        if isinstance(item, (int, np.integer)):
+            indices.append(int(item))
+    return indices
+
+
+def _format_answer(values: List[Any]) -> Any:
+    if not values:
+        return None
+    clean = [str(_py_scalar(v)) for v in values]
+    if len(clean) == 1:
+        return clean[0]
+    return ", ".join(clean)
+
+
+def _norm_answer(value: Any) -> str:
+    value = _py_scalar(value)
+    if value is None:
+        return ""
+    return str(value).strip()
+
+
+def _pick_metric(
+    metric: Dict[str, Any],
+    benchmark_base: str,
+    preferred_metrics: Optional[List[str]] = None,
+) -> tuple[Optional[str], Optional[float]]:
+    if not isinstance(metric, dict) or not metric:
+        return None, None
+
+    preferred = preferred_metrics or BENCHMARK_BASE_TO_METRICS.get(benchmark_base, [])
+    for name in preferred:
+        if name in metric:
+            val = _to_float_scalar(metric.get(name))
+            if val is not None:
+                return name, val
+
+    # Fallback for known detail formats.
+    for name in ["normalized_score_norm", "BERTScore-F", "acc", "accuracy"]:
+        if name in metric:
+            val = _to_float_scalar(metric.get(name))
+            if val is not None:
+                return name, val
+
+    for name, raw_val in metric.items():
+        val = _to_float_scalar(raw_val)
+        if val is not None:
+            return str(name), val
+    return None, None
+
+
+def _is_binary_metric_name(metric_name: Optional[str]) -> bool:
+    if not metric_name:
+        return False
+    n = metric_name.lower()
+    return (
+        n.startswith("acc")
+        or "accuracy" in n
+        or "score_norm" in n
+        or n in {"exact_match", "fann_or_flop"}
+    )
+
+
+def _is_choice_metric_name(metric_name: Optional[str]) -> bool:
+    if not metric_name:
+        return False
+    n = metric_name.lower()
+    return (
+        n.startswith("acc")
+        or "mc_prob" in n
+        or "score_norm" in n
+        or n in {"exact_match", "fann_or_flop"}
+    )
+
+
 def _extract_predicted_answer(model_response: Dict[str, Any], choices: List[Any]) -> Any:
     logprobs = model_response.get("logprobs")
     if logprobs is not None and choices:
@@ -469,7 +587,12 @@ def _first_non_empty(values: Any) -> Optional[str]:
     return None
 
 
-def _read_detail_parquet(path: str, subtask: str) -> List[Dict[str, Any]]:
+def _read_detail_parquet(
+    path: str,
+    subtask: str,
+    benchmark_base: str,
+    preferred_metrics: Optional[List[str]] = None,
+) -> List[Dict[str, Any]]:
     try:
         df = pd.read_parquet(path)
     except Exception as e:
@@ -484,19 +607,14 @@ def _read_detail_parquet(path: str, subtask: str) -> List[Dict[str, Any]]:
 
         choices = _as_list(doc.get("choices"))
         choices = [_py_scalar(c) for c in choices]
-        gold_idx = doc.get("gold_index")
-        gold_answer = None
-        if isinstance(gold_idx, (int, np.integer)) and 0 <= int(gold_idx) < len(choices):
-            gold_answer = choices[int(gold_idx)]
-
-        metric_value = None
-        metric_name = None
-        if isinstance(metric, dict) and metric:
-            metric_name = next(iter(metric.keys()))
-            try:
-                metric_value = float(next(iter(metric.values())))
-            except Exception:
-                metric_value = None
+        gold_indices = _normalize_indices(doc.get("gold_index"))
+        gold_values: List[Any] = []
+        for idx in gold_indices:
+            if 0 <= idx < len(choices):
+                gold_values.append(choices[idx])
+        gold_answer = _format_answer(gold_values)
+
+        metric_name, metric_value = _pick_metric(metric, benchmark_base, preferred_metrics)
 
         model_response_dict = model_response if isinstance(model_response, dict) else {}
         predicted_answer = _extract_predicted_answer(model_response_dict, choices)
@@ -507,8 +625,23 @@ def _read_detail_parquet(path: str, subtask: str) -> List[Dict[str, Any]]:
             output_text = str(predicted_answer)
 
         is_correct = None
-        if metric_value is not None and metric_value in (0.0, 1.0):
+        if metric_value is not None and _is_binary_metric_name(metric_name) and metric_value in (0.0, 1.0):
             is_correct = bool(metric_value)
+        else:
+            binary_score = _to_float_scalar(metric.get("normalized_score_norm"))
+            if binary_score is not None and binary_score in (0.0, 1.0):
+                is_correct = bool(binary_score)
+
+        # For multi-gold classification (e.g. Mizan), accept prediction if it matches any gold option.
+        pred_norm = _norm_answer(predicted_answer)
+        choice_norms = {_norm_answer(c) for c in choices if _norm_answer(c)}
+        gold_norms = {_norm_answer(g) for g in gold_values if _norm_answer(g)}
+        if _is_choice_metric_name(metric_name) and pred_norm and pred_norm in choice_norms and gold_norms:
+            is_correct = pred_norm in gold_norms
+
+        predicted_answer = _py_scalar(predicted_answer)
+        if isinstance(predicted_answer, list):
+            predicted_answer = _format_answer(predicted_answer)
 
         prompt = (
             doc.get("query")
@@ -557,39 +690,64 @@ def load_benchmark_details(
     if not benchmark_bases:
         benchmark_bases = [benchmark_display]
 
-    selected_entries: List[tuple[str, Dict[str, Any]]] = []
+    selected_entries: List[tuple[str, str, Dict[str, Any], List[str]]] = []
     for base in benchmark_bases:
         subtasks = model_bucket.get(base, {})
+        selected_base = base
         if not subtasks:
             base_l = base.strip().lower()
             for indexed_base, bucket in model_bucket.items():
                 if indexed_base.strip().lower() == base_l:
+                    selected_base = indexed_base
                     subtasks = bucket
                     break
+        display_metric_bucket = BENCHMARK_DISPLAY_TO_BASE_METRICS.get(benchmark_display, {})
+        preferred_metrics = display_metric_bucket.get(selected_base)
+        if preferred_metrics is None:
+            # Case-insensitive fallback.
+            for k, v in display_metric_bucket.items():
+                if k.strip().lower() == selected_base.strip().lower():
+                    preferred_metrics = v
+                    break
+        preferred_metrics = preferred_metrics or BENCHMARK_BASE_TO_METRICS.get(selected_base, [])
         for subtask, info in subtasks.items():
-            selected_entries.append((subtask, info))
+            selected_entries.append((selected_base, subtask, info, preferred_metrics))
 
     if not selected_entries:
         return {"benchmark": benchmark_display, "subtasks": [], "rows": []}
 
-    selected_entries.sort(key=lambda x: x[0].lower())
+    selected_entries.sort(key=lambda x: x[1].lower())
 
     rows_by_subtask: List[List[Dict[str, Any]]] = []
     subtasks_summary: List[Dict[str, Any]] = []
-    for subtask, info in selected_entries:
-        rows = _read_detail_parquet(info["path"], subtask)
+    for base, subtask, info, preferred_metrics in selected_entries:
+        rows = _read_detail_parquet(info["path"], subtask, base, preferred_metrics)
         rows_by_subtask.append(rows)
 
-        valid = [r for r in rows if isinstance(r.get("is_correct"), bool)]
-        correct = sum(1 for r in valid if r["is_correct"])
-        total = len(valid)
-        accuracy = round((correct / total) * 100, 2) if total > 0 else None
+        scored_rows = [r for r in rows if r.get("metric") is not None]
+        metric_name = next((str(r.get("metric_name")) for r in scored_rows if r.get("metric_name")), None)
+        use_metric_mode = metric_name is not None and not _is_binary_metric_name(metric_name)
+
+        if use_metric_mode:
+            correct = None
+            scored = len(scored_rows)
+            avg_metric = (sum(float(r["metric"]) for r in scored_rows) / scored) if scored > 0 else None
+            accuracy = round(avg_metric * 100, 2) if avg_metric is not None else None
+            summary_mode = "metric"
+        else:
+            binary_rows = [r for r in rows if isinstance(r.get("is_correct"), bool)]
+            correct = sum(1 for r in binary_rows if r["is_correct"])
+            scored = len(binary_rows)
+            accuracy = round((correct / scored) * 100, 2) if scored > 0 else None
+            summary_mode = "binary"
+
         subtasks_summary.append({
             "subtask": subtask,
             "total": len(rows),
-            "scored": total,
+            "scored": scored,
             "correct": correct,
             "accuracy": accuracy,
+            "mode": summary_mode,
         })
 
     total_rows = sum(len(rows) for rows in rows_by_subtask)

frontend/leaderboard.html

@@ -763,16 +763,21 @@
                             <div class="p-3 rounded-lg border border-slate-200 dark:border-slate-700 bg-slate-50 dark:bg-slate-800/70">
                                 <div class="text-xs text-slate-500 dark:text-slate-400">${escapeHtml(s.subtask)}</div>
                                 <div class="text-sm font-bold text-slate-800 dark:text-slate-100 mt-1">${s.accuracy === null ? "Unknown" : `${s.accuracy}%`}</div>
-                                <div class="text-xs text-slate-500 dark:text-slate-400 mt-0.5">${s.correct}/${s.scored} correct</div>
+                                <div class="text-xs text-slate-500 dark:text-slate-400 mt-0.5">${s.mode === "metric" ? `${s.scored} scored` : `${s.correct}/${s.scored} correct`}</div>
                             </div>
                         `).join("");
 
                         $('#benchmarkRows').innerHTML = rows.map(r => {
-                            const correctBadge = r.is_correct === true
-                                ? `<span class="text-emerald-600 dark:text-emerald-400 font-semibold">Correct</span>`
-                                : (r.is_correct === false
-                                    ? `<span class="text-rose-600 dark:text-rose-400 font-semibold">Wrong</span>`
-                                    : `<span class="text-slate-500 dark:text-slate-400 font-semibold">Unknown</span>`);
+                            let correctBadge = `<span class="text-slate-500 dark:text-slate-400 font-semibold">Unknown</span>`;
+                            if (r.is_correct === true) {
+                                correctBadge = `<span class="text-emerald-600 dark:text-emerald-400 font-semibold">Correct</span>`;
+                            } else if (r.is_correct === false) {
+                                correctBadge = `<span class="text-rose-600 dark:text-rose-400 font-semibold">Wrong</span>`;
+                            } else if (r.metric !== null && r.metric !== undefined) {
+                                const n = Number(r.metric);
+                                const scoreText = Number.isFinite(n) ? n.toFixed(4) : escapeHtml(r.metric);
+                                correctBadge = `<span class="text-sky-600 dark:text-sky-400 font-semibold">Score: ${scoreText}</span>`;
+                            }
                             const prompt = escapeHtml(asUnknown(r.prompt));
                             const output = escapeHtml(asUnknown(r.output));
                             const sampleMeta = [