mcq tasks

backend/config.py

@@ -1,29 +1,40 @@
-# backend/config.py
 import os
+from typing import Dict, List, Tuple, Optional
 from huggingface_hub import HfApi
 
-# Hugging Face dataset repos (PUBLIC)
-REQUESTS_REPO_ID = "qimma/leaderboard-requests"
-RESULTS_REPO_ID  = "qimma/leaderboard-results"
+# --- 1. Repository & API Configuration ---
+OWNER: str = "qimma"
+REPO_ID: str = f"{OWNER}/Qimma-Leaderboard"
+RESULTS_REPO_ID: str = f"{OWNER}/results_v3"
+REQUESTS_REPO_ID: str = f"{OWNER}/requests_v3"
 
-# Tasks definition (task_key, metric_key, display_name)
-TASKS = [
-    ("arc", "acc_norm", "ARC"),
-    ("mmlu", "acc", "MMLU"),
-    ("hellaswag", "acc_norm", "HellaSwag"),
-]
+SLACK_WEBHOOK_URL = os.getenv("SLACK_WEBHOOK_URL", "")
 
-MODEL_TYPE_TO_EMOJI = {
-    "base": "🧱",
-    "chat": "💬",
-    "instruct": "🧠",
-}
+hf_api_token: Optional[str] = os.getenv("HF_API_TOKEN")
+API = HfApi(token=hf_api_token)
 
-# Optional token (read-only)
-hf_api_token = os.getenv("HF_TOKEN")
+# --- 2. Task Definitions ---
+# Format: (dataset_key, metric_field, display_name)
+TASKS: List[Tuple[str, str, str]] = [
+    ("qimma-AraDiCE-Culture:_average|0",    "acc:logprob_normalization=LogProbCharNorm(name='norm', ignore_first_space=False)",     "AraDiCE-Culture"),
+    ("qimma-ArabicMMLU:_average|0",         "acc:logprob_normalization=LogProbCharNorm(name='norm', ignore_first_space=False)",     "ArabicMMLU"),
+    ("qimma-MedArabiQ:_average|0",          "acc:logprob_normalization=LogProbCharNorm(name='norm', ignore_first_space=False)",     "MedArabiQ"),
+    ("qimma-ArabCulture:_average|0",        "acc:logprob_normalization=LogProbCharNorm(name='norm', ignore_first_space=False)",     "ArabCulture"),
+    ("qimma-SyntheticQA:_average|0",        "acc:logprob_normalization=LogProbCharNorm(name='norm', ignore_first_space=False)",     "SyntheticQA"),
+    ("qimma-PALMX-2025|0",                  "acc:logprob_normalization=LogProbCharNorm(name='norm', ignore_first_space=False)",     "PALMX"),
+    ("qimma-NativeQA|0",                    "acc:logprob_normalization=LogProbCharNorm(name='norm', ignore_first_space=False)",     "NativeQA"),
+    ("qimma-NativeQA-RDP|0",                "acc:logprob_normalization=LogProbCharNorm(name='norm', ignore_first_space=False)",     "NativeQA-RDP"),
+    ("qimma-AraTrust|0",                    "acc:logprob_normalization=LogProbCharNorm(name='norm', ignore_first_space=False)",     "AraTrust"),
+    ("qimma-MizanQA|0",                     "acc:logprob_normalization=LogProbCharNorm(name='norm', ignore_first_space=False)",     "MizanQA"),
+]
 
-# SAFE: API always exists
-API = HfApi(token=hf_api_token) if hf_api_token else HfApi()
+# --- 3. Model Metadata ---
 
-# SAFE: always defined
-SLACK_WEBHOOK_URL = None
+MODEL_TYPE_TO_EMOJI: Dict[str, str] = {
+    "🟢 : pretrained": "🟢",
+    "🟩 : continuously pretrained": "🟩",
+    "💬 : chat models (RLHF, DPO, IFT, ...)": "💬",
+    "🔶 : fine-tuned on domain-specific datasets": "🔶",
+    "🤝 : base merges and merges": "🤝",
+    "Missing": "?",
+}

frontend/leaderboard.html

@@ -746,7 +746,7 @@
                     $('#modalLinkHF').href = `https://huggingface.co/${model["Model Name"]}`;
 
                     const cleanName = model["Model Name"].replace(/\//g, '__');
-                    const datasetId = `OALL/details_${cleanName}_v2`;
+                    const datasetId = `OALL/details_${cleanName}`;
                     $('#modalLinkDetails').href = `https://huggingface.co/datasets/${datasetId}`;
 
                     const chartContainer = $('#modalChart');