Probe deploy

README.md

@@ -1,12 +1,17 @@
 ---
-title: AVBenchLB
-emoji: 👀
-colorFrom: purple
-colorTo: gray
+title: AVBench Leaderboard
+emoji: "🥇"
+colorFrom: green
+colorTo: indigo
 sdk: gradio
-sdk_version: 6.10.0
+sdk_version: 5.43.1
 app_file: app.py
-pinned: false
+pinned: true
+license: apache-2.0
+short_description: AVBench leaderboard table (normal/hard splits)
+python_version: "3.10"
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# AVBench Leaderboard
+
+This Space displays the fixed benchmark table for Normal and Hard splits.

app.py

@@ -0,0 +1,114 @@
+import gradio as gr
+import pandas as pd
+
+from constants import (
+    AVBENCH_INTRODUCTION,
+    HARD_ROWS,
+    METRIC_COLUMNS,
+    METRIC_WEIGHTS,
+    NORMAL_ROWS,
+)
+
+
+DISPLAY_COLUMNS = ["Medal", "Model"] + METRIC_COLUMNS + ["Overall"]
+
+
+def compute_normalized_overall(df):
+    norm_df = df.copy()
+    total_weight = sum(METRIC_WEIGHTS[c] for c in METRIC_COLUMNS)
+
+    weighted_sum = 0.0
+    for c in METRIC_COLUMNS:
+        col_min = norm_df[c].min()
+        col_max = norm_df[c].max()
+        if col_max > col_min:
+            col_norm = (norm_df[c] - col_min) / (col_max - col_min)
+        else:
+            # If all values are identical in a metric, it contributes equally to all rows.
+            col_norm = 0.0
+        weighted_sum = weighted_sum + col_norm * METRIC_WEIGHTS[c]
+
+    norm_df["Overall"] = (weighted_sum / total_weight).round(4)
+    return norm_df
+
+
+def add_medals(df):
+    medal_df = df.copy().reset_index(drop=True)
+    medals = [""] * len(medal_df)
+    if len(medal_df) > 0:
+        medals[0] = "🥇"
+    if len(medal_df) > 1:
+        medals[1] = "🥈"
+    if len(medal_df) > 2:
+        medals[2] = "🥉"
+    medal_df["Medal"] = medals
+    return medal_df
+
+
+def build_df(rows, split_name):
+    df = pd.DataFrame(rows, columns=["Model"] + METRIC_COLUMNS)
+    df = compute_normalized_overall(df)
+    df["Split"] = split_name
+    return df
+
+
+def refresh_table(split_name, sort_metric):
+    if split_name == "Normal":
+        df = normal_df.copy()
+    elif split_name == "Hard":
+        df = hard_df.copy()
+    else:
+        df = pd.concat([normal_df, hard_df], ignore_index=True)
+
+    sort_col = "Overall" if sort_metric == "Overall" else sort_metric
+    df = df.sort_values(by=sort_col, ascending=False).reset_index(drop=True)
+    df = add_medals(df)
+    return df[["Split"] + DISPLAY_COLUMNS] if split_name == "All" else df[DISPLAY_COLUMNS]
+
+
+normal_df = build_df(NORMAL_ROWS, "Normal")
+hard_df = build_df(HARD_ROWS, "Hard")
+
+with gr.Blocks(title="AVBench Leaderboard") as demo:
+    gr.Markdown(AVBENCH_INTRODUCTION)
+
+    with gr.Row():
+        split_selector = gr.Dropdown(
+            choices=["All", "Normal", "Hard"],
+            value="All",
+            label="Split",
+        )
+        sort_selector = gr.Dropdown(
+            choices=["Overall"] + METRIC_COLUMNS,
+            value="Overall",
+            label="Sort By",
+        )
+
+    leaderboard = gr.Dataframe(
+        value=refresh_table("All", "Overall"),
+        interactive=False,
+        wrap=True,
+        label="Leaderboard",
+    )
+
+    with gr.Accordion("Metric Groups", open=False):
+        gr.Markdown(
+            "- Cross-Modal Alignment & Sync: AV, AT, VT, SyncNet\n"
+            "- Unimodal Generation Quality: SC, DF-Arena, NISQA, Audiobox, DOVER++, Aesthetic\n"
+            "- Overall: min-max normalize each metric first, then weighted sum (current default: equal weights)."
+        )
+
+    split_selector.change(
+        refresh_table,
+        inputs=[split_selector, sort_selector],
+        outputs=leaderboard,
+    )
+    sort_selector.change(
+        refresh_table,
+        inputs=[split_selector, sort_selector],
+        outputs=leaderboard,
+    )
+
+
+if __name__ == "__main__":
+    demo.launch()

constants.py

@@ -0,0 +1,57 @@
+AVBENCH_INTRODUCTION = """# AVBench Leaderboard
+
+Which T2AV model performs better on AVBench?
+
+This leaderboard reports two splits:
+- Normal Split
+- Hard Split
+
+Metrics are grouped into:
+- Cross-Modal Alignment & Sync: AV, AT, VT, SyncNet
+- Unimodal Generation Quality: SC, DF-Arena, NISQA, Audiobox, DOVER++, Aesthetic
+
+All metrics are higher-is-better.
+"""
+
+METRIC_COLUMNS = [
+    "AV",
+    "AT",
+    "VT",
+    "SyncNet",
+    "SC",
+    "DF-Arena",
+    "NISQA",
+    "Audiobox",
+    "DOVER++",
+    "Aesthetic",
+]
+
+# Equal weight by default, can be adjusted later.
+METRIC_WEIGHTS = {
+    "AV": 1.0,
+    "AT": 1.0,
+    "VT": 1.0,
+    "SyncNet": 1.0,
+    "SC": 1.0,
+    "DF-Arena": 1.0,
+    "NISQA": 1.0,
+    "Audiobox": 1.0,
+    "DOVER++": 1.0,
+    "Aesthetic": 1.0,
+}
+
+NORMAL_ROWS = [
+    ["Sora 2", 0.8713, 0.5844, 0.7599, 4.9057, 87.8391, 0.4328, 2.3784, 3.1759, 60.0125, 4.0704],
+    ["Veo 3 Fast", 0.6924, 0.5708, 0.7235, 6.5943, 77.4950, 0.3043, 2.8191, 3.5877, 69.2275, 4.9967],
+    ["Wan 2.6", 0.8207, 0.5826, 0.7556, 4.5016, 91.5568, 0.0441, 3.0289, 3.9271, 71.6473, 4.7790],
+    ["Kling 2.6", 0.7626, 0.5603, 0.7501, 8.1027, 68.7844, 0.1665, 3.3141, 3.8082, 65.6786, 5.4885],
+    ["Seedance 1.5 Pro", 0.6536, 0.5764, 0.7363, 5.0146, 84.9268, 0.1602, 3.6411, 4.1686, 71.7205, 4.7373],
+]
+
+HARD_ROWS = [
+    ["Sora 2", 0.9320, 0.5573, 0.7190, 3.7932, 76.7905, 0.5498, 2.0564, 3.1339, 58.1538, 4.0434],
+    ["Veo 3 Fast", 0.7766, 0.5130, 0.6943, 3.4535, 70.3144, 0.3827, 2.3321, 3.6113, 67.0833, 5.1438],
+    ["Wan 2.6", 0.8780, 0.5517, 0.7482, 3.0488, 84.4512, 0.0498, 3.0726, 4.0924, 71.5229, 4.7721],
+    ["Kling 2.6", 0.8813, 0.4970, 0.7105, 3.9844, 69.0691, 0.1469, 3.2425, 3.8912, 62.9994, 5.5033],
+    ["Seedance 1.5 Pro", 0.7409, 0.5525, 0.7398, 3.3239, 80.8029, 0.2059, 3.4093, 4.1618, 69.4430, 4.7707],
+]

requirements.txt

@@ -0,0 +1,2 @@
+gradio==5.43.1
+pandas>=2.0.0