改动汇总

core/plotter.py — _aggregate_by_layer() 改为两步：先按 (layer, kv_head) 分组取 median，再跨 kv_head 取 median/q25/q75
core/plotter_plotly.py — _agg() 同上
core/table_gen.py — 加 _pseudobulk(df, col) helper，所有 make_table 函数调用它而非直接 .median()
db/writer.py — 加 _pseudobulk_col(rows, col) 和 refresh_all_summaries(conn)，_calc_summary_row() 和 update_model_summary() 全部改用伪重复安全的聚合方式
ui/tab_leaderboard.py — Refresh 按钮触发 load_leaderboard()，其中静默调用 refresh_all_summaries()，在展示排行榜前先把历史数据全部用新方式重算

core/plotter.py

@@ -73,19 +73,33 @@ BAND_COLORS = {
 
 def _aggregate_by_layer(df: pd.DataFrame, col: str):
     """
-    Group by layer, return (layers, median, q25, q75).
-    Excludes kv_shared=True rows for KV metrics to avoid theoretical-value bias.
+    Pseudo-bulk two-step aggregation per layer (Nature Comms 2021).
+    Step 1: median across Q heads within each (layer, kv_head) group.
+    Step 2: median / q25 / q75 across kv_head groups per layer.
+    Avoids pseudoreplication bias in GQA models (e.g. 4Q:1K).
+    Excludes kv_shared rows for KV metrics (theoretical-value bias).
     """
     kv_cols = {"ssr_KV", "pearson_KV", "cosU_KV", "cosV_KV", "alpha_KV"}
     if col in kv_cols:
         df = df[df["kv_shared"] == 0] if "kv_shared" in df.columns else df
 
-    grp = df.groupby("layer")[col]
     layers = np.array(sorted(df["layer"].unique()))
-    med    = grp.median().reindex(layers).values
-    q25    = grp.quantile(0.25).reindex(layers).values
-    q75    = grp.quantile(0.75).reindex(layers).values
-    return layers, med, q25, q75
+    med_vals, q25_vals, q75_vals = [], [], []
+
+    for layer in layers:
+        ldf = df[df["layer"] == layer]
+        # Step 1: median within each kv_head group
+        if "kv_head" in ldf.columns:
+            step1 = ldf.groupby("kv_head")[col].median().values
+        else:
+            step1 = ldf[col].dropna().values
+        step1 = step1[~np.isnan(step1)] if len(step1) > 0 else step1
+        # Step 2: statistics across kv_head medians
+        med_vals.append(float(np.median(step1)) if len(step1) > 0 else np.nan)
+        q25_vals.append(float(np.percentile(step1, 25)) if len(step1) > 0 else np.nan)
+        q75_vals.append(float(np.percentile(step1, 75)) if len(step1) > 0 else np.nan)
+
+    return layers, np.array(med_vals), np.array(q25_vals), np.array(q75_vals)
 
 
 def _global_layers(df: pd.DataFrame):

core/plotter_plotly.py

@@ -63,15 +63,37 @@ TOTAL_HEIGHT   = SUBPLOT_HEIGHT * len(PANELS) + 120   # +header
 # ─────────────────────────────────────────────────────────────────────────────
 
 def _agg(df: pd.DataFrame, col: str):
-    """Per-layer median + IQR. Excludes kv_shared rows for KV metrics."""
+    """
+    Pseudo-bulk two-step aggregation per layer (Nature Comms 2021).
+    Step 1: median across Q heads within each (layer, kv_head) group.
+    Step 2: median / q25 / q75 across kv_head groups per layer.
+    Avoids pseudoreplication bias in GQA models (e.g. 4Q:1K).
+    Excludes kv_shared rows for KV metrics (theoretical-value bias).
+    """
     kv_cols = {"ssr_KV", "pearson_KV", "cosU_KV", "cosV_KV", "alpha_KV"}
-    d = df[df["kv_shared"] == 0] if col in kv_cols and "kv_shared" in df.columns else df
-    grp    = d.groupby("layer")[col]
-    layers = np.array(sorted(d["layer"].unique()), dtype=int)
-    med    = grp.median().reindex(layers).values.astype(float)
-    q25    = grp.quantile(0.25).reindex(layers).values.astype(float)
-    q75    = grp.quantile(0.75).reindex(layers).values.astype(float)
-    return layers, med, q25, q75
+    if col in kv_cols and "kv_shared" in df.columns:
+        df = df[df["kv_shared"] == 0]
+
+    layers = np.array(sorted(df["layer"].unique()), dtype=int)
+    med_vals, q25_vals, q75_vals = [], [], []
+
+    for layer in layers:
+        ldf = df[df["layer"] == layer]
+        # Step 1: median within each kv_head group
+        if "kv_head" in ldf.columns:
+            step1 = ldf.groupby("kv_head")[col].median().values
+        else:
+            step1 = ldf[col].dropna().values
+        step1 = step1[~np.isnan(step1.astype(float))] if len(step1) > 0 else step1
+        # Step 2: statistics across kv_head medians
+        med_vals.append(float(np.median(step1)) if len(step1) > 0 else np.nan)
+        q25_vals.append(float(np.percentile(step1, 25)) if len(step1) > 0 else np.nan)
+        q75_vals.append(float(np.percentile(step1, 75)) if len(step1) > 0 else np.nan)
+
+    return (layers,
+            np.array(med_vals, dtype=float),
+            np.array(q25_vals, dtype=float),
+            np.array(q75_vals, dtype=float))
 
 
 def _global_layers(df: pd.DataFrame) -> list[int]:

core/table_gen.py

@@ -32,6 +32,43 @@ def _mean(series) -> Optional[float]:
     return float(v.mean()) if len(v) > 0 else None
 
 
+def _pseudobulk(df: pd.DataFrame, col: str) -> np.ndarray:
+    """
+    Pseudo-bulk two-step aggregation (Nature Comms 2021).
+    Step 1: median across Q heads within each (layer, kv_head) group.
+    Step 2: median across kv_head groups per layer.
+    Returns 1-D array of per-layer medians.
+    For MHA models this equals a plain per-layer median.
+    """
+    if df.empty or col not in df.columns:
+        return np.array([])
+    layers = sorted(df["layer"].unique())
+    per_layer = []
+    for layer in layers:
+        ldf = df[df["layer"] == layer]
+        if "kv_head" in ldf.columns:
+            step1 = ldf.groupby("kv_head")[col].median().values
+        else:
+            step1 = ldf[col].dropna().values
+        step1 = np.array(step1, dtype=float)
+        step1 = step1[~np.isnan(step1)]
+        if len(step1) > 0:
+            per_layer.append(float(np.median(step1)))
+    return np.array(per_layer, dtype=float)
+
+
+def _pb_med(df: pd.DataFrame, col: str) -> Optional[float]:
+    """Pseudo-bulk median across layers."""
+    v = _pseudobulk(df, col)
+    return float(np.median(v)) if len(v) > 0 else None
+
+
+def _pb_mean(df: pd.DataFrame, col: str) -> Optional[float]:
+    """Pseudo-bulk mean across layers."""
+    v = _pseudobulk(df, col)
+    return float(np.mean(v)) if len(v) > 0 else None
+
+
 def _fmt(x, decimals=6) -> str:
     if x is None or (isinstance(x, float) and np.isnan(x)):
         return "—"
@@ -134,10 +171,10 @@ def make_table1(
             "Model":         _short(model_id),
             "Std Layers":    n_layers,
             "Global Layers": n_global if n_global > 0 else "—",
-            "Median Pearson":_fmt(_med(std["pearson_QK"]), 4),
-            "Mean Pearson":  _fmt(_mean(std["pearson_QK"]), 4),
-            "Median SSR":    _fmt(_med(std["ssr_QK"]), 6),
-            "Mean SSR":      _fmt(_mean(std["ssr_QK"]), 6),
+            "Median Pearson":_fmt(_pb_med(std, "pearson_QK"), 4),
+            "Mean Pearson":  _fmt(_pb_mean(std, "pearson_QK"), 4),
+            "Median SSR":    _fmt(_pb_med(std, "ssr_QK"), 6),
+            "Mean SSR":      _fmt(_pb_mean(std, "ssr_QK"), 6),
         })
     return pd.DataFrame(rows)
 
@@ -166,13 +203,13 @@ def make_table2(
     for lo, hi in group_bounds:
         label = f"{lo}–{hi}"
         grp_a = std_a[(std_a["layer"] >= lo) & (std_a["layer"] <= hi)]
-        ssr_a = _med(grp_a["ssr_QK"])
+        ssr_a = _pb_med(grp_a, "ssr_QK")
 
         row = {"Layer Group": label, f"{_short(name_a)} SSR": _fmt(ssr_a, 6)}
 
         if std_b is not None and name_b:
             grp_b = std_b[(std_b["layer"] >= lo) & (std_b["layer"] <= hi)]
-            ssr_b = _med(grp_b["ssr_QK"])
+            ssr_b = _pb_med(grp_b, "ssr_QK")
             row[f"{_short(name_b)} SSR"] = _fmt(ssr_b, 6)
             if ssr_a and ssr_b and ssr_a > 0:
                 improvement = (ssr_a - ssr_b) / ssr_a * 100
@@ -207,9 +244,9 @@ def make_table3(
             "Model":           _short(model_id),
             "d_h":             head_dim,
             "Random 1/√d_h":  _fmt(baseline, 4),
-            "cosU(Q,K)":      _fmt(_med(std["cosU_QK"]), 4),
-            "cosU(Q,V)":      _fmt(_med(std["cosU_QV"]), 4),
-            "cosU(K,V)":      _fmt(_med(std["cosU_KV"]), 4),
+            "cosU(Q,K)":      _fmt(_pb_med(std, "cosU_QK"), 4),
+            "cosU(Q,V)":      _fmt(_pb_med(std, "cosU_QV"), 4),
+            "cosU(K,V)":      _fmt(_pb_med(std, "cosU_KV"), 4),
         })
     return pd.DataFrame(rows)
 
@@ -237,9 +274,9 @@ def make_table4(
             "Model":           _short(model_id),
             "d_model":         d_model,
             "Random 1/√D":    _fmt(baseline, 4),
-            "cosV(Q,K)":      _fmt(_med(std["cosV_QK"]), 4),
-            "cosV(Q,V)":      _fmt(_med(std["cosV_QV"]), 4),
-            "cosV(K,V)":      _fmt(_med(std["cosV_KV"]), 4),
+            "cosV(Q,K)":      _fmt(_pb_med(std, "cosV_QK"), 4),
+            "cosV(Q,V)":      _fmt(_pb_med(std, "cosV_QV"), 4),
+            "cosV(K,V)":      _fmt(_pb_med(std, "cosV_KV"), 4),
         })
     return pd.DataFrame(rows)
 
@@ -267,12 +304,12 @@ def make_table5(
         deep = std[std["layer"] > std["layer"].min()]
         rows.append({
             "Model":            _short(model_id),
-            "Median κ(Q) all":  _fmt(_med(std["cond_Q"]), 1),
-            "Median κ(K) all":  _fmt(_med(std["cond_K"]), 1),
-            "κ(Q) Layer 0":     _fmt(_med(l0["cond_Q"]), 1),
-            "κ(K) Layer 0":     _fmt(_med(l0["cond_K"]), 1),
-            "Median κ(Q) deep": _fmt(_med(deep["cond_Q"]), 1),
-            "Median κ(K) deep": _fmt(_med(deep["cond_K"]), 1),
+            "Median κ(Q) all":  _fmt(_pb_med(std, "cond_Q"), 1),
+            "Median κ(K) all":  _fmt(_pb_med(std, "cond_K"), 1),
+            "κ(Q) Layer 0":     _fmt(_pb_med(l0,  "cond_Q"), 1),
+            "κ(K) Layer 0":     _fmt(_pb_med(l0,  "cond_K"), 1),
+            "Median κ(Q) deep": _fmt(_pb_med(deep, "cond_Q"), 1),
+            "Median κ(K) deep": _fmt(_pb_med(deep, "cond_K"), 1),
         })
     return pd.DataFrame(rows)
 
@@ -293,9 +330,9 @@ def make_table6(
         std = _standard_only(df)
         if std.empty:
             continue
-        med_ssr     = _med(std["ssr_QK"])
+        med_ssr     = _pb_med(std, "ssr_QK")
         wang_score  = 1 - med_ssr if med_ssr is not None else None
-        med_pearson = _med(std["pearson_QK"])
+        med_pearson = _pb_med(std, "pearson_QK")
         rows.append({
             "Model":          _short(model_id),
             "Std Layers":     std["layer"].nunique(),

db/writer.py

@@ -2,7 +2,7 @@
 """
 数据库写入模块
 - 写入分析结果到 layer_head_metrics
-- 计算并写入 model_summary
+- 计算并写入 model_summary（pseudo-bulk 两步聚合，避免 GQA 伪重复）
 - 支持断点续传（以 prefix+layer 为粒度）
 - 写入权限验证
 """
@@ -10,6 +10,7 @@
 import os
 import sqlite3
 import numpy as np
+from collections import defaultdict
 from datetime import datetime
 from db.schema import get_connection, init_db
 
@@ -19,21 +20,10 @@ from db.schema import get_connection, init_db
 # ─────────────────────────────────────────────
 
 def infer_layer_type(kv_shared: bool) -> str:
-    """
-    从结构特征推断层类型
-    kv_shared=True  → 'global'  （K=V共享，如 Gemma 全局层）
-    kv_shared=False → 'standard'
-    """
     return "global" if kv_shared else "standard"
 
 
 def infer_modality(prefix: str) -> str:
-    """
-    从组件前缀推断模态
-    纯关键词匹配，不 hard coding 模型名
-    未匹配到任何关键词 → 默认 'language'
-    （覆盖纯语言模型，如 "model." 前缀的 LLaMA/Qwen）
-    """
     p = prefix.lower()
     if "vision" in p or "visual" in p or "image" in p:
         return "vision"
@@ -47,15 +37,6 @@ def infer_modality(prefix: str) -> str:
 # ─────────────────────────────────────────────
 
 def check_write_permission(admin_token: str) -> bool:
-    """
-    验证管理员写入权限。
-    WRITE_TOKEN 存储在 HF Space Secrets（加密，不进入 git repo）。
-    运行时由 HF 注入为环境变量，只在服务端比对，不返回给前端。
-
-    返回：
-      True  = 有写入权限
-      False = 只读模式（分析可以跑，结果不写库）
-    """
     server_token = os.environ.get("WRITE_TOKEN", "")
     if not server_token:
         return False
@@ -71,7 +52,6 @@ def get_analyzed_layers(
     model_id: str,
     prefix:   str,
 ) -> set:
-    """返回已完成分析的层号集合"""
     cur = conn.cursor()
     cur.execute(
         """SELECT DISTINCT layer FROM layer_head_metrics
@@ -88,7 +68,6 @@ def is_layer_complete(
     layer:            int,
     expected_records: int,
 ) -> bool:
-    """检查某层是否已完整写入"""
     cur = conn.cursor()
     cur.execute(
         """SELECT COUNT(*) FROM layer_head_metrics
@@ -226,12 +205,67 @@ def write_layer_records(
     conn.commit()
 
 
+# ─────────────────────────────────────────────
+# Pseudo-bulk 聚合核心函数
+# ─────────────────────────────────────────────
+
+def _pseudobulk(rows, col_name: str) -> np.ndarray:
+    """
+    Pseudo-bulk two-step aggregation (Nature Comms 2021).
+    Avoids GQA pseudoreplication (e.g. 4Q:1K → 4 correlated records per KV head).
+
+    Step 1: median within each (layer, kv_head) group
+            → one value per KV-head per layer
+    Step 2: return flat array of Step-1 values
+            → caller computes final median / mean / quantile
+
+    Works with both sqlite3.Row objects and plain dicts.
+    """
+    groups: dict[tuple, list] = defaultdict(list)
+    for r in rows:
+        try:
+            v       = r["ssr_QK"] if col_name == "ssr_QK" else r[col_name]
+            layer   = int(r["layer"])
+            kv_head = int(r["kv_head"]) if r["kv_head"] is not None else 0
+        except (KeyError, TypeError, IndexError):
+            continue
+        if v is None:
+            continue
+        groups[(layer, kv_head)].append(float(v))
+
+    if not groups:
+        return np.array([])
+
+    # Step 1: median within each (layer, kv_head) group
+    return np.array([float(np.median(vals)) for vals in groups.values()])
+
+
+def _pseudobulk_col(rows, col_name: str) -> np.ndarray:
+    """Generic version of _pseudobulk for any column name."""
+    groups: dict[tuple, list] = defaultdict(list)
+    for r in rows:
+        try:
+            v       = r[col_name]
+            layer   = int(r["layer"])
+            kv_head = int(r["kv_head"]) if r["kv_head"] is not None else 0
+        except (KeyError, TypeError, IndexError):
+            continue
+        if v is None:
+            continue
+        groups[(layer, kv_head)].append(float(v))
+
+    if not groups:
+        return np.array([])
+
+    return np.array([float(np.median(vals)) for vals in groups.values()])
+
+
 # ─────────────────────────────────────────────
 # 计算并写入 model_summary
 # ─────────────────────────────────────────────
 
 def _calc_summary_row(
-    rows:       list,
+    rows,
     model_id:   str,
     prefix:     str,
     layer_type: str,
@@ -239,14 +273,13 @@ def _calc_summary_row(
     if not rows:
         return None
 
-    def col(name):
-        vals = [r[name] for r in rows if r[name] is not None]
-        return np.array(vals, dtype=float) if vals else np.array([])
+    def pb(col):
+        return _pseudobulk_col(rows, col)
 
     def med(arr): return float(np.median(arr)) if len(arr) > 0 else None
     def avg(arr): return float(np.mean(arr))   if len(arr) > 0 else None
 
-    ssr_qk     = col("ssr_QK")
+    ssr_qk     = pb("ssr_QK")
     wang_score = float(1 - np.median(ssr_qk)) if len(ssr_qk) > 0 else None
     n_layers   = len(set(r["layer"] for r in rows))
     n_records  = len(rows)
@@ -255,18 +288,18 @@ def _calc_summary_row(
         "model_id":          model_id,
         "prefix":            prefix,
         "layer_type":        layer_type,
-        "median_pearson_QK": med(col("pearson_QK")),
-        "mean_pearson_QK":   avg(col("pearson_QK")),
+        "median_pearson_QK": med(pb("pearson_QK")),
+        "mean_pearson_QK":   avg(pb("pearson_QK")),
         "median_ssr_QK":     med(ssr_qk),
         "mean_ssr_QK":       avg(ssr_qk),
-        "median_ssr_QV":     med(col("ssr_QV")),
-        "mean_ssr_QV":       avg(col("ssr_QV")),
-        "median_cond_Q":     med(col("cond_Q")),
-        "mean_cond_Q":       avg(col("cond_Q")),
-        "median_cosU_QK":    med(col("cosU_QK")),
-        "median_cosU_QV":    med(col("cosU_QV")),
-        "median_cosV_QK":    med(col("cosV_QK")),
-        "median_cosV_QV":    med(col("cosV_QV")),
+        "median_ssr_QV":     med(pb("ssr_QV")),
+        "mean_ssr_QV":       avg(pb("ssr_QV")),
+        "median_cond_Q":     med(pb("cond_Q")),
+        "mean_cond_Q":       avg(pb("cond_Q")),
+        "median_cosU_QK":    med(pb("cosU_QK")),
+        "median_cosU_QV":    med(pb("cosU_QV")),
+        "median_cosV_QK":    med(pb("cosV_QK")),
+        "median_cosV_QV":    med(pb("cosV_QV")),
         "wang_score":        wang_score,
         "n_layers":          n_layers,
         "n_records":         n_records,
@@ -280,21 +313,20 @@ def update_model_summary(
     prefix:   str,
 ):
     """
-    重新计算并写入 model_summary（all / standard / global 三行）
-    wang_score 统一用 standard 层计算
+    重新计算并写入 model_summary（all / standard / global 三行）。
+    wang_score 统一用 standard 层 pseudo-bulk median(SSR_QK) 计算。
     """
     cur = conn.cursor()
+    cur.row_factory = sqlite3.Row
 
-    # 预取 standard 层的 ssr_QK（wang_score 统一用这个）
+    # ── Wang Score: standard 层 pseudo-bulk ──────────────────────────────
     cur.execute(
-        """SELECT ssr_QK FROM layer_head_metrics
-           WHERE model_id = ? AND prefix = ? AND layer_type = 'standard'""",
+        """SELECT layer, kv_head, ssr_QK FROM layer_head_metrics
+           WHERE model_id = ? AND prefix = ? AND layer_type = 'standard'
+             AND kv_shared = 0""",
         (model_id, prefix)
     )
-    std_ssr_rows = cur.fetchall()
-    std_ssr = np.array(
-        [r[0] for r in std_ssr_rows if r[0] is not None], dtype=float
-    )
+    std_ssr       = _pseudobulk_col(cur.fetchall(), "ssr_QK")
     std_wang_score = float(1 - np.median(std_ssr)) if len(std_ssr) > 0 else None
 
     for layer_type in ["all", "standard", "global"]:
@@ -315,8 +347,7 @@ def update_model_summary(
         if summary is None:
             continue
 
-        # wang_score 统一用 standard 层
-        summary["wang_score"] = std_wang_score
+        summary["wang_score"] = std_wang_score  # always from standard pseudo-bulk
 
         conn.execute(
             """INSERT OR REPLACE INTO model_summary(
@@ -341,4 +372,24 @@ def update_model_summary(
             summary
         )
 
-    conn.commit()
+    conn.commit()
+
+
+# ─────────────────────────────────────────────
+# 批量刷新所有模型的 model_summary
+# ─────────────────────────────────────────────
+
+def refresh_all_summaries(conn: sqlite3.Connection) -> int:
+    """
+    Re-compute model_summary for every (model_id, prefix) in the DB.
+    Called by Tab 3 Refresh button to migrate historical data to pseudo-bulk.
+    Returns number of (model_id, prefix) pairs refreshed.
+    """
+    cur = conn.cursor()
+    cur.execute(
+        "SELECT DISTINCT model_id, prefix FROM layer_head_metrics"
+    )
+    pairs = cur.fetchall()
+    for model_id, prefix in pairs:
+        update_model_summary(conn, model_id, prefix)
+    return len(pairs)

ui/tab_leaderboard.py

@@ -1,7 +1,8 @@
 # ui/tab_leaderboard.py
 """
 Tab3: Wang's Five Laws Leaderboard
-- Ranked by wang_score (= 1 − median SSR_QK, standard layers only)
+- Ranked by wang_score (= 1 − pseudo-bulk median SSR_QK, standard layers only)
+- On Refresh: silently re-computes all model_summary rows (pseudo-bulk migration)
 - Filter by modality (default: language)
 - Filter by layer_type (default: standard)
 """
@@ -12,6 +13,7 @@ import numpy as np
 
 from db.schema import init_db
 from db.reader import get_leaderboard
+from db.writer import refresh_all_summaries
 
 
 def _format_leaderboard(df: pd.DataFrame) -> pd.DataFrame:
@@ -47,8 +49,12 @@ def load_leaderboard(
     layer_type: str,
 ) -> tuple[pd.DataFrame, str]:
     conn = init_db()
-    lt   = layer_type if layer_type != "all" else "standard"
-    mod  = modality
+
+    # ── Silently refresh all summaries (pseudo-bulk migration) ────────────
+    n_refreshed = refresh_all_summaries(conn)
+
+    lt  = layer_type if layer_type != "all" else "standard"
+    mod = modality
 
     df = get_leaderboard(conn, modality=mod, layer_type=lt, limit=100)
 
@@ -62,7 +68,8 @@ def load_leaderboard(
     formatted = _format_leaderboard(df)
     status = (
         f"✅ {len(formatted)} entries  "
-        f"| modality={mod}  layer_type={lt}"
+        f"| modality={mod}  layer_type={lt}  "
+        f"| summaries refreshed: {n_refreshed}"
     )
     return formatted, status
 
@@ -72,11 +79,12 @@ def build_tab_leaderboard():
         gr.Markdown(r"""
         ## Wang's Five Laws — Model Leaderboard
 
-        **Wang Score = 1 − median(SSR\_QK)**  Higher is better. Theoretical max = 1.  
+        **Wang Score = 1 − median(SSR\_QK)**  Higher is better. Theoretical max = 1.
         Computed from `standard` layers only (global/KV-shared layers excluded).
+        Metrics use **pseudo-bulk aggregation** (Nature Comms 2021) to avoid GQA pseudoreplication.
 
         > 王氏评分 = 1 − median(SSR_QK)，越高越好，理论极值=1。
-        > 仅基于 standard 层计算（排除 K=V 共享的全局层干扰）。
+        > 仅基于 standard 层计算。采用 pseudo-bulk 两步聚合避免 GQA 伪重复计数。
         """)
 
         with gr.Row():