hypersphere_convergence_analysis.py

#!/usr/bin/env python3
"""
GEOLIP HYPERSPHERE MANIFOLD VISUALIZATION
==========================================
6-panel manifold view + 3-panel expert perspective divergence.
S^255 projected to S^2 via PCA.
"""

import torch
import torch.nn.functional as F
import numpy as np
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import math

DEVICE = "cpu"

# ══════════════════════════════════════════════════════════════════
# LOAD + EMBED
# ══════════════════════════════════════════════════════════════════

print("Loading soup...")
ckpt = torch.load("checkpoints/dual_stream_best.pt", map_location="cpu", weights_only=False)
sd = ckpt["state_dict"]
D_ANCHOR = ckpt["config"]["d_anchor"]
N_ANCHORS = ckpt["config"]["n_anchors"]
anchors = F.normalize(sd["constellation.anchors"], dim=-1)

EXPERTS = ["clip_l14_openai", "dinov2_b14", "siglip_b16_384"]
COCO_CLASSES = [
    "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train",
    "truck", "boat", "traffic light", "fire hydrant", "stop sign",
    "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep",
    "cow", "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella",
    "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard",
    "sports ball", "kite", "baseball bat", "baseball glove", "skateboard",
    "surfboard", "tennis racket", "bottle", "wine glass", "cup", "fork",
    "knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange",
    "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair",
    "couch", "potted plant", "bed", "dining table", "toilet", "tv",
    "laptop", "mouse", "remote", "keyboard", "cell phone", "microwave",
    "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase",
    "scissors", "teddy bear", "hair drier", "toothbrush",
]

print("Loading features...")
from datasets import load_dataset

ref = load_dataset("AbstractPhil/bulk-coco-features", EXPERTS[0], split="val")
val_ids = ref["image_id"]; N_val = len(val_ids)
val_id_map = {iid: i for i, iid in enumerate(val_ids)}
val_labels = torch.zeros(N_val, 80)
for i, labs in enumerate(ref["labels"]):
    for l in labs:
        if l < 80: val_labels[i, l] = 1.0

val_raw = {}
for name in EXPERTS:
    ds = load_dataset("AbstractPhil/bulk-coco-features", name, split="val")
    feats = torch.zeros(N_val, 768)
    for row in ds:
        if row["image_id"] in val_id_map:
            feats[val_id_map[row["image_id"]]] = torch.tensor(row["features"], dtype=torch.float32)
    val_raw[name] = feats; del ds

def project_expert(feats, i):
    prefix = f"projectors.{i}.proj_shared" if f"projectors.{i}.proj_shared.0.weight" in sd else f"projectors.{i}.proj"
    W = sd[f"{prefix}.0.weight"]
    b = sd[f"{prefix}.0.bias"]
    lw = sd[f"{prefix}.1.weight"]
    lb = sd[f"{prefix}.1.bias"]
    x = feats @ W.T + b
    mu = x.mean(-1, keepdim=True); var = x.var(-1, keepdim=True, unbiased=False)
    x = (x - mu) / (var + 1e-5).sqrt() * lw + lb
    return F.normalize(x, dim=-1)

print("Generating embeddings...")
with torch.no_grad():
    projected = [project_expert(val_raw[name], i) for i, name in enumerate(EXPERTS)]
    fused = F.normalize(sum(projected) / 3, dim=-1)

# ══════════════════════════════════════════════════════════════════
# PCA → 3D
# ══════════════════════════════════════════════════════════════════

emb = fused.numpy()
emb_centered = emb - emb.mean(axis=0, keepdims=True)
U, S, Vt = np.linalg.svd(emb_centered[:5000], full_matrices=False)
pca3 = Vt[:3]

emb_3d = emb @ pca3.T
anchors_3d = anchors.numpy() @ pca3.T

var_explained = S[:3]**2 / (S**2).sum()
print(f"PCA 3D variance: {var_explained.sum()*100:.1f}% "
      f"({var_explained[0]*100:.1f}%, {var_explained[1]*100:.1f}%, {var_explained[2]*100:.1f}%)")

def to_sphere(pts):
    norms = np.linalg.norm(pts, axis=-1, keepdims=True)
    return pts / (norms + 1e-8)

emb_s = to_sphere(emb_3d)
anchors_s = to_sphere(anchors_3d)

# Reference sphere wireframe
phi = np.linspace(0, 2*np.pi, 60)
theta = np.linspace(0, np.pi, 30)
xs = np.outer(np.cos(phi), np.sin(theta))
ys = np.outer(np.sin(phi), np.sin(theta))
zs = np.outer(np.ones_like(phi), np.cos(theta))

# Primary class per image (most specific)
class_freq = val_labels.sum(0).numpy()
primary_class = np.zeros(N_val, dtype=int)
for i in range(N_val):
    present = np.where(val_labels[i].numpy() > 0)[0]
    if len(present) > 0:
        primary_class[i] = present[class_freq[present].argmin()]

cmap20 = plt.cm.tab20(np.linspace(0, 1, 20))
class_colors = np.array([cmap20[primary_class[i] % 20] for i in range(N_val)])


# ══════════════════════════════════════════════════════════════════
# HELPER
# ══════════════════════════════════════════════════════════════════

def setup_ax(ax, title):
    ax.set_facecolor('black')
    ax.xaxis.pane.fill = False; ax.yaxis.pane.fill = False; ax.zaxis.pane.fill = False
    ax.xaxis.pane.set_edgecolor('gray'); ax.yaxis.pane.set_edgecolor('gray')
    ax.zaxis.pane.set_edgecolor('gray')
    ax.set_xlabel('PC1', color='gray', fontsize=8)
    ax.set_ylabel('PC2', color='gray', fontsize=8)
    ax.set_zlabel('PC3', color='gray', fontsize=8)
    ax.tick_params(colors='gray', labelsize=6)
    ax.set_title(title, color='white', fontsize=11, pad=10)
    ax.plot_wireframe(xs*0.98, ys*0.98, zs*0.98, alpha=0.03, color='white', linewidth=0.3)
    ax.set_xlim(-1.3, 1.3); ax.set_ylim(-1.3, 1.3); ax.set_zlim(-1.3, 1.3)


# ══════════════════════════════════════════════════════════════════
# FIGURE 1: 6-PANEL MANIFOLD VIEW
# ══════════════════════════════════════════════════════════════════

print("Rendering figure 1...")
fig = plt.figure(figsize=(24, 16), facecolor='black')
fig.suptitle(
    'GeoLIP Hypersphere Manifold — S²⁵⁵ projected to S²\n'
    f'{N_ANCHORS} anchors × {D_ANCHOR}-d × 3 experts | mAP={ckpt["mAP"]:.3f} | eff_dim=76.9',
    color='white', fontsize=16, y=0.98)

# Panel 1: Full manifold
ax1 = fig.add_subplot(231, projection='3d')
setup_ax(ax1, f'Full Manifold — {N_val} embeddings + {N_ANCHORS} anchors')
ax1.scatter(emb_s[:, 0], emb_s[:, 1], emb_s[:, 2],
            c=class_colors, s=1, alpha=0.3)
ax1.scatter(anchors_s[:, 0], anchors_s[:, 1], anchors_s[:, 2],
            c='red', s=8, alpha=0.6, marker='^')

# Panel 2: Class centroids
ax2 = fig.add_subplot(232, projection='3d')
setup_ax(ax2, '80 COCO Class Centroids')
centroids = np.zeros((80, emb.shape[1]))
for c in range(80):
    mask = val_labels[:, c].numpy() > 0
    if mask.sum() > 0:
        centroids[c] = emb[mask].mean(0)
centroids_3d = to_sphere(centroids @ pca3.T)
sizes = val_labels.sum(0).numpy()
sizes_scaled = 20 + 200 * (sizes / sizes.max())
colors80 = plt.cm.hsv(np.linspace(0, 0.95, 80))
ax2.scatter(centroids_3d[:, 0], centroids_3d[:, 1], centroids_3d[:, 2],
            c=colors80, s=sizes_scaled, alpha=0.8, edgecolors='white', linewidth=0.3)
for c in [0, 2, 14, 15, 16, 22, 23, 56, 62]:
    if sizes[c] > 30:
        ax2.text(centroids_3d[c, 0]*1.15, centroids_3d[c, 1]*1.15,
                 centroids_3d[c, 2]*1.15,
                 COCO_CLASSES[c], color='white', fontsize=7, ha='center')

# Panel 3: 50 random with anchor connections
ax3 = fig.add_subplot(233, projection='3d')
setup_ax(ax3, '50 Random — nearest anchor connections')
np.random.seed(42)
idx50 = np.random.choice(N_val, 50, replace=False)
emb_50 = emb_s[idx50]
colors_50 = class_colors[idx50]
with torch.no_grad():
    cos_50 = fused[idx50] @ anchors.T
    nearest_50 = cos_50.argmax(-1).numpy()
ax3.scatter(anchors_s[:, 0], anchors_s[:, 1], anchors_s[:, 2],
            c='red', s=4, alpha=0.2, marker='^')
ax3.scatter(emb_50[:, 0], emb_50[:, 1], emb_50[:, 2],
            c=colors_50, s=40, alpha=0.9, edgecolors='white', linewidth=0.5)
for i in range(50):
    a = nearest_50[i]
    ax3.plot([emb_50[i, 0], anchors_s[a, 0]],
             [emb_50[i, 1], anchors_s[a, 1]],
             [emb_50[i, 2], anchors_s[a, 2]],
             color='yellow', alpha=0.3, linewidth=0.5)

# Panel 4: 10 random — triangulation heatmap
ax4 = fig.add_subplot(234, projection='3d')
setup_ax(ax4, '10 Random — anchor affinity heatmap')
idx10 = np.random.choice(N_val, 10, replace=False)
emb_10 = emb_s[idx10]
with torch.no_grad():
    cos_10 = (fused[idx10] @ anchors.T).numpy()
    mean_cos = cos_10.mean(0)
anchor_heat = (mean_cos - mean_cos.min()) / (mean_cos.max() - mean_cos.min() + 1e-8)
anchor_colors = plt.cm.hot(anchor_heat)
ax4.scatter(anchors_s[:, 0], anchors_s[:, 1], anchors_s[:, 2],
            c=anchor_colors, s=10, alpha=0.6)
ax4.scatter(emb_10[:, 0], emb_10[:, 1], emb_10[:, 2],
            c='cyan', s=80, alpha=1.0, edgecolors='white', linewidth=1, zorder=10)

# Panel 5: Single encoding
ax5 = fig.add_subplot(235, projection='3d')
single_idx = 42
single_class = primary_class[single_idx]
setup_ax(ax5, f'Single Encoding: "{COCO_CLASSES[single_class]}" — top 5 anchors')
with torch.no_grad():
    cos_single = (fused[single_idx] @ anchors.T).numpy()
single_heat = (cos_single - cos_single.min()) / (cos_single.max() - cos_single.min() + 1e-8)
single_colors = plt.cm.plasma(single_heat)
single_sizes = 2 + 50 * single_heat**3
ax5.scatter(anchors_s[:, 0], anchors_s[:, 1], anchors_s[:, 2],
            c=single_colors, s=single_sizes, alpha=0.7)
single_pt = emb_s[single_idx]
ax5.scatter([single_pt[0]], [single_pt[1]], [single_pt[2]],
            c='lime', s=150, alpha=1.0, edgecolors='white', linewidth=2,
            zorder=10, marker='*')
top5 = np.argsort(cos_single)[::-1][:5]
for a in top5:
    ax5.plot([single_pt[0], anchors_s[a, 0]],
             [single_pt[1], anchors_s[a, 1]],
             [single_pt[2], anchors_s[a, 2]],
             color='lime', alpha=0.6, linewidth=1.5)

# Panel 6: Radial deviation
ax6 = fig.add_subplot(236, projection='3d')
radii = np.linalg.norm(emb_3d, axis=-1)
setup_ax(ax6, f'PCA Projection Radii — mean={radii.mean():.4f} std={radii.std():.4f}')
radius_dev = radii - radii.mean()
dev_norm = (radius_dev - radius_dev.min()) / (radius_dev.max() - radius_dev.min() + 1e-8)
dev_colors = plt.cm.coolwarm(dev_norm)
scale = 1.0 / radii.max()
ax6.scatter(emb_3d[:, 0]*scale, emb_3d[:, 1]*scale, emb_3d[:, 2]*scale,
            c=dev_colors, s=2, alpha=0.4)

plt.tight_layout(rect=[0, 0, 1, 0.95])
plt.savefig("hypersphere_manifold.png", dpi=200, facecolor='black',
            bbox_inches='tight', pad_inches=0.3)
print("Saved: hypersphere_manifold.png")
plt.close()


# ══════════════════════════════════════════════════════════════════
# FIGURE 2: EXPERT PERSPECTIVES
# ══════════════════════════════════════════════════════════════════

print("Rendering figure 2...")
fig2 = plt.figure(figsize=(21, 7), facecolor='black')
fig2.suptitle('Expert Perspective Divergence — Same sphere, three lenses',
              color='white', fontsize=14, y=1.02)

has_expert_rot = f"constellation.expert_rotations.0" in sd
if has_expert_rot:
    expert_R = [sd[f"constellation.expert_rotations.{i}"] for i in range(3)]
    expert_W = [sd[f"constellation.expert_whiteners.{i}"] for i in range(3)]
    expert_mu = [sd[f"constellation.expert_means.{i}"] for i in range(3)]
else:
    expert_R = [torch.eye(D_ANCHOR) for _ in range(3)]
    expert_W = [torch.eye(D_ANCHOR) for _ in range(3)]
    expert_mu = [torch.zeros(D_ANCHOR) for _ in range(3)]

with torch.no_grad():
    for i, name in enumerate(EXPERTS):
        ax = fig2.add_subplot(1, 3, i+1, projection='3d')

        if has_expert_rot:
            centered = fused.float() - expert_mu[i]
            whitened = centered @ expert_W[i]
            rotated = F.normalize(whitened @ expert_R[i].T, dim=-1)
        elif f"projectors.{i}.proj_native.0.weight" in sd:
            W = sd[f"projectors.{i}.proj_native.0.weight"]
            b = sd[f"projectors.{i}.proj_native.0.bias"]
            lw = sd[f"projectors.{i}.proj_native.1.weight"]
            lb = sd[f"projectors.{i}.proj_native.1.bias"]
            x = val_raw[name] @ W.T + b
            mu_v = x.mean(-1, keepdim=True); var_v = x.var(-1, keepdim=True, unbiased=False)
            x = (x - mu_v) / (var_v + 1e-5).sqrt() * lw + lb
            rotated = F.normalize(x, dim=-1)
        else:
            rotated = projected[i]

        rot_np = rotated.numpy()
        rot_c = rot_np - rot_np.mean(axis=0, keepdims=True)
        _, S_r, Vt_r = np.linalg.svd(rot_c[:5000], full_matrices=False)
        rot_3d = to_sphere(rot_np @ Vt_r[:3].T)

        var_exp = S_r[:3]**2 / (S_r**2).sum()
        setup_ax(ax, f'{name[:25]}\nPC variance: {var_exp.sum()*100:.1f}%')
        ax.scatter(rot_3d[:, 0], rot_3d[:, 1], rot_3d[:, 2],
                   c=class_colors, s=2, alpha=0.4)

plt.tight_layout()
plt.savefig("expert_perspectives.png", dpi=200, facecolor='black',
            bbox_inches='tight', pad_inches=0.3)
print("Saved: expert_perspectives.png")
plt.close()


# ══════════════════════════════════════════════════════════════════
# FIGURE 3: ANCHORS ONLY
# ══════════════════════════════════════════════════════════════════

print("Rendering figure 3 — anchors only...")

# Anchor visit counts for coloring
with torch.no_grad():
    cos_all = fused @ anchors.T
    nearest_all = cos_all.argmax(dim=-1)
    vc = torch.zeros(N_ANCHORS)
    for n in nearest_all:
        vc[n] += 1
    vc_np = vc.numpy()

fig3 = plt.figure(figsize=(24, 8), facecolor='black')
fig3.suptitle(f'Constellation — {N_ANCHORS} anchors × {D_ANCHOR}-d on S²⁵⁵',
              color='white', fontsize=14, y=1.02)

# Panel 1: Anchors colored by visit count
ax_a1 = fig3.add_subplot(131, projection='3d')
setup_ax(ax_a1, f'Anchor Utilization — {int((vc_np>0).sum())}/{N_ANCHORS} active')
heat = np.zeros(N_ANCHORS)
active_mask = vc_np > 0
heat[active_mask] = np.log1p(vc_np[active_mask])
heat = heat / (heat.max() + 1e-8)
a_colors = plt.cm.inferno(heat)
a_sizes = 5 + 60 * heat
# Dead anchors in blue
dead_mask = vc_np == 0
ax_a1.scatter(anchors_s[dead_mask, 0], anchors_s[dead_mask, 1], anchors_s[dead_mask, 2],
              c='dodgerblue', s=8, alpha=0.4, marker='x', label=f'dead ({int(dead_mask.sum())})')
ax_a1.scatter(anchors_s[active_mask, 0], anchors_s[active_mask, 1], anchors_s[active_mask, 2],
              c=a_colors[active_mask], s=a_sizes[active_mask], alpha=0.8)

# Panel 2: Anchors colored by nearest neighbor distance
ax_a2 = fig3.add_subplot(132, projection='3d')
anchor_sim = (anchors.numpy() @ anchors.numpy().T)
np.fill_diagonal(anchor_sim, -1)
max_neighbor_cos = anchor_sim.max(axis=1)
nn_heat = (max_neighbor_cos - max_neighbor_cos.min()) / (max_neighbor_cos.max() - max_neighbor_cos.min() + 1e-8)
nn_colors = plt.cm.viridis(nn_heat)
setup_ax(ax_a2, f'Anchor Isolation — nearest neighbor cosine\n'
         f'mean={max_neighbor_cos.mean():.3f} max={max_neighbor_cos.max():.3f}')
ax_a2.scatter(anchors_s[:, 0], anchors_s[:, 1], anchors_s[:, 2],
              c=nn_colors, s=15, alpha=0.8)

# Panel 3: Anchors colored by expert divergence at that anchor
ax_a3 = fig3.add_subplot(133, projection='3d')
with torch.no_grad():
    expert_tri_stack = []
    if has_expert_rot:
        for i in range(3):
            centered = fused.float() - expert_mu[i]
            whitened = centered @ expert_W[i]
            rotated = F.normalize(whitened @ expert_R[i].T, dim=-1)
            expert_tri_stack.append(1.0 - (rotated @ anchors.T))
    elif f"projectors.0.proj_native.0.weight" in sd:
        def _pn(feats, i):
            W = sd[f"projectors.{i}.proj_native.0.weight"]
            b = sd[f"projectors.{i}.proj_native.0.bias"]
            lw = sd[f"projectors.{i}.proj_native.1.weight"]
            lb = sd[f"projectors.{i}.proj_native.1.bias"]
            x = feats @ W.T + b
            mu = x.mean(-1, keepdim=True); var = x.var(-1, keepdim=True, unbiased=False)
            x = (x - mu) / (var + 1e-5).sqrt() * lw + lb
            return F.normalize(x, dim=-1)
        for i, name in enumerate(EXPERTS):
            nat = _pn(val_raw[name], i)
            expert_tri_stack.append(1.0 - (nat @ anchors.T))
    else:
        for p in projected:
            expert_tri_stack.append(1.0 - (p @ anchors.T))
    tri_stack = torch.stack(expert_tri_stack, dim=-1)
    per_anchor_div = tri_stack.std(dim=-1).mean(dim=0).numpy()

div_heat = (per_anchor_div - per_anchor_div.min()) / (per_anchor_div.max() - per_anchor_div.min() + 1e-8)
div_colors = plt.cm.coolwarm(div_heat)
setup_ax(ax_a3, f'Expert Divergence per Anchor\n'
         f'mean={per_anchor_div.mean():.4f} range=[{per_anchor_div.min():.4f}, {per_anchor_div.max():.4f}]')
ax_a3.scatter(anchors_s[:, 0], anchors_s[:, 1], anchors_s[:, 2],
              c=div_colors, s=15, alpha=0.8)

# Add connections between closest anchor pairs (top 20)
flat_sim = anchor_sim.copy()
np.fill_diagonal(flat_sim, -999)
for panel_ax in [ax_a1, ax_a2]:
    for _ in range(20):
        idx_flat = np.argmax(flat_sim)
        i_a, j_a = np.unravel_index(idx_flat, flat_sim.shape)
        flat_sim[i_a, j_a] = -999; flat_sim[j_a, i_a] = -999
        panel_ax.plot([anchors_s[i_a, 0], anchors_s[j_a, 0]],
                      [anchors_s[i_a, 1], anchors_s[j_a, 1]],
                      [anchors_s[i_a, 2], anchors_s[j_a, 2]],
                      color='white', alpha=0.15, linewidth=0.5)

plt.tight_layout()
plt.savefig("anchors_only.png", dpi=200, facecolor='black',
            bbox_inches='tight', pad_inches=0.3)
print("Saved: anchors_only.png")
plt.close()


# ══════════════════════════════════════════════════════════════════
# FIGURE 4: PAIRWISE EXPERT DIFFERENCES
# ══════════════════════════════════════════════════════════════════

print("Rendering figure 4 — pairwise expert diffs...")

with torch.no_grad():
    # Compute per-expert triangulations
    # For dual-stream: use native projectors (the actual expert perspectives)
    # For fused constellation: use expert rotations
    expert_tris = []

    if has_expert_rot:
        # Fused constellation: rotate through R/W/mu
        for i in range(3):
            centered = fused.float() - expert_mu[i]
            whitened = centered @ expert_W[i]
            rotated = F.normalize(whitened @ expert_R[i].T, dim=-1)
            tri = 1.0 - (rotated @ anchors.T)
            expert_tris.append(tri)
    elif f"projectors.0.proj_native.0.weight" in sd:
        # Dual-stream: use native projector embeddings
        def _proj_native(feats, i):
            W = sd[f"projectors.{i}.proj_native.0.weight"]
            b = sd[f"projectors.{i}.proj_native.0.bias"]
            lw = sd[f"projectors.{i}.proj_native.1.weight"]
            lb = sd[f"projectors.{i}.proj_native.1.bias"]
            x = feats @ W.T + b
            mu = x.mean(-1, keepdim=True); var = x.var(-1, keepdim=True, unbiased=False)
            x = (x - mu) / (var + 1e-5).sqrt() * lw + lb
            return F.normalize(x, dim=-1)
        for i, name in enumerate(EXPERTS):
            native_emb = _proj_native(val_raw[name], i)
            tri = 1.0 - (native_emb @ anchors.T)
            expert_tris.append(tri)
    else:
        # Fallback: use shared projections (will be near-identical)
        for p in projected:
            tri = 1.0 - (p @ anchors.T)
            expert_tris.append(tri)

    # Pairwise diffs
    diff_cd = expert_tris[0] - expert_tris[1]
    diff_cs = expert_tris[0] - expert_tris[2]
    diff_ds = expert_tris[1] - expert_tris[2]
    diffs = [diff_cd, diff_cs, diff_ds]
    diff_names = ["CLIP − DINOv2", "CLIP − SigLIP", "DINOv2 − SigLIP"]

    abs_tri = expert_tris[0]

    print(f"\n  Pairwise diff statistics:")
    for name, d in zip(diff_names, diffs):
        print(f"    {name:20s}: mean={d.mean():.6f} std={d.std():.6f} "
              f"min={d.min():.6f} max={d.max():.6f}")
    print(f"    Absolute tri std:    {abs_tri.std():.6f}")
    diff_std = diffs[0].std().item()
    abs_std = abs_tri.std().item()
    print(f"    Ratio (diff/abs):    {diff_std / abs_std:.4f}" if abs_std > 1e-10 else
          f"    Ratio (diff/abs):    N/A (zero abs std)")

    # PCA of the diff space
    diff_stacked = torch.cat(diffs, dim=-1).numpy()
    diff_centered = diff_stacked - diff_stacked.mean(axis=0, keepdims=True)
    _, S_diff, Vt_diff = np.linalg.svd(diff_centered[:5000], full_matrices=False)

    # Guard against zero SVDs
    s_sum = (S_diff**2).sum()
    if s_sum > 1e-20:
        diff_3d = to_sphere(diff_centered @ Vt_diff[:3].T)
        var_diff = S_diff[:3]**2 / s_sum
        eff_dim_diff = float(((S_diff / S_diff.sum())**2).sum()**-1)
    else:
        diff_3d = np.zeros((len(diff_centered), 3))
        var_diff = np.zeros(3)
        eff_dim_diff = 0.0
    print(f"\n  Diff space effective dim: {eff_dim_diff:.1f}")
    print(f"  Diff PCA 3D variance: {var_diff.sum()*100:.1f}%")

    abs_stacked = abs_tri.numpy()
    abs_centered = abs_stacked - abs_stacked.mean(axis=0, keepdims=True)
    _, S_abs, Vt_abs = np.linalg.svd(abs_centered[:5000], full_matrices=False)
    abs_eff = float(((S_abs / S_abs.sum())**2).sum()**-1) if S_abs.sum() > 1e-20 else 0.0
    print(f"  Absolute tri effective dim: {abs_eff:.1f}")

    full_stacked = np.concatenate([abs_stacked, diff_stacked], axis=-1)
    full_centered = full_stacked - full_stacked.mean(axis=0, keepdims=True)
    _, S_full, Vt_full = np.linalg.svd(full_centered[:5000], full_matrices=False)
    full_eff = float(((S_full / S_full.sum())**2).sum()**-1) if S_full.sum() > 1e-20 else 0.0
    full_3d = to_sphere(full_centered @ Vt_full[:3].T) if S_full.sum() > 1e-20 else np.zeros((len(full_centered), 3))
    print(f"  Full (abs+diffs) effective dim: {full_eff:.1f}")
    print(f"  Information gain from diffs: {full_eff - abs_eff:.1f} dimensions")

fig4 = plt.figure(figsize=(28, 14), facecolor='black')
fig4.suptitle(
    'Expert Pairwise Differences — Where the discriminative signal lives\n'
    f'Diff eff_dim={eff_dim_diff:.1f} | Abs eff_dim={abs_eff:.1f} | '
    f'Combined eff_dim={full_eff:.1f} | Info gain: +{full_eff-abs_eff:.1f} dims',
    color='white', fontsize=14, y=0.98)

# Row 1: Three pairwise diff distributions on sphere
for col, (name, d) in enumerate(zip(diff_names, diffs)):
    ax = fig4.add_subplot(2, 4, col+1, projection='3d')
    d_np = d.numpy()

    # Per-image: magnitude of diff vector
    diff_mag = np.linalg.norm(d_np, axis=-1)
    mag_heat = (diff_mag - diff_mag.min()) / (diff_mag.max() - diff_mag.min() + 1e-8)
    mag_colors = plt.cm.magma(mag_heat)

    setup_ax(ax, f'{name}\nstd={d_np.std():.5f}')
    ax.scatter(emb_s[:, 0], emb_s[:, 1], emb_s[:, 2],
               c=mag_colors, s=2, alpha=0.5)

# Panel 4: Diff space PCA
ax_dp = fig4.add_subplot(244, projection='3d')
setup_ax(ax_dp, f'Diff Space PCA\neff_dim={eff_dim_diff:.1f} var={var_diff.sum()*100:.1f}%')
ax_dp.scatter(diff_3d[:, 0], diff_3d[:, 1], diff_3d[:, 2],
              c=class_colors, s=2, alpha=0.4)

# Row 2: Per-anchor diff analysis
# Per-anchor mean absolute diff (where do experts disagree most?)
with torch.no_grad():
    per_anchor_cd = diff_cd.abs().mean(dim=0).numpy()
    per_anchor_cs = diff_cs.abs().mean(dim=0).numpy()
    per_anchor_ds = diff_ds.abs().mean(dim=0).numpy()
    per_anchor_total = (per_anchor_cd + per_anchor_cs + per_anchor_ds) / 3

# Panel 5: Anchor-level divergence map (total)
ax_a = fig4.add_subplot(245, projection='3d')
total_heat = (per_anchor_total - per_anchor_total.min()) / (per_anchor_total.max() - per_anchor_total.min() + 1e-8)
total_colors = plt.cm.hot(total_heat)
total_sizes = 5 + 40 * total_heat
setup_ax(ax_a, f'Anchor Divergence (all pairs)\n'
         f'range=[{per_anchor_total.min():.5f}, {per_anchor_total.max():.5f}]')
ax_a.scatter(anchors_s[:, 0], anchors_s[:, 1], anchors_s[:, 2],
             c=total_colors, s=total_sizes, alpha=0.8)

# Panel 6: Abs tri PCA vs diff PCA side by side
ax_abs = fig4.add_subplot(246, projection='3d')
abs_3d = to_sphere(abs_centered @ Vt_abs[:3].T)
var_abs_3 = S_abs[:3]**2 / (S_abs**2).sum()
setup_ax(ax_abs, f'Absolute Tri PCA\neff_dim={abs_eff:.1f} var={var_abs_3.sum()*100:.1f}%')
ax_abs.scatter(abs_3d[:, 0], abs_3d[:, 1], abs_3d[:, 2],
               c=class_colors, s=2, alpha=0.4)

# Panel 7: Combined PCA
ax_full = fig4.add_subplot(247, projection='3d')
var_full_3 = S_full[:3]**2 / (S_full**2).sum()
setup_ax(ax_full, f'Combined (abs+diffs) PCA\neff_dim={full_eff:.1f} var={var_full_3.sum()*100:.1f}%')
ax_full.scatter(full_3d[:, 0], full_3d[:, 1], full_3d[:, 2],
                c=class_colors, s=2, alpha=0.4)

# Panel 8: Histogram of diff magnitudes
ax_hist = fig4.add_subplot(248)
ax_hist.set_facecolor('black')
for name, d, color in zip(diff_names, diffs,
                           ['#ff6b6b', '#4ecdc4', '#ffe66d']):
    d_np = d.numpy()
    per_image_mag = np.linalg.norm(d_np, axis=-1)
    ax_hist.hist(per_image_mag, bins=50, alpha=0.6, color=color,
                 label=name, density=True)
ax_hist.set_xlabel('Diff magnitude (L2)', color='white', fontsize=9)
ax_hist.set_ylabel('Density', color='white', fontsize=9)
ax_hist.set_title('Per-image diff magnitudes', color='white', fontsize=11)
ax_hist.legend(fontsize=8, facecolor='black', edgecolor='gray',
               labelcolor='white')
ax_hist.tick_params(colors='gray', labelsize=7)
ax_hist.spines['bottom'].set_color('gray'); ax_hist.spines['left'].set_color('gray')
ax_hist.spines['top'].set_visible(False); ax_hist.spines['right'].set_visible(False)

plt.tight_layout(rect=[0, 0, 1, 0.95])
plt.savefig("pairwise_diffs.png", dpi=200, facecolor='black',
            bbox_inches='tight', pad_inches=0.3)
print("Saved: pairwise_diffs.png")
plt.close()

print("\nDone.")