const headerRowTop = document.getElementById('header-row-top'); const headerRowSub = document.getElementById('header-row-sub'); const bodyRow = document.getElementById('body-row'); const searchInput = document.getElementById('search'); const typeFilters = Array.from(document.querySelectorAll('.type-filter')); const copyCitationBtn = document.getElementById('copy-citation-btn'); const citationText = document.getElementById('citation-text'); const GROUPS = [ { id: 'avg', label: 'Avg. (5)', prefix: 'avg' }, { id: 'lc', label: 'LangChain', prefix: 'lc' }, { id: 'yolo', label: 'Yolo v7 & v8', prefix: 'yolo' }, { id: 'laravel', label: 'Laravel 10 & 11', prefix: 'laravel' }, { id: 'angular', label: 'Angular 16, 17 & 18', prefix: 'angular' }, { id: 'godot', label: 'Godot4', prefix: 'godot' } ]; const METRICS = [ { id: 'a10', label: 'α@10' }, { id: 'c20', label: 'C@20' }, { id: 'r50', label: 'R@50' } ]; let rows = []; let sortKey = 'avg_r50'; let sortAsc = false; const PLOT_METRICS = [ { id: 'alpha_ndcg_10', key: 'avg_a10', plotId: 'plot-avg-alpha10', title: 'alpha-nDCG@10', yLabel: 'α@10 (Avg. 5)', yMin: 0.1, yMax: 0.541 }, { id: 'coverage_20', key: 'avg_c20', plotId: 'plot-avg-c20', title: 'Coverage@20', yLabel: 'C@20 (Avg. 5)', yMin: 0.25, yMax: 0.868 }, { id: 'recall_50', key: 'avg_r50', plotId: 'plot-avg-r50', title: 'Recall@50', yLabel: 'R@50 (Avg. 5)', yMin: 0.15, yMax: 0.755 } ]; const DATE_PLOT_METRICS = [ { id: 'alpha_ndcg_10', key: 'avg_a10', plotId: 'plot-date-avg-alpha10', title: 'alpha-nDCG@10', yLabel: 'α@10 (Avg. 5)', yMin: 0.1, yMax: 0.541 }, { id: 'coverage_20', key: 'avg_c20', plotId: 'plot-date-avg-c20', title: 'Coverage@20', yLabel: 'C@20 (Avg. 5)', yMin: 0.25, yMax: 0.868 }, { id: 'recall_50', key: 'avg_r50', plotId: 'plot-date-avg-r50', title: 'Recall@50', yLabel: 'R@50 (Avg. 5)', yMin: 0.15, yMax: 0.755 } ]; function num(v) { return typeof v === 'number' ? v.toFixed(3) : '-'; } function typeBadge(type) { const labels = { open_source: 'Open Source', proprietary: 'Proprietary', upper_baseline: 'Oracle' }; return `${labels[type] || type}`; } function isNewModel(dateStr) { if (!dateStr) return false; const modelDate = new Date(dateStr); if (Number.isNaN(modelDate.getTime())) return false; const now = new Date(); const daysDiff = (now - modelDate) / (1000 * 60 * 60 * 24); return daysDiff >= 0 && daysDiff <= 90; } function parseSizeToBillions(sizeStr) { if (!sizeStr || sizeStr === '-') return null; const m = String(sizeStr).trim().match(/^([\d.]+)\s*([BMK])$/i); if (!m) return null; const numValue = parseFloat(m[1]); const unit = m[2].toUpperCase(); if (Number.isNaN(numValue)) return null; if (unit === 'B') return numValue; if (unit === 'M') return numValue / 1000; if (unit === 'K') return numValue / 1e6; return null; } function formatParameterSize(sizeInBillions) { if (sizeInBillions === undefined || sizeInBillions === null || Number.isNaN(sizeInBillions)) return '-'; if (sizeInBillions < 1) { const inMillions = sizeInBillions * 1000; return `${parseFloat(inMillions.toFixed(2))}M`; } return `${parseFloat(sizeInBillions.toFixed(3))}B`; } function inferFamily(name) { const n = String(name || '').toLowerCase(); if (n.includes('stella') || n.includes('jasper')) return 'Stella'; if (n.includes('harrier')) return 'Harrier OSS'; if (n.includes('voyage')) return 'Voyage'; if (n.includes('jina')) return 'Jina'; if (n.includes('qwen3')) return 'Qwen3'; if (n.includes('granite')) return 'IBM Granite'; if (n.includes('arctic embed')) return 'Arctic Embed'; if (n.includes('perplexity embed')) return 'Perplexity Embed'; if (n.includes('nomic embed') || n.includes('coderankembed')) return 'Nomic Embed'; if (n.includes('bge')) return 'BGE'; if (n.includes('e5')) return 'E5'; if (n.includes('gte')) return 'GTE'; if (n.includes('bm25')) return 'BM25'; if (n.includes('fusion')) return 'Fusion'; return 'Other'; } function normalizeModelName(rawName) { return String(rawName || '').toLowerCase().replace(/^oracle:\s*/i, '').trim(); } function isReferenceBaselineModel(rawName) { const name = normalizeModelName(rawName); return name === 'bm25' || name === 'fusion (bm25, bge, e5, voyage)'; } function parseReleaseDate(dateStr) { if (!dateStr) return null; const d = new Date(dateStr); return Number.isNaN(d.getTime()) ? null : d; } const FAMILY_COLORS = { 'Stella': '#1f77b4', 'Harrier OSS': '#ff7f0e', 'Voyage': '#009688', 'Jina': '#d62728', 'Qwen3': '#9467bd', 'IBM Granite': '#8c564b', 'Arctic Embed': '#e377c2', 'Perplexity Embed': '#17becf', 'Nomic Embed': '#6a1b9a', 'BGE': '#7f7f7f', 'E5': '#393b79', 'GTE': '#bcbd22', 'BM25': '#969696', 'Fusion': '#e6550d', 'Other': '#9e9e9e' }; function familyMarkerHtml(name, type) { const family = inferFamily(name); const color = FAMILY_COLORS[family] || '#9e9e9e'; // Shape by model type; color by model family const symbol = type === 'proprietary' ? '◆' : '●'; return `${symbol}`; } function mapRow(item) { return { name: item.info.name, size: item.info.size, date: item.info.date, type: item.info.type, link: item.info.link || '', avg_a10: item.datasets.average.alpha_ndcg_10, avg_c20: item.datasets.average.coverage_20, avg_r50: item.datasets.average.recall_50, lc_a10: item.datasets.langchain.alpha_ndcg_10, lc_c20: item.datasets.langchain.coverage_20, lc_r50: item.datasets.langchain.recall_50, yolo_a10: item.datasets.yolo.alpha_ndcg_10, yolo_c20: item.datasets.yolo.coverage_20, yolo_r50: item.datasets.yolo.recall_50, laravel_a10: item.datasets.laravel.alpha_ndcg_10, laravel_c20: item.datasets.laravel.coverage_20, laravel_r50: item.datasets.laravel.recall_50, angular_a10: item.datasets.angular.alpha_ndcg_10, angular_c20: item.datasets.angular.coverage_20, angular_r50: item.datasets.angular.recall_50, godot_a10: item.datasets.godot.alpha_ndcg_10, godot_c20: item.datasets.godot.coverage_20, godot_r50: item.datasets.godot.recall_50 }; } function renderHeaders() { headerRowTop.innerHTML = ` # Retriever Type Params Date ${GROUPS.map(g => `${g.label}`).join('')} `; headerRowSub.innerHTML = GROUPS.map(g => METRICS.map(m => { const key = `${g.prefix}_${m.id}`; const arrow = sortKey === key ? (sortAsc ? ' ↑' : ' ↓') : ''; return `${m.label}${arrow}`; }).join('') ).join(''); document.querySelectorAll('th[data-key]').forEach(th => { th.addEventListener('click', () => { const key = th.dataset.key; if (!key || key === 'rank') return; if (sortKey === key) sortAsc = !sortAsc; else { sortKey = key; sortAsc = false; } renderHeaders(); renderBody(); }); }); } function activeTypes() { return typeFilters.filter(cb => cb.checked).map(cb => cb.value); } function renderBody() { const q = searchInput.value.trim().toLowerCase(); const types = activeTypes(); let filtered = rows.filter(r => types.includes(r.type) && (`${r.name} ${r.size} ${r.date} ${r.type}`).toLowerCase().includes(q) ); filtered.sort((a, b) => { const av = a[sortKey]; const bv = b[sortKey]; if (typeof av === 'number' && typeof bv === 'number') return sortAsc ? av - bv : bv - av; return sortAsc ? String(av).localeCompare(String(bv)) : String(bv).localeCompare(String(av)); }); bodyRow.innerHTML = filtered.map((r, idx) => ` ${idx + 1} ${familyMarkerHtml(r.name, r.type)} ${r.link ? `${r.name}` : r.name} ${isNewModel(r.date) ? '🆕' : ''} ${typeBadge(r.type)} ${r.size || '-'} ${r.date || '-'} ${num(r.avg_a10)}${num(r.avg_c20)}${num(r.avg_r50)} ${num(r.lc_a10)}${num(r.lc_c20)}${num(r.lc_r50)} ${num(r.yolo_a10)}${num(r.yolo_c20)}${num(r.yolo_r50)} ${num(r.laravel_a10)}${num(r.laravel_c20)}${num(r.laravel_r50)} ${num(r.angular_a10)}${num(r.angular_c20)}${num(r.angular_r50)} ${num(r.godot_a10)}${num(r.godot_c20)}${num(r.godot_r50)} `).join(''); } function renderPlots() { if (typeof Plotly === 'undefined') return; const active = activeTypes(); const filtered = rows.filter(r => active.includes(r.type)); const filteredNoBaselines = filtered.filter(r => !isReferenceBaselineModel(r.name)); PLOT_METRICS.forEach(metric => { const grouped = {}; filteredNoBaselines.forEach(r => { const x = parseSizeToBillions(r.size); const y = r[metric.key]; if (x === null || typeof y !== 'number') return; const fam = inferFamily(r.name); if (!grouped[fam]) grouped[fam] = { x: [], y: [], text: [] }; grouped[fam].x.push(x); grouped[fam].y.push(y); grouped[fam].text.push(r.name); }); const traces = Object.keys(grouped).sort().map(fam => ({ type: 'scatter', mode: 'markers', name: fam, x: grouped[fam].x, y: grouped[fam].y, text: grouped[fam].text, customdata: grouped[fam].x.map(v => formatParameterSize(v)), marker: { color: FAMILY_COLORS[fam] || '#9e9e9e', size: 11, line: { width: 1, color: '#fff' } }, hovertemplate: '%{text}
Params: %{customdata}
Score: %{y:.3f}' })); const bm25 = filtered.find(r => String(r.name).toLowerCase() === 'bm25'); const fusion = filtered.find(r => String(r.name).toLowerCase() === 'fusion (bm25, bge, e5, voyage)'); const xs = traces.flatMap(t => t.x || []); if (xs.length) { const xmin = Math.min(...xs); const xmax = Math.max(...xs); if (bm25 && typeof bm25[metric.key] === 'number') { traces.push({ type: 'scatter', mode: 'lines', name: 'BM25', x: [xmin, xmax], y: [bm25[metric.key], bm25[metric.key]], line: { color: 'rgba(97,97,97,0.55)', width: 1.1, dash: 'dash' } }); } if (fusion && typeof fusion[metric.key] === 'number') { traces.push({ type: 'scatter', mode: 'lines', name: 'Fusion', x: [xmin, xmax], y: [fusion[metric.key], fusion[metric.key]], line: { color: 'rgba(106,27,154,0.55)', width: 1.1, dash: 'dot' } }); } } Plotly.newPlot(metric.plotId, traces, { title: { text: metric.title, x: 0.01, xanchor: 'left', font: { size: 16 } }, height: 460, margin: { t: 46, r: 12, b: 178, l: 56 }, xaxis: { title: { text: 'Model Parameters (Billions)', standoff: 26 }, type: 'log', automargin: true, showgrid: true }, yaxis: { title: metric.yLabel, range: [metric.yMin, metric.yMax], tickformat: '.2f', automargin: true, showgrid: true }, legend: { orientation: 'h', y: -0.46, x: 0.5, xanchor: 'center' }, hovermode: 'closest' }, { responsive: true, displaylogo: false }); }); DATE_PLOT_METRICS.forEach(metric => { const grouped = {}; filteredNoBaselines.forEach(r => { const x = parseReleaseDate(r.date); const y = r[metric.key]; if (x === null || typeof y !== 'number') return; const fam = inferFamily(r.name); if (!grouped[fam]) grouped[fam] = { x: [], y: [], text: [] }; grouped[fam].x.push(x); grouped[fam].y.push(y); grouped[fam].text.push(r.name); }); const traces = Object.keys(grouped).sort().map(fam => ({ type: 'scatter', mode: 'markers', name: fam, x: grouped[fam].x, y: grouped[fam].y, text: grouped[fam].text, marker: { color: FAMILY_COLORS[fam] || '#9e9e9e', size: 11, line: { width: 1, color: '#fff' } }, hovertemplate: '%{text}
Release date: %{x|%Y-%m-%d}
Score: %{y:.3f}' })); const bm25 = filtered.find(r => normalizeModelName(r.name) === 'bm25'); const fusion = filtered.find(r => normalizeModelName(r.name) === 'fusion (bm25, bge, e5, voyage)'); const xs = traces.flatMap(t => t.x || []); const xMin = xs.length ? new Date(Math.min(...xs.map(d => d.getTime()))) : null; const xMax = xs.length ? new Date(Math.max(...xs.map(d => d.getTime()))) : null; const xMinMonthStart = xMin ? new Date(xMin.getFullYear(), xMin.getMonth(), 1) : null; if (xMin && xMax) { if (bm25 && typeof bm25[metric.key] === 'number') { traces.push({ type: 'scatter', mode: 'lines', name: 'BM25', x: [xMin, xMax], y: [bm25[metric.key], bm25[metric.key]], line: { color: 'rgba(97,97,97,0.55)', width: 1.1, dash: 'dash' } }); } if (fusion && typeof fusion[metric.key] === 'number') { traces.push({ type: 'scatter', mode: 'lines', name: 'Fusion', x: [xMin, xMax], y: [fusion[metric.key], fusion[metric.key]], line: { color: 'rgba(106,27,154,0.55)', width: 1.1, dash: 'dot' } }); } } Plotly.newPlot(metric.plotId, traces, { title: { text: metric.title, x: 0.01, xanchor: 'left', font: { size: 16 } }, height: 460, margin: { t: 46, r: 12, b: 172, l: 56 }, xaxis: { title: { text: 'Model Release Date', standoff: 26 }, type: 'date', tickmode: 'linear', tick0: xMinMonthStart ? xMinMonthStart.toISOString().slice(0, 10) : undefined, dtick: 'M1', tickformat: '%b %Y', tickangle: -45, automargin: true, showgrid: true }, yaxis: { title: metric.yLabel, range: [metric.yMin, metric.yMax], tickformat: '.2f', automargin: true, showgrid: true }, legend: { orientation: 'h', y: -0.45, x: 0.5, xanchor: 'center', entrywidthmode: 'pixels', entrywidth: 125, itemsizing: 'constant' }, hovermode: 'closest' }, { responsive: true, displaylogo: false }); }); } async function init() { const resp = await fetch('./leaderboard_data.json'); const data = await resp.json(); rows = data.leaderboardData.map(mapRow); renderHeaders(); renderBody(); renderPlots(); } searchInput.addEventListener('input', renderBody); typeFilters.forEach(cb => cb.addEventListener('change', () => { renderBody(); renderPlots(); })); document.getElementById('toggle-submit').addEventListener('click', () => { document.getElementById('submit-panel').classList.toggle('hidden'); }); document.getElementById('toggle-metrics').addEventListener('click', () => { document.getElementById('metrics-panel').classList.toggle('hidden'); }); copyCitationBtn.addEventListener('click', async () => { try { await navigator.clipboard.writeText(citationText.textContent); copyCitationBtn.innerHTML = ' Copied'; setTimeout(() => { copyCitationBtn.innerHTML = ' Copy'; }, 1400); } catch (_) {} }); init().catch(() => { bodyRow.innerHTML = 'Failed to load leaderboard data.'; });