Hub API: read compacted web-index parquets

Home page: 454 reqs (1 tree + 226 manifest + 227 rounds shards) → 2 reqs
(index/web/maps.parquet + index/web/rounds.parquet).
Match page: 10 per-player chunks-preview shards → 1 per-(match,map)
chunks-preview.parquet.

Drops the now-unused listTree helper and PLAYER_COUNT constant.

src/lib/api/hf.ts

@@ -1,5 +1,5 @@
-import { PLAYER_COUNT, type Match, type PreviewChunk, type Round } from '$lib/types';
-import { listTree, resolveUrl, type FetchOpts } from '$lib/api/hub';
+import type { Match, PreviewChunk, Round } from '$lib/types';
+import { resolveUrl, type FetchOpts } from '$lib/api/hub';
 import { fetchParquetRows } from '$lib/api/parquet';
 
 export type FetchOptions = FetchOpts;
@@ -8,50 +8,16 @@ let matchesPromise: Promise<Match[]> | null = null;
 let roundsPromise: Promise<Round[]> | null = null;
 const matchPreviewsCache = new Map<string, Promise<PreviewChunk[]>>();
 
-// The dataset is sharded by upload batch. Index files are named like
-// `index/manifest-<shard_id>.parquet` / `index/rounds-<shard_id>.parquet`.
-// Discover them at runtime so new shards don't require a code change.
-async function listIndexShards(
-	prefix: 'manifest' | 'rounds',
-	opts: FetchOptions
-): Promise<string[]> {
-	const entries = await listTree('index', opts);
-	return entries
-		.filter((e) => e.type === 'file')
-		.map((e) => e.path)
-		.filter((p) => {
-			const name = p.split('/').pop() ?? '';
-			return name.startsWith(`${prefix}-`) && name.endsWith('.parquet');
-		});
-}
-
-type IndexRow = { match_id: number; map_name: string; uploaded_at: string; round?: number };
-
-async function loadShardedIndex<T extends IndexRow>(
-	prefix: 'manifest' | 'rounds',
-	opts: FetchOptions
-): Promise<T[]> {
-	const paths = await listIndexShards(prefix, opts);
-	if (!paths.length) return [];
-	const shards = await Promise.all(paths.map((p) => fetchParquetRows<T>(resolveUrl(p), opts)));
-	// A re-rendered batch produces a second shard whose rows collide with the
-	// original on (match,map[,round]). Keep the freshest copy per key.
-	const keyOf =
-		prefix === 'manifest'
-			? (r: T) => `${r.match_id}|${r.map_name}`
-			: (r: T) => `${r.match_id}|${r.map_name}|${r.round}`;
-	const latest = new Map<string, T>();
-	for (const r of shards.flat()) {
-		const k = keyOf(r);
-		const cur = latest.get(k);
-		if (!cur || r.uploaded_at > cur.uploaded_at) latest.set(k, r);
-	}
-	return Array.from(latest.values());
-}
+// Compacted web-tier index. Single deduped parquet per scope, written by the
+// ingestion pipeline after each upload batch.
+const WEB_MAPS = 'index/web/maps.parquet';
+const WEB_ROUNDS = 'index/web/rounds.parquet';
+const matchPreviewsPath = (matchId: number, mapName: string) =>
+	`data/match_id=${matchId}/map_name=${mapName}/chunks-preview.parquet`;
 
 export function listMatches(opts: FetchOptions = {}): Promise<Match[]> {
 	if (matchesPromise) return matchesPromise;
-	matchesPromise = loadShardedIndex<Match>('manifest', opts)
+	matchesPromise = fetchParquetRows<Match>(resolveUrl(WEB_MAPS), opts)
 		.then((rows) => {
 			rows.sort(
 				(a, b) =>
@@ -67,13 +33,12 @@ export function listMatches(opts: FetchOptions = {}): Promise<Match[]> {
 	return matchesPromise;
 }
 
-export async function listAllRounds(opts: FetchOptions = {}): Promise<Round[]> {
-	if (!roundsPromise) {
-		roundsPromise = loadShardedIndex<Round>('rounds', opts).catch((err) => {
-			roundsPromise = null;
-			throw err;
-		});
-	}
+export function listAllRounds(opts: FetchOptions = {}): Promise<Round[]> {
+	if (roundsPromise) return roundsPromise;
+	roundsPromise = fetchParquetRows<Round>(resolveUrl(WEB_ROUNDS), opts).catch((err) => {
+		roundsPromise = null;
+		throw err;
+	});
 	return roundsPromise;
 }
 
@@ -89,9 +54,9 @@ export async function listRounds(
 }
 
 /**
- * Fetch all preview rows for every player on this (match, map) in parallel.
- * The shard_id from the match's manifest row is enough to construct each
- * player's parquet URL — no tree-API discovery needed.
+ * Single deduped per-(match, map) parquet with every player's preview rows.
+ * Each row's `preview_path` is relative to its own `player=N` dir, so the
+ * absolute URL is rebuilt using `r.player`.
  */
 async function loadMatchPreviews(
 	matchId: number,
@@ -102,32 +67,23 @@ async function loadMatchPreviews(
 	const cached = matchPreviewsCache.get(key);
 	if (cached) return cached;
 
-	const promise = (async () => {
-		const matches = await listMatches(opts);
-		const match = matches.find((m) => m.match_id === matchId && m.map_name === mapName);
-		if (!match) return [];
-
-		const players = Array.from({ length: PLAYER_COUNT }, (_, i) => i);
-		const results = await Promise.all(
-			players.map(async (player) => {
-				const dir = `data/match_id=${matchId}/map_name=${mapName}/player=${player}`;
-				try {
-					const rows = await fetchParquetRows<PreviewChunk>(
-						resolveUrl(`${dir}/chunks-preview-${match.shard_id}.parquet`),
-						opts
-					);
-					for (const r of rows) r.preview_video = { src: resolveUrl(`${dir}/${r.preview_path}`) };
-					return rows;
-				} catch {
-					return [] as PreviewChunk[];
-				}
-			})
-		);
-		return results.flat();
-	})().catch((err) => {
-		matchPreviewsCache.delete(key);
-		throw err;
-	});
+	const baseDir = `data/match_id=${matchId}/map_name=${mapName}`;
+	const promise = fetchParquetRows<PreviewChunk>(
+		resolveUrl(matchPreviewsPath(matchId, mapName)),
+		opts
+	)
+		.then((rows) => {
+			for (const r of rows) {
+				r.preview_video = {
+					src: resolveUrl(`${baseDir}/player=${r.player}/${r.preview_path}`)
+				};
+			}
+			return rows;
+		})
+		.catch((err) => {
+			matchPreviewsCache.delete(key);
+			throw err;
+		});
 
 	matchPreviewsCache.set(key, promise);
 	return promise;

src/lib/api/hub.ts

@@ -8,14 +8,3 @@ export type FetchOpts = { fetch?: typeof fetch; signal?: AbortSignal };
 export function resolveUrl(repoPath: string): string {
 	return `https://huggingface.co/datasets/${DATASET}/resolve/${REF}/${repoPath}`;
 }
-
-export type TreeEntry = { type: 'file' | 'directory'; path: string; size?: number };
-
-/** List entries under `<dataset>/<dirPath>` via the HF tree API. */
-export async function listTree(dirPath: string, opts: FetchOpts = {}): Promise<TreeEntry[]> {
-	const fetchFn = opts.fetch ?? fetch;
-	const url = `https://huggingface.co/api/datasets/${DATASET}/tree/${REF}/${dirPath}`;
-	const r = await fetchFn(url, { signal: opts.signal });
-	if (!r.ok) throw new Error(`tree ${dirPath}: ${r.status}`);
-	return (await r.json()) as TreeEntry[];
-}

src/lib/types.ts

@@ -1,5 +1,4 @@
 export const DATASET = 'blanchon/cs2_dataset_render';
-export const PLAYER_COUNT = 10;
 
 export type Winner = 'team1' | 'team2';
 export type WinnerSide = 'ct' | 't' | null;