/** * videoStorage.ts * * Downloads generated videos from Grok CDN and stores them in an * S3-compatible object storage (hi168.com OSS). * * Upload: @aws-sdk/lib-storage (multipart, handles large files) * Download / stream: @aws-sdk/client-s3 (GetObject) */ import { S3Client, CreateBucketCommand, HeadBucketCommand, GetObjectCommand, HeadObjectCommand, } from "@aws-sdk/client-s3"; import { Upload } from "@aws-sdk/lib-storage"; import { Readable } from "stream"; import { randomUUID } from "crypto"; // ─── S3 configuration ──────────────────────────────────────────────────────── const S3_ENDPOINT = process.env.S3_ENDPOINT ?? ""; const S3_BUCKET = process.env.S3_BUCKET ?? "starforge-videos"; const S3_REGION = process.env.S3_REGION ?? "us-east-1"; const ACCESS_KEY = process.env.S3_ACCESS_KEY ?? ""; const SECRET_KEY = process.env.S3_SECRET_KEY ?? ""; let s3: S3Client | null = null; function getS3(): S3Client { if (!s3) { s3 = new S3Client({ endpoint: S3_ENDPOINT, region: S3_REGION, credentials: { accessKeyId: ACCESS_KEY, secretAccessKey: SECRET_KEY }, forcePathStyle: true, // required for most non-AWS S3 endpoints }); } return s3; } /** Returns true if S3 credentials are configured. */ export function isStorageReady(): boolean { return !!(S3_ENDPOINT && ACCESS_KEY && SECRET_KEY); } // ─── Bucket init (create if missing) ───────────────────────────────────────── let bucketReady = false; async function ensureBucket(): Promise { if (bucketReady) return; const client = getS3(); try { await client.send(new HeadBucketCommand({ Bucket: S3_BUCKET })); bucketReady = true; console.log(`[video-storage] bucket "${S3_BUCKET}" exists`); } catch (err: any) { if (err.$metadata?.httpStatusCode === 404 || err.name === "NoSuchBucket" || err.name === "NotFound") { try { await client.send(new CreateBucketCommand({ Bucket: S3_BUCKET })); bucketReady = true; console.log(`[video-storage] bucket "${S3_BUCKET}" created`); } catch (createErr: any) { console.error("[video-storage] bucket create error:", createErr?.message ?? createErr); } } else { // treat unknown errors as "bucket exists, proceed" console.warn("[video-storage] HeadBucket warn (proceeding):", err?.message ?? err); bucketReady = true; } } } // ─── Helpers ────────────────────────────────────────────────────────────────── const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"; // ─── Upload ─────────────────────────────────────────────────────────────────── /** * Download a video from remoteUrl and upload it to S3. * * Returns the serving path `/api/videos/stored/.mp4`, or null on failure. */ export async function downloadAndStoreVideo( remoteUrl: string, bearerToken: string | null, ext = "mp4", ): Promise { if (!isStorageReady()) { console.warn("[video-storage] S3 not configured — skipping download"); return null; } await ensureBucket(); const id = randomUUID(); const key = `videos/${id}.${ext}`; let hostname = ""; try { hostname = new URL(remoteUrl).hostname; } catch { /* ok */ } // Cloudflare R2 pre-signed URLs are publicly accessible — no auth needed. const isR2 = hostname.endsWith(".r2.cloudflarestorage.com"); // assets.grok.com is behind Cloudflare; use grok.com as Referer+Origin. const isGrokCdn = hostname === "assets.grok.com"; const referer = isGrokCdn ? "https://grok.com/" : "https://geminigen.ai/"; const origin = isGrokCdn ? "https://grok.com" : "https://geminigen.ai"; const baseHeaders: Record = { "User-Agent": USER_AGENT, Accept: "video/mp4,video/*,*/*", ...(isR2 ? {} : { Referer: referer, Origin: origin }), }; // R2 pre-signed URLs work without any auth; try without token first. // For other CDNs, try with Bearer token first. const strategies: Array> = isR2 ? [{ ...baseHeaders }] : [ ...(bearerToken ? [{ ...baseHeaders, Authorization: `Bearer ${bearerToken}` }] : []), { ...baseHeaders }, ]; let videoBuffer: Buffer | null = null; let contentType = "video/mp4"; for (const headers of strategies) { try { const resp = await fetch(remoteUrl, { headers }); console.log( `[video-storage] fetch → ${resp.status} from ${new URL(remoteUrl).hostname}`, ); if (resp.ok) { const ab = await resp.arrayBuffer(); videoBuffer = Buffer.from(ab); const ct = resp.headers.get("content-type"); if (ct && ct.startsWith("video/")) contentType = ct; break; } } catch (err) { console.warn( "[video-storage] fetch error:", err instanceof Error ? err.message : err, ); } } if (!videoBuffer || videoBuffer.length < 1024) { console.warn("[video-storage] all download strategies failed"); return null; } try { const upload = new Upload({ client: getS3(), params: { Bucket: S3_BUCKET, Key: key, Body: videoBuffer, ContentType: contentType, }, }); await upload.done(); console.log(`[video-storage] uploaded ${videoBuffer.length} bytes → s3://${S3_BUCKET}/${key}`); return `/api/videos/stored/${id}.${ext}`; } catch (err) { console.error( "[video-storage] S3 upload error:", err instanceof Error ? err.message : err, ); return null; } } // ─── Stream / serve ─────────────────────────────────────────────────────────── /** * Serve a stored video from S3 to an Express response. * objectPath: the `.mp4` portion after /api/videos/stored/ */ export async function streamStoredVideo( objectPath: string, res: import("express").Response, rangeHeader?: string, ): Promise { if (!isStorageReady()) { res.status(503).json({ error: "Storage not configured" }); return; } const key = `videos/${objectPath}`; try { const client = getS3(); // HEAD to get size & content-type let size = 0; let contentType = "video/mp4"; try { const head = await client.send( new HeadObjectCommand({ Bucket: S3_BUCKET, Key: key }), ); size = Number(head.ContentLength ?? 0); contentType = head.ContentType ?? contentType; } catch { // fallback: stream without size } res.setHeader("Content-Type", contentType); res.setHeader("Accept-Ranges", "bytes"); res.setHeader("Cache-Control", "public, max-age=86400"); let byteRange: { start: number; end: number } | undefined; if (rangeHeader && size) { const m = /bytes=(\d*)-(\d*)/.exec(rangeHeader); if (m) { const start = m[1] ? parseInt(m[1]) : 0; const end = m[2] ? parseInt(m[2]) : size - 1; byteRange = { start, end }; res.setHeader("Content-Range", `bytes ${start}-${end}/${size}`); res.setHeader("Content-Length", end - start + 1); res.status(206); } } else if (size) { res.setHeader("Content-Length", size); res.status(200); } else { res.status(200); } const getCmd = new GetObjectCommand({ Bucket: S3_BUCKET, Key: key, ...(byteRange ? { Range: `bytes=${byteRange.start}-${byteRange.end}` } : {}), }); const obj = await client.send(getCmd); if (!obj.Body) { throw new Error("Empty body from S3"); } // obj.Body is a SdkStreamMixin — convert to Node.js Readable const stream = obj.Body as Readable; stream.pipe(res); stream.on("error", (err) => { console.error("[video-storage] stream pipe error:", err.message); if (!res.headersSent) res.status(500).end(); }); } catch (err) { console.error( "[video-storage] stream error:", err instanceof Error ? err.message : err, ); if (!res.headersSent) res.status(404).json({ error: "Video not found in storage" }); } }