artifacts/api-server/src/lib/videoStorage.ts

/**
 * videoStorage.ts
 *
 * Downloads generated videos from Grok CDN and stores them in an
 * S3-compatible object storage (hi168.com OSS).
 *
 * Upload: @aws-sdk/lib-storage  (multipart, handles large files)
 * Download / stream: @aws-sdk/client-s3 (GetObject)
 */
import {
  S3Client,
  CreateBucketCommand,
  HeadBucketCommand,
  GetObjectCommand,
  HeadObjectCommand,
} from "@aws-sdk/client-s3";
import { Upload } from "@aws-sdk/lib-storage";
import { Readable } from "stream";
import { randomUUID } from "crypto";

// ─── S3 configuration ────────────────────────────────────────────────────────

const S3_ENDPOINT = process.env.S3_ENDPOINT ?? "";
const S3_BUCKET   = process.env.S3_BUCKET   ?? "starforge-videos";
const S3_REGION   = process.env.S3_REGION   ?? "us-east-1";
const ACCESS_KEY  = process.env.S3_ACCESS_KEY ?? "";
const SECRET_KEY  = process.env.S3_SECRET_KEY ?? "";

let s3: S3Client | null = null;

function getS3(): S3Client {
  if (!s3) {
    s3 = new S3Client({
      endpoint: S3_ENDPOINT,
      region: S3_REGION,
      credentials: { accessKeyId: ACCESS_KEY, secretAccessKey: SECRET_KEY },
      forcePathStyle: true,   // required for most non-AWS S3 endpoints
    });
  }
  return s3;
}

/** Returns true if S3 credentials are configured. */
export function isStorageReady(): boolean {
  return !!(S3_ENDPOINT && ACCESS_KEY && SECRET_KEY);
}

// ─── Bucket init (create if missing) ─────────────────────────────────────────

let bucketReady = false;

async function ensureBucket(): Promise<void> {
  if (bucketReady) return;
  const client = getS3();
  try {
    await client.send(new HeadBucketCommand({ Bucket: S3_BUCKET }));
    bucketReady = true;
    console.log(`[video-storage] bucket "${S3_BUCKET}" exists`);
  } catch (err: any) {
    if (err.$metadata?.httpStatusCode === 404 || err.name === "NoSuchBucket" || err.name === "NotFound") {
      try {
        await client.send(new CreateBucketCommand({ Bucket: S3_BUCKET }));
        bucketReady = true;
        console.log(`[video-storage] bucket "${S3_BUCKET}" created`);
      } catch (createErr: any) {
        console.error("[video-storage] bucket create error:", createErr?.message ?? createErr);
      }
    } else {
      // treat unknown errors as "bucket exists, proceed"
      console.warn("[video-storage] HeadBucket warn (proceeding):", err?.message ?? err);
      bucketReady = true;
    }
  }
}

// ─── Helpers ──────────────────────────────────────────────────────────────────

const USER_AGENT =
  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " +
  "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";

// ─── Upload ───────────────────────────────────────────────────────────────────

/**
 * Download a video from remoteUrl and upload it to S3.
 *
 * Returns the serving path `/api/videos/stored/<id>.mp4`, or null on failure.
 */
export async function downloadAndStoreVideo(
  remoteUrl: string,
  bearerToken: string | null,
  ext = "mp4",
): Promise<string | null> {
  if (!isStorageReady()) {
    console.warn("[video-storage] S3 not configured — skipping download");
    return null;
  }

  await ensureBucket();

  const id = randomUUID();
  const key = `videos/${id}.${ext}`;

  let hostname = "";
  try { hostname = new URL(remoteUrl).hostname; } catch { /* ok */ }

  // Cloudflare R2 pre-signed URLs are publicly accessible — no auth needed.
  const isR2 = hostname.endsWith(".r2.cloudflarestorage.com");
  // assets.grok.com is behind Cloudflare; use grok.com as Referer+Origin.
  const isGrokCdn = hostname === "assets.grok.com";
  const referer = isGrokCdn ? "https://grok.com/" : "https://geminigen.ai/";
  const origin  = isGrokCdn ? "https://grok.com"  : "https://geminigen.ai";

  const baseHeaders: Record<string, string> = {
    "User-Agent": USER_AGENT,
    Accept: "video/mp4,video/*,*/*",
    ...(isR2 ? {} : { Referer: referer, Origin: origin }),
  };

  // R2 pre-signed URLs work without any auth; try without token first.
  // For other CDNs, try with Bearer token first.
  const strategies: Array<Record<string, string>> = isR2
    ? [{ ...baseHeaders }]
    : [
        ...(bearerToken ? [{ ...baseHeaders, Authorization: `Bearer ${bearerToken}` }] : []),
        { ...baseHeaders },
      ];

  let videoBuffer: Buffer | null = null;
  let contentType = "video/mp4";

  for (const headers of strategies) {
    try {
      const resp = await fetch(remoteUrl, { headers });
      console.log(
        `[video-storage] fetch → ${resp.status} from ${new URL(remoteUrl).hostname}`,
      );
      if (resp.ok) {
        const ab = await resp.arrayBuffer();
        videoBuffer = Buffer.from(ab);
        const ct = resp.headers.get("content-type");
        if (ct && ct.startsWith("video/")) contentType = ct;
        break;
      }
    } catch (err) {
      console.warn(
        "[video-storage] fetch error:",
        err instanceof Error ? err.message : err,
      );
    }
  }

  if (!videoBuffer || videoBuffer.length < 1024) {
    console.warn("[video-storage] all download strategies failed");
    return null;
  }

  try {
    const upload = new Upload({
      client: getS3(),
      params: {
        Bucket: S3_BUCKET,
        Key: key,
        Body: videoBuffer,
        ContentType: contentType,
      },
    });
    await upload.done();
    console.log(`[video-storage] uploaded ${videoBuffer.length} bytes → s3://${S3_BUCKET}/${key}`);
    return `/api/videos/stored/${id}.${ext}`;
  } catch (err) {
    console.error(
      "[video-storage] S3 upload error:",
      err instanceof Error ? err.message : err,
    );
    return null;
  }
}

// ─── Stream / serve ───────────────────────────────────────────────────────────

/**
 * Serve a stored video from S3 to an Express response.
 * objectPath: the `<id>.mp4` portion after /api/videos/stored/
 */
export async function streamStoredVideo(
  objectPath: string,
  res: import("express").Response,
  rangeHeader?: string,
): Promise<void> {
  if (!isStorageReady()) {
    res.status(503).json({ error: "Storage not configured" });
    return;
  }

  const key = `videos/${objectPath}`;

  try {
    const client = getS3();

    // HEAD to get size & content-type
    let size = 0;
    let contentType = "video/mp4";
    try {
      const head = await client.send(
        new HeadObjectCommand({ Bucket: S3_BUCKET, Key: key }),
      );
      size = Number(head.ContentLength ?? 0);
      contentType = head.ContentType ?? contentType;
    } catch {
      // fallback: stream without size
    }

    res.setHeader("Content-Type", contentType);
    res.setHeader("Accept-Ranges", "bytes");
    res.setHeader("Cache-Control", "public, max-age=86400");

    let byteRange: { start: number; end: number } | undefined;

    if (rangeHeader && size) {
      const m = /bytes=(\d*)-(\d*)/.exec(rangeHeader);
      if (m) {
        const start = m[1] ? parseInt(m[1]) : 0;
        const end   = m[2] ? parseInt(m[2]) : size - 1;
        byteRange = { start, end };
        res.setHeader("Content-Range", `bytes ${start}-${end}/${size}`);
        res.setHeader("Content-Length", end - start + 1);
        res.status(206);
      }
    } else if (size) {
      res.setHeader("Content-Length", size);
      res.status(200);
    } else {
      res.status(200);
    }

    const getCmd = new GetObjectCommand({
      Bucket: S3_BUCKET,
      Key: key,
      ...(byteRange
        ? { Range: `bytes=${byteRange.start}-${byteRange.end}` }
        : {}),
    });

    const obj = await client.send(getCmd);
    if (!obj.Body) {
      throw new Error("Empty body from S3");
    }

    // obj.Body is a SdkStreamMixin — convert to Node.js Readable
    const stream = obj.Body as Readable;
    stream.pipe(res);
    stream.on("error", (err) => {
      console.error("[video-storage] stream pipe error:", err.message);
      if (!res.headersSent) res.status(500).end();
    });
  } catch (err) {
    console.error(
      "[video-storage] stream error:",
      err instanceof Error ? err.message : err,
    );
    if (!res.headersSent) res.status(404).json({ error: "Video not found in storage" });
  }
}