Harden YouTube downloads on Hugging Face

Dockerfile

@@ -6,9 +6,9 @@ ENV PYTHONUNBUFFERED=1 \
 
 WORKDIR /app
 
-RUN apt-get update && \
-    apt-get install -y ffmpeg && \
-    rm -rf /var/lib/apt/lists/*
+RUN apt-get update && \
+    apt-get install -y ffmpeg nodejs ca-certificates && \
+    rm -rf /var/lib/apt/lists/*
 
 COPY . /app
 

README.md

@@ -31,9 +31,13 @@ Required Space secrets:
 
 - `GOOGLE_API_KEY` or `GEMINI_API_KEY`, or `OPENROUTER_API_KEY`
 - `OPENAI_API_KEY` or `ELEVENLABS_API_KEY`
+- Optional for YouTube links on cloud IPs: `YTDLP_COOKIES_B64`
 
 If `HUMEO_TRANSCRIBE_PROVIDER` is not set, the Space uses ElevenLabs when
 `ELEVENLABS_API_KEY` exists, otherwise OpenAI Whisper.
+If YouTube blocks Hugging Face with a bot/sign-in challenge, export a Netscape
+`cookies.txt` from a logged-in browser, base64 encode it, and set it as the
+`YTDLP_COOKIES_B64` Space secret. Local file upload does not need this.
 
 ## Repo layout
 

src/humeo/ingest.py

@@ -7,13 +7,14 @@ Responsibilities:
   - Generate word-level timestamped transcript.
 """
 
-import json
-import logging
-import os
-import shutil
-import subprocess
-from math import ceil
-from pathlib import Path
+import json
+import logging
+import os
+import shutil
+import subprocess
+import base64
+from math import ceil
+from pathlib import Path
 
 import httpx
 
@@ -27,9 +28,63 @@ OPENAI_MIN_CHUNK_SEC = 300.0
 ELEVENLABS_TRANSCRIBE_URL = "https://api.elevenlabs.io/v1/speech-to-text"
 TRANSCRIPT_META_FILENAME = "transcript.meta.json"
 ELEVENLABS_SCRIBE_MODEL = "scribe_v2"
-_ELEVENLABS_SEGMENT_MAX_GAP_SEC = 0.65
-_ELEVENLABS_SEGMENT_MAX_DURATION_SEC = 6.0
-_ELEVENLABS_SEGMENT_MAX_WORDS = 18
+_ELEVENLABS_SEGMENT_MAX_GAP_SEC = 0.65
+_ELEVENLABS_SEGMENT_MAX_DURATION_SEC = 6.0
+_ELEVENLABS_SEGMENT_MAX_WORDS = 18
+YTDLP_BROWSER_USER_AGENT = (
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
+)
+
+
+def _decode_cookie_secret(raw: str) -> str:
+    text = raw.strip()
+    if "\\n" in text and "\n" not in text:
+        text = text.replace("\\n", "\n")
+    return text
+
+
+def _yt_dlp_cookie_file(output_dir: Path) -> Path | None:
+    raw = (
+        os.environ.get("YTDLP_COOKIES")
+        or os.environ.get("YOUTUBE_COOKIES")
+        or ""
+    ).strip()
+    encoded = (
+        os.environ.get("YTDLP_COOKIES_B64")
+        or os.environ.get("YOUTUBE_COOKIES_B64")
+        or ""
+    ).strip()
+    if not raw and encoded:
+        try:
+            raw = base64.b64decode(encoded).decode("utf-8")
+        except Exception as exc:
+            raise RuntimeError("Could not decode YTDLP_COOKIES_B64.") from exc
+    if not raw:
+        return None
+
+    cookie_path = output_dir / "yt-dlp-cookies.txt"
+    cookie_path.write_text(_decode_cookie_secret(raw).rstrip() + "\n", encoding="utf-8")
+    try:
+        cookie_path.chmod(0o600)
+    except OSError:
+        pass
+    return cookie_path
+
+
+def _yt_dlp_error(exc: subprocess.CalledProcessError) -> RuntimeError:
+    stdout = (exc.stdout or "").strip()
+    stderr = (exc.stderr or "").strip()
+    details = stderr or stdout or str(exc)
+    lowered = details.lower()
+    hint = ""
+    if any(token in lowered for token in ("sign in", "not a bot", "confirm you're not a bot", "cookies")):
+        hint = (
+            "\n\nYouTube blocked the Hugging Face downloader. Add a Space secret named "
+            "YTDLP_COOKIES_B64 containing a base64 encoded Netscape cookies.txt export "
+            "from a logged-in browser, or upload the MP4 directly."
+        )
+    return RuntimeError(f"yt-dlp failed to download the YouTube video:\n{details}{hint}")
 
 
 def stage_local_video(source: str | Path, output_dir: Path) -> Path:
@@ -72,22 +127,46 @@ def download_video(youtube_url: str, output_dir: Path) -> Path:
 
     Returns the path to the downloaded MP4 file.
     """
-    output_template = str(output_dir / "source.%(ext)s")
-    cmd = [
-        "yt-dlp",
-        "--format", "bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best",
-        "--merge-output-format", "mp4",
-        "--output", output_template,
-        "--no-playlist",
-        "--write-info-json",
-        "--quiet",
-        youtube_url,
-    ]
-
-    logger.info("Downloading video: %s", youtube_url)
-    result = subprocess.run(cmd, check=True, capture_output=True, text=True)
-    if result.stderr:
-        logger.warning(result.stderr.strip())
+    output_template = str(output_dir / "source.%(ext)s")
+    cmd = [
+        "yt-dlp",
+        "--format",
+        "bv*[ext=mp4]+ba[ext=m4a]/bv*+ba/best[ext=mp4]/best",
+        "--merge-output-format",
+        "mp4",
+        "--output",
+        output_template,
+        "--no-playlist",
+        "--write-info-json",
+        "--retries",
+        "5",
+        "--fragment-retries",
+        "5",
+        "--extractor-retries",
+        "3",
+        "--socket-timeout",
+        "30",
+        "--force-ipv4",
+        "--user-agent",
+        YTDLP_BROWSER_USER_AGENT,
+        "--extractor-args",
+        (os.environ.get("YTDLP_EXTRACTOR_ARGS") or "youtube:player_client=default,web_creator"),
+        "--quiet",
+    ]
+    if shutil.which("node"):
+        cmd.extend(["--js-runtimes", "node", "--remote-components", "ejs:github"])
+    cookie_path = _yt_dlp_cookie_file(output_dir)
+    if cookie_path is not None:
+        cmd.extend(["--cookies", str(cookie_path)])
+    cmd.append(youtube_url)
+
+    logger.info("Downloading video: %s", youtube_url)
+    try:
+        result = subprocess.run(cmd, check=True, capture_output=True, text=True)
+    except subprocess.CalledProcessError as exc:
+        raise _yt_dlp_error(exc) from exc
+    if result.stderr:
+        logger.warning(result.stderr.strip())
 
     # yt-dlp should produce source.mp4
     video_path = output_dir / "source.mp4"