Fix: split remote script to avoid ARG_MAX / ENAMETOOLONG on bash -lc

training/hf_remote_run.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# Executed inside the HF Job container after the repo is cloned to /work.
+# Called by hf_run_space_train_job.sh bootstrap command.
+set -euo pipefail
+
+cd /work
+
+papermill --log-output --progress-bar --execution-timeout "${NB_EXEC_TIMEOUT}" \
+  training/train_grpo.ipynb training/train_grpo.executed.ipynb
+
+STAGE="/work/_run_upload"
+rm -rf "$STAGE" && mkdir -p "$STAGE/training"
+cp -a training/train_grpo.executed.ipynb "$STAGE/training/"
+[ -d plots ] && cp -a plots "$STAGE/"
+[ -d viraltest_trained_adapter ] && cp -a viraltest_trained_adapter "$STAGE/"
+if [ "${INCLUDE_CHECKPOINTS:-0}" = "1" ] && [ -d checkpoints ]; then
+  cp -a checkpoints "$STAGE/"
+fi
+
+TARBALL="/tmp/r.tgz"
+( cd "$STAGE" && tar -czf "$TARBALL" . )
+
+export HOME=/tmp HF_HOME=/tmp/hf HUGGINGFACE_HUB_CACHE=/tmp/hfc HF_HUB_CACHE=/tmp/hfc TMPDIR=/tmp XDG_CACHE_HOME=/tmp
+mkdir -p "$HF_HOME" "$HF_HUB_CACHE"
+
+python3 -c "
+import os
+os.environ.setdefault('HOME', '/tmp')
+os.environ.setdefault('HF_HOME', '/tmp/hf')
+os.environ.setdefault('HUGGINGFACE_HUB_CACHE', '/tmp/hfc')
+os.environ.setdefault('HF_HUB_CACHE', '/tmp/hfc')
+os.environ.setdefault('TMPDIR', '/tmp')
+os.environ.setdefault('XDG_CACHE_HOME', '/tmp')
+from huggingface_hub import HfApi
+with open('/tmp/r.tgz', 'rb') as f:
+    HfApi(token=os.environ.get('HF_TOKEN')).upload_file(
+        path_or_fileobj=f,
+        path_in_repo='run-output/artifacts.tar.gz',
+        repo_id=os.environ['SPACE_REPO'],
+        repo_type='space',
+    )
+print('Artifacts uploaded.')
+"

training/hf_run_space_train_job.sh

@@ -32,68 +32,7 @@ if ! hf auth list &>/dev/null; then
   exit 1
 fi
 
-REMOTE_SCRIPT=$(cat <<'EOS'
-set -euo pipefail
-export DEBIAN_FRONTEND=noninteractive
-apt-get update -qq && apt-get install -y --no-install-recommends git curl ca-certificates
-pip install -q --root-user-action=ignore --upgrade "typing_extensions>=4.15.0" jupyter nbconvert nbclient ipykernel huggingface_hub papermill
-rm -rf /work
-git clone --depth 1 "https://user:${HF_TOKEN}@huggingface.co/spaces/${SPACE_REPO}" /work
-cd /work
-papermill --log-output --progress-bar --execution-timeout "${NB_EXEC_TIMEOUT}" \
-  training/train_grpo.ipynb training/train_grpo.executed.ipynb
-
-# Artifacts: keep paths short; tar once, upload one file. Large uploads: huggingface_hub
-# uses Xet (hf_xet) when the file is a path + Xet is installed, which can hit ENAMETOOLONG
-# in ~/.cache/.../xet on some job images. Open the tarball in binary mode and pass the
-# file object so the client does not offer Xet and falls back to LFS/HTTP
-# (see huggingface_hub _commit_api._upload_preupload_* / transfers list).
-# LoRA: viraltest_trained_adapter. Checkpoints: optional INCLUDE_CHECKPOINTS=1.
-STAGE="/work/_run_upload"
-rm -rf "$STAGE" && mkdir -p "$STAGE/training"
-cp -a training/train_grpo.executed.ipynb "$STAGE/training/"
-[ -d plots ] && cp -a plots "$STAGE/"
-[ -d viraltest_trained_adapter ] && cp -a viraltest_trained_adapter "$STAGE/"
-if [ "${INCLUDE_CHECKPOINTS:-0}" = "1" ] && [ -d checkpoints ]; then
-  cp -a checkpoints "$STAGE/"
-fi
-
-TARBALL="/tmp/r.tgz"
-( cd "$STAGE" && tar -czf "$TARBALL" . )
-
-# Short cache/home so anything still touching ~/.cache stays under /tmp
-export HOME=/tmp
-export HF_HOME=/tmp/hf
-export HUGGINGFACE_HUB_CACHE=/tmp/hfc
-export HF_HUB_CACHE=/tmp/hfc
-export TMPDIR=/tmp
-export XDG_CACHE_HOME=/tmp
-mkdir -p "$HF_HOME" "$HF_HUB_CACHE"
-
-python <<'PY'
-import os
-
-# Before importing huggingface_hub (constants read cache paths at import)
-os.environ.setdefault("HOME", "/tmp")
-os.environ.setdefault("HF_HOME", "/tmp/hf")
-os.environ.setdefault("HUGGINGFACE_HUB_CACHE", "/tmp/hfc")
-os.environ.setdefault("HF_HUB_CACHE", "/tmp/hfc")
-os.environ.setdefault("TMPDIR", "/tmp")
-os.environ.setdefault("XDG_CACHE_HOME", "/tmp")
-
-from huggingface_hub import HfApi
-
-# Buffered file object: hub omits "xet" from LFS transfer offers (avoids xet chunk-cache path blowups)
-with open("/tmp/r.tgz", "rb") as f:
-    HfApi(token=os.environ.get("HF_TOKEN")).upload_file(
-        path_or_fileobj=f,
-        path_in_repo="run-output/artifacts.tar.gz",
-        repo_id=os.environ["SPACE_REPO"],
-        repo_type="space",
-    )
-PY
-EOS
-)
+BOOTSTRAP='set -euo pipefail; export DEBIAN_FRONTEND=noninteractive; apt-get update -qq && apt-get install -y --no-install-recommends git curl ca-certificates; pip install -q --root-user-action=ignore --upgrade "typing_extensions>=4.15.0" jupyter nbconvert nbclient ipykernel huggingface_hub papermill; rm -rf /work; git clone --depth 1 "https://user:${HF_TOKEN}@huggingface.co/spaces/${SPACE_REPO}" /work; cd /work; bash training/hf_remote_run.sh'
 
 exec hf jobs run \
   --namespace "$NAMESPACE" \
@@ -105,4 +44,4 @@ exec hf jobs run \
   --env "NB_EXEC_TIMEOUT=$NB_EXEC_TIMEOUT" \
   --env "INCLUDE_CHECKPOINTS=$INCLUDE_CHECKPOINTS" \
   "$IMAGE" \
-  bash -lc "$REMOTE_SCRIPT"
+  bash -lc "$BOOTSTRAP"