training: split remote runner into in-repo script

- Add training/hf_remote_run.sh executed inside the HF Job container.
- Shrink training/hf_run_space_train_job.sh to a minimal bootstrap
(clone Space + exec in-repo script) to avoid argv "File name too long".
- Default flavor a100x4 (4xA100, 48 vCPU, 568 GB RAM, 320 GB VRAM).

Made-with: Cursor

training/hf_remote_run.sh

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+# Remote runner executed inside the HF Job container.
+# The launcher (training/hf_run_space_train_job.sh) clones the Space into /work
+# and then exec's this script. Keeping the heavy lifting in a file (instead of
+# inlining via `bash -lc`) avoids the argv "File name too long" failure.
+#
+# Required env (set by launcher):
+#   SPACE_REPO          e.g. ycwhencpp/train-new
+#   HF_TOKEN            secret, injected via --secrets
+#   NB_EXEC_TIMEOUT     papermill execution timeout in seconds (default 14400)
+#   SMOKE_MODE          0 = full training, 1 = smoke (default 0 here)
+
+set -euo pipefail
+
+: "${SPACE_REPO:?SPACE_REPO must be set}"
+: "${HF_TOKEN:?HF_TOKEN must be set (use --secrets HF_TOKEN)}"
+
+NB_EXEC_TIMEOUT="${NB_EXEC_TIMEOUT:-14400}"
+export SMOKE_MODE="${SMOKE_MODE:-0}"
+export HF_HUB_ENABLE_HF_TRANSFER="${HF_HUB_ENABLE_HF_TRANSFER:-1}"
+export TOKENIZERS_PARALLELISM="${TOKENIZERS_PARALLELISM:-false}"
+export DEBIAN_FRONTEND=noninteractive
+
+echo "===== hf_remote_run.sh starting at $(date -u +%FT%TZ) ====="
+echo "SPACE_REPO=${SPACE_REPO}"
+echo "SMOKE_MODE=${SMOKE_MODE}"
+echo "NB_EXEC_TIMEOUT=${NB_EXEC_TIMEOUT}"
+
+nvidia-smi || true
+
+echo "----- pip installs -----"
+pip install -q --root-user-action=ignore --upgrade \
+  "typing_extensions>=4.15.0" \
+  jupyter nbconvert nbclient ipykernel \
+  huggingface_hub hf_transfer papermill
+
+echo "----- executing notebook with papermill -----"
+mkdir -p plots run-output checkpoints
+papermill --log-output --progress-bar \
+  --execution-timeout "${NB_EXEC_TIMEOUT}" \
+  training/train_grpo.ipynb \
+  training/train_grpo.executed.ipynb
+
+echo "----- uploading artifacts back to Space (run-output/) -----"
+python - <<'PY'
+import os
+from huggingface_hub import HfApi
+
+api = HfApi(token=os.environ["HF_TOKEN"])
+api.upload_folder(
+    folder_path=".",
+    path_in_repo="run-output",
+    repo_id=os.environ["SPACE_REPO"],
+    repo_type="space",
+    allow_patterns=[
+        "training/train_grpo.executed.ipynb",
+        "plots/**",
+        "checkpoints/**/adapter_*",
+        "checkpoints/**/lora-*/**",
+        "**/lora-*/**",
+    ],
+    commit_message="HF Job: train_grpo run output",
+)
+print("upload complete")
+PY
+
+echo "===== hf_remote_run.sh done at $(date -u +%FT%TZ) ====="

training/hf_run_space_train_job.sh

@@ -1,36 +1,37 @@
 #!/usr/bin/env bash
-# Same environment as your HF Job (Space clone + nbconvert + upload to Space).
-# Old UI command was invalid shell (no &&); this version is a proper chain.
+# Launch a HF Job that:
+#   1. clones the Space repo (default: ycwhencpp/train-new) into /work
+#   2. execs training/hf_remote_run.sh from that clone (heavy lifting lives there)
 #
-# Requires: hf auth login (token is sent via --secrets HF_TOKEN from the CLI cache)
-# Optional: HF_SPACE_REPO_ID (default vaibhavkhandare/train-bhai-train)
+# We keep the inline bootstrap intentionally tiny — anything larger risks the
+# "File name too long" failure when the whole command becomes argv to bash -lc.
+#
+# Requires: hf auth login (token forwarded via --secrets HF_TOKEN)
 
 set -euo pipefail
 
 IMAGE="${HF_JOB_IMAGE:-pytorch/pytorch:2.5.1-cuda12.4-cudnn9-runtime}"
 FLAVOR="${HF_JOB_FLAVOR:-a100x4}"
 TIMEOUT="${HF_JOB_TIMEOUT:-8h}"
-SPACE_REPO="${HF_SPACE_REPO_ID:-vaibhavkhandare/train-bhai-train}"
-NB_EXEC_TIMEOUT="${NB_EXEC_TIMEOUT:-3600}"
+SPACE_REPO="${HF_SPACE_REPO_ID:-ycwhencpp/train-new}"
+NB_EXEC_TIMEOUT="${NB_EXEC_TIMEOUT:-14400}"
+SMOKE_MODE="${SMOKE_MODE:-0}"
 
 if ! hf auth whoami &>/dev/null; then
   echo "Run: hf auth login" >&2
   exit 1
 fi
 
-REMOTE_SCRIPT=$(cat <<'EOS'
-set -euo pipefail
+# Tiny bootstrap: install git, clone the Space, hand off to the in-repo script.
+BOOTSTRAP='set -euo pipefail
 export DEBIAN_FRONTEND=noninteractive
-apt-get update -qq && apt-get install -y --no-install-recommends git curl ca-certificates
-pip install -q --root-user-action=ignore --upgrade "typing_extensions>=4.15.0" jupyter nbconvert nbclient ipykernel huggingface_hub papermill
+apt-get update -qq
+apt-get install -y --no-install-recommends git curl ca-certificates >/dev/null
 rm -rf /work
 git clone --depth 1 "https://user:${HF_TOKEN}@huggingface.co/spaces/${SPACE_REPO}" /work
 cd /work
-papermill --log-output --progress-bar --execution-timeout "${NB_EXEC_TIMEOUT}" \
-  training/train_grpo.ipynb training/train_grpo.executed.ipynb
-python -c "import os; from huggingface_hub import HfApi; HfApi().upload_folder(folder_path='.', path_in_repo='run-output', repo_id=os.environ['SPACE_REPO'], repo_type='space', allow_patterns=['training/train_grpo.executed.ipynb','plots/**','**/lora-*/**'])"
-EOS
-)
+chmod +x training/hf_remote_run.sh
+exec bash training/hf_remote_run.sh'
 
 exec hf jobs run \
   --flavor "$FLAVOR" \
@@ -39,5 +40,6 @@ exec hf jobs run \
   --secrets HF_TOKEN \
   --env "SPACE_REPO=$SPACE_REPO" \
   --env "NB_EXEC_TIMEOUT=$NB_EXEC_TIMEOUT" \
+  --env "SMOKE_MODE=$SMOKE_MODE" \
   "$IMAGE" \
-  bash -lc "$REMOTE_SCRIPT"
+  bash -lc "$BOOTSTRAP"