File size: 3,890 Bytes
6c999c3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 | #!/bin/bash
# ============================================================================
# LLaMA Factory training launch script for HYV3
#
# This script sets up the environment and launches training via torchrun.
#
# We use train_hy_v3.py as the entry point (not llamafactory-cli)
# because we need to inject HYV3-specific monkey-patches and register
# the hy_v3 chat template BEFORE LLaMA Factory starts.
# train_hy_v3.py directly calls run_exp() in each torchrun worker,
# ensuring all patches are active.
#
# Usage:
# Single node: bash train_lf.sh
# Multi-node: Run this script on EACH node with the same IP_LIST.
# IP_LIST="10.0.0.1,10.0.0.2" bash train_lf.sh
# ============================================================================
set -euo pipefail
# -------------------- Network Configuration --------------------
NET_TYPE="high"
export NCCL_DEBUG=WARN
export NCCL_P2P_LEVEL=NVL
export NCCL_IB_TIMEOUT=24
export NCCL_NVLS_ENABLE=0
export NCCL_MPI_PROFILE_PRIMS_ENABLE=0
export CUDA_DEVICE_MAX_CONNECTIONS=1
export TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC=3600
if [[ "${NET_TYPE}" = "low" ]]; then
export NCCL_SOCKET_IFNAME=eth1
export NCCL_IB_GID_INDEX=3
export NCCL_IB_HCA=mlx5_2:1
export NCCL_IB_SL=3
export NCCL_CHECK_DISABLE=1
export NCCL_P2P_DISABLE=0
export NCCL_LL_THRESHOLD=16384
export NCCL_IB_CUDA_SUPPORT=1
else
export NCCL_IB_GID_INDEX=3
export NCCL_IB_SL=3
export NCCL_CHECK_DISABLE=1
export NCCL_P2P_DISABLE=0
export NCCL_IB_DISABLE=0
export NCCL_LL_THRESHOLD=16384
export NCCL_IB_CUDA_SUPPORT=1
export NCCL_SOCKET_IFNAME=bond1
export UCX_NET_DEVICES=bond1
export NCCL_IB_HCA=mlx5_bond_1,mlx5_bond_5,mlx5_bond_3,mlx5_bond_7,mlx5_bond_4,mlx5_bond_8,mlx5_bond_2,mlx5_bond_6
export NCCL_COLLNET_ENABLE=0
export SHARP_COLL_ENABLE_SAT=0
export NCCL_NET_GDR_LEVEL=2
export NCCL_IB_QPS_PER_CONNECTION=4
export NCCL_IB_TC=160
export NCCL_PXN_DISABLE=1
fi
# Skip LLaMA Factory version check (we use a newer transformers branch)
export DISABLE_VERSION_CHECK=1
# -------------------- Node Configuration --------------------
export HOST_GPU_NUM=8
# IP list, comma separated. e.g. "10.0.0.1,10.0.0.2" or single node "127.0.0.1"
export IP_LIST=${IP_LIST:-"127.0.0.1"}
MASTER_PORT=${MASTER_PORT:-29500}
IFS=',' read -ra IP_ARRAY <<< "$IP_LIST"
NODES=${#IP_ARRAY[@]}
MASTER_ADDR=${IP_ARRAY[0]}
# -------------------- Paths --------------------
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
YAML_FILE="${SCRIPT_DIR}/hy_v3_full_sft.yaml"
ENTRY_SCRIPT="${SCRIPT_DIR}/train_hy_v3.py"
# -------------------- Distributed Environment --------------------
export MASTER_ADDR="${MASTER_ADDR}"
export MASTER_PORT="${MASTER_PORT}"
export NNODES="${NODES}"
if [ ${NODES} -gt 1 ]; then
# Determine local node rank by matching local IP against IP_LIST
LOCAL_IP=$(hostname -i | awk '{print $1}')
NODE_RANK=0
for i in "${!IP_ARRAY[@]}"; do
if [[ "${IP_ARRAY[$i]}" == "${LOCAL_IP}" ]]; then
NODE_RANK=$i
break
fi
done
export RANK="${NODE_RANK}"
else
export RANK=0
fi
echo "============================================"
echo " HYV3 LLaMA Factory Training"
echo " Nodes: ${NNODES}, Rank: ${RANK}"
echo " Master: ${MASTER_ADDR}:${MASTER_PORT}"
echo " GPUs per node: ${HOST_GPU_NUM}"
echo " Total GPUs: $((NODES * HOST_GPU_NUM))"
echo "============================================"
# -------------------- Launch --------------------
# We launch torchrun directly (instead of FORCE_TORCHRUN) so that each
# worker process runs train_hy_v3.py with all HYV3 patches applied.
torchrun \
--nnodes "${NNODES}" \
--node_rank "${RANK}" \
--nproc_per_node "${HOST_GPU_NUM}" \
--master_addr "${MASTER_ADDR}" \
--master_port "${MASTER_PORT}" \
"${ENTRY_SCRIPT}" "${YAML_FILE}"
|