#!/bin/bash
# Create folder structure for ALL rows across Tables 1, 3, 4, 5, 7 and
# freeze the current experiments/ code into each one. After this you can
# cd into any
// and run ./run.sh to submit 5 SLURM seeds.
#
# Re-running this script is safe: it will re-freeze the code (overwrite the
# snapshot), but won't clobber any existing seeds/ outputs.
set -euo pipefail
BASEDIR=${BASEDIR:-${PULSE_ROOT}}
EXP=${BASEDIR}/experiments
SETUP="${EXP}/setup_row.sh"
COMMON="--epochs 40 --batch_size 32 --lr 3e-4 --weight_decay 1e-4 \
--patience 12 --label_smoothing 0.05 --use_class_weights \
--num_workers 2"
ALL5="imu,emg,eyetrack,mocap,pressure"
row () {
# $1=table $2=row $3=desc $4=cli
bash "${SETUP}" --table "$1" --row "$2" --desc "$3" --cli "$4 ${COMMON}"
}
# ============================================================
# Table 1: Main comparison at T_fut=2s
# ============================================================
T1=table1_main_comparison
cat > "${BASEDIR}/${T1}/README.md" <<'EOF'
# Table 1: Main Comparison (Next-Action Prediction, T_fut = 2 s)
Each baseline is run on its most favourable modality subset; our model
(DailyActFormer) uses all 5 synchronised modalities. 5 seeds per row;
report mean ± std of Verb fine Top-1/5, Noun Top-1/5, Hand Top-1, Action
Top-1 (= verb ∧ noun ∧ hand). Action Top-1 is the headline metric.
| Row | Method | Family | Modalities |
|-----|-------------------|-----------------|---------------------|
| 01 | DailyActFormer | cross-modal Trf | imu+emg+eye+mocap+P |
| 02 | DeepConvLSTM | CNN+LSTM (IMU) | imu |
| 03 | DeepConvLSTM 3mod | CNN+LSTM | imu+mocap+emg |
| 04 | RULSTM | rolling LSTM | imu+mocap |
| 05 | FUTR | long-term Trf | mocap+imu+emg |
| 06 | AFFT | multimodal Trf | imu+emg+eye+mocap |
| 07 | HandFormer | hand-pose Trf | mocap (fingers) |
| 08 | ActionLLM (LoRA) | LLM-based | imu+emg+eye |
EOF
mkdir -p "${BASEDIR}/${T1}"
row ${T1} row01_ours_dailyactformer_all5 \
"Our model, all 5 modalities (headline row)" \
"--model dailyactformer --modalities ${ALL5} --t_obs 8 --t_fut 2"
row ${T1} row02_deepconvlstm_imu \
"DeepConvLSTM on IMU only (classic HAR baseline)" \
"--model deepconvlstm --modalities imu --t_obs 8 --t_fut 2"
row ${T1} row03_deepconvlstm_3mod \
"DeepConvLSTM on IMU+MoCap+EMG (best 3-modality concat)" \
"--model deepconvlstm --modalities imu,mocap,emg --t_obs 8 --t_fut 2"
row ${T1} row04_rulstm_imu_mocap \
"RULSTM, rolling-unrolling LSTM (IMU + MoCap late fusion)" \
"--model rulstm --modalities imu,mocap --t_obs 8 --t_fut 2"
row ${T1} row05_futr_3mod \
"FUTR (causal transformer) on MoCap+IMU+EMG" \
"--model futr --modalities mocap,imu,emg --t_obs 8 --t_fut 2"
row ${T1} row06_afft_4mod \
"AFFT (anticipative feature fusion transformer) on 4 modalities" \
"--model afft --modalities imu,emg,eyetrack,mocap --t_obs 8 --t_fut 2"
row ${T1} row07_handformer_mocap \
"HandFormer (skeleton-only ECCV'24) on MoCap finger joints" \
"--model handformer --modalities mocap --t_obs 8 --t_fut 2"
row ${T1} row08_actionllm_3mod \
"ActionLLM (Qwen2.5-0.5B + LoRA) on IMU+EMG+EyeTrack" \
"--model actionllm --modalities imu,emg,eyetrack --t_obs 8 --t_fut 2"
# ============================================================
# Table 3: Horizon curve (DailyActFormer)
# ============================================================
T3=table3_horizon_curve
mkdir -p "${BASEDIR}/${T3}"
cat > "${BASEDIR}/${T3}/README.md" <<'EOF'
# Table 3: Prediction Horizon Curve (DailyActFormer, all 5 modalities)
Same model, varying T_fut. Expect monotonic drop in Action Top-1 as
horizon grows; plot line graph in the paper alongside this table.
EOF
HORIZONS=(1 2 5 10 15)
for i in "${!HORIZONS[@]}"; do
tfut="${HORIZONS[$i]}"
idx=$(printf "%02d" $((i+1)))
row ${T3} row${idx}_ours_tfut${tfut}s \
"Our model at T_fut=${tfut}s" \
"--model dailyactformer --modalities ${ALL5} --t_obs 8 --t_fut ${tfut}"
done
# ============================================================
# Table 4: Modality ablation on DailyActFormer (T_fut=2s)
# ============================================================
T4=table4_modality_ablation
mkdir -p "${BASEDIR}/${T4}"
cat > "${BASEDIR}/${T4}/README.md" <<'EOF'
# Table 4: Modality Ablation (DailyActFormer, T_fut = 2 s)
Same model, progressively remove modalities. Each row trained from scratch.
EOF
row ${T4} row01_full_5mod "Full 5-modality (reference)" "--model dailyactformer --modalities imu,emg,eyetrack,mocap,pressure --t_obs 8 --t_fut 2"
row ${T4} row02_no_pressure "Drop pressure" "--model dailyactformer --modalities imu,emg,eyetrack,mocap --t_obs 8 --t_fut 2"
row ${T4} row03_no_eyetrack "Drop eye-tracking" "--model dailyactformer --modalities imu,emg,mocap,pressure --t_obs 8 --t_fut 2"
row ${T4} row04_no_emg "Drop EMG" "--model dailyactformer --modalities imu,eyetrack,mocap,pressure --t_obs 8 --t_fut 2"
row ${T4} row05_no_imu "Drop IMU" "--model dailyactformer --modalities emg,eyetrack,mocap,pressure --t_obs 8 --t_fut 2"
row ${T4} row06_no_mocap "Drop MoCap" "--model dailyactformer --modalities imu,emg,eyetrack,pressure --t_obs 8 --t_fut 2"
row ${T4} row07_imu_emg_only "Only IMU + EMG (physiology-light)" "--model dailyactformer --modalities imu,emg --t_obs 8 --t_fut 2"
row ${T4} row08_mocap_only "Only MoCap (skeleton-only)" "--model dailyactformer --modalities mocap --t_obs 8 --t_fut 2"
# ============================================================
# Table 5: Component ablation (DailyActFormer switches)
# ============================================================
T5=table5_component_ablation
mkdir -p "${BASEDIR}/${T5}"
cat > "${BASEDIR}/${T5}/README.md" <<'EOF'
# Table 5: Component Ablation (DailyActFormer, T_fut = 2 s)
Each row toggles one architectural/training component of our model.
Component flags are implemented as CLI switches on train_seqpred.py;
see models_seqpred.py for the corresponding model options.
EOF
row ${T5} row01_full \
"Full model (reference)" \
"--model dailyactformer --modalities ${ALL5} --t_obs 8 --t_fut 2"
row ${T5} row02_no_composite_head \
"Drop the auxiliary verb-composite head (lambda=0)" \
"--model dailyactformer --modalities ${ALL5} --t_obs 8 --t_fut 2 --lambda_verb_composite 0.0"
row ${T5} row03_equal_lambda \
"Equal-weight all 4 heads (no prior on verb>hand)" \
"--model dailyactformer --modalities ${ALL5} --t_obs 8 --t_fut 2 --lambda_verb_composite 1.0 --lambda_hand 1.0"
row ${T5} row04_no_class_weight \
"No inverse-frequency class weighting" \
"--model dailyactformer --modalities ${ALL5} --t_obs 8 --t_fut 2 --lambda_verb_composite 0.5"
# row04 re-exposes the default; the variable-off is the absence of --use_class_weights
# We patch this manually — strip the flag out of COMMON.
ROW_DIR="${BASEDIR}/${T5}/row04_no_class_weight/run.sh"
if [[ -e "${ROW_DIR}" ]]; then
sed -i 's/--use_class_weights //g' "${ROW_DIR}"
fi
row ${T5} row05_no_label_smoothing \
"Label smoothing off" \
"--model dailyactformer --modalities ${ALL5} --t_obs 8 --t_fut 2 --label_smoothing 0.0"
# ============================================================
# Table 7: Missing-modality robustness (train once, eval 6 ways)
# ============================================================
T7=table7_missing_modality
mkdir -p "${BASEDIR}/${T7}"
cat > "${BASEDIR}/${T7}/README.md" <<'EOF'
# Table 7: Missing-Modality Robustness (T_fut = 2 s)
Train DailyActFormer with random per-modality dropout (p=0.3). At test time,
evaluate under 6 configurations: full / drop one modality each. Only the
training job has its own folder; eval uses the trained checkpoint to fill
multiple rows of the final table.
EOF
row ${T7} row01_train_with_modality_dropout \
"DailyActFormer trained with --modality_dropout 0.3" \
"--model dailyactformer --modalities ${ALL5} --t_obs 8 --t_fut 2 --modality_dropout 0.3"
# The 6 test-time configurations (full / no_P / no_E / no_emg / no_imu /
# no_mocap) will be produced by a separate eval script that loads the
# checkpoint from row01 and runs evaluate() with modality subsets. See
# experiments/tasks/eval_missing_modality.py (TBD).
echo ""
echo "[ok] Froze rows under:"
echo " ${BASEDIR}/{${T1},${T3},${T4},${T5},${T7}}/"