run_llm_answers_all.sh

#!/usr/bin/env bash
# Run llm_answer_generator.py across dataset folders and tasks
# Processes both MCQ and open_text CSVs for tasks: count, duration, order, volume

set -euo pipefail
export CUDA_VISIBLE_DEVICES=7
PY_SCRIPT="$(dirname "$0")/llm_answer_generator.py"
BASE_DIR="$(dirname "$0")"

DATA_SPLITS=(train validation test_large test_ood)
TASKS=(count duration order volume)

echo "Running LLM answer generation script across splits: ${DATA_SPLITS[*]} and tasks: ${TASKS[*]}"

for split in "${DATA_SPLITS[@]}"; do
  for task in "${TASKS[@]}"; do
    # open_text file
    ot_csv="${BASE_DIR}/dataset_v2/${split}/${task}/${task}_open_text.csv"
    if [ -f "${ot_csv}" ]; then
      echo "[OPEN_TEXT] Processing ${ot_csv}"
      python "${PY_SCRIPT}" --input "${ot_csv}" --mode open_text --task "${task}"
    else
      echo "[OPEN_TEXT] Not found: ${ot_csv}"
    fi
  done
done

echo "All tasks processed."