runtime error
Exit code: 1. Reason: ollowing exception: Traceback (most recent call last): File "/app/training/grpo_train.py", line 285, in run_grpo from trl import GRPOConfig, GRPOTrainer File "<frozen importlib._bootstrap>", line 1075, in _handle_fromlist File "/usr/local/lib/python3.10/dist-packages/trl/_lazy_module.py", line 62, in __getattr__ value = getattr(module, name) File "/usr/local/lib/python3.10/dist-packages/trl/_lazy_module.py", line 61, in __getattr__ module = self._get_module(self._class_to_module[name]) File "/usr/local/lib/python3.10/dist-packages/trl/_lazy_module.py", line 73, in _get_module raise RuntimeError( RuntimeError: Failed to import trl.trainer.grpo_trainer because of the following error (look up to see its traceback): Failed to import trl.models.utils because of the following error (look up to see its traceback): cannot import name 'FSDPModule' from 'torch.distributed.fsdp' (/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/__init__.py) The above exception was the direct cause of the following exception: Traceback (most recent call last): File "/usr/lib/python3.10/runpy.py", line 196, in _run_module_as_main return _run_code(code, main_globals, None, File "/usr/lib/python3.10/runpy.py", line 86, in _run_code exec(code, run_globals) File "/app/training/grpo_train.py", line 439, in <module> asyncio.run(_main()) File "/usr/lib/python3.10/asyncio/runners.py", line 44, in run return loop.run_until_complete(main) File "/usr/lib/python3.10/asyncio/base_events.py", line 649, in run_until_complete return future.result() File "/app/training/grpo_train.py", line 435, in _main run_grpo(args) File "/app/training/grpo_train.py", line 287, in run_grpo raise RuntimeError( RuntimeError: Failed to initialize TRL GRPO stack. On this machine, this is usually due to Windows blocking pyarrow dataset binaries in the local virtualenv. Use the provided Colab notebook (`training/colab_train.ipynb`) for GRPO runs, or fix local pyarrow/datasets installation first.
Container logs:
Fetching error logs...