Upload folder using huggingface_hub
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitattributes +6 -0
- dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/argv.txt +37 -0
- dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000/ema_net.pth +3 -0
- dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000/model.safetensors +3 -0
- dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000/optimizer.bin +3 -0
- dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000/random_states_0.pkl +3 -0
- dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000/random_states_1.pkl +3 -0
- dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000/random_states_2.pkl +3 -0
- dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000/random_states_3.pkl +3 -0
- dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000/scheduler.bin +3 -0
- dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/envs.txt +17 -0
- dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/run_config.json +300 -0
- dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/debug-internal.log +0 -0
- dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/debug.log +25 -0
- dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142630-q4pxxlgy/files/config.yaml +436 -0
- dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142630-q4pxxlgy/files/output.log +224 -0
- dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142630-q4pxxlgy/files/requirements.txt +205 -0
- dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142630-q4pxxlgy/files/wandb-metadata.json +129 -0
- dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142630-q4pxxlgy/files/wandb-summary.json +1 -0
- dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142630-q4pxxlgy/logs/debug-internal.log +0 -0
- dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142630-q4pxxlgy/logs/debug.log +25 -0
- dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142630-q4pxxlgy/run-q4pxxlgy.wandb +3 -0
- dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/argv.txt +37 -0
- dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/checkpoints/ckpt_40000/ema_net.pth +3 -0
- dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/checkpoints/ckpt_40000/model.safetensors +3 -0
- dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/checkpoints/ckpt_40000/optimizer.bin +3 -0
- dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/checkpoints/ckpt_40000/random_states_0.pkl +3 -0
- dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/checkpoints/ckpt_40000/random_states_1.pkl +3 -0
- dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/checkpoints/ckpt_40000/random_states_2.pkl +3 -0
- dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/checkpoints/ckpt_40000/random_states_3.pkl +3 -0
- dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/checkpoints/ckpt_40000/scheduler.bin +3 -0
- dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/envs.txt +17 -0
- dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/run_config.json +300 -0
- dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/wandb/debug-internal.log +0 -0
- dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/wandb/debug.log +25 -0
- dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/wandb/run-20260410_020501-sjem5sh4/files/config.yaml +436 -0
- dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/wandb/run-20260410_020501-sjem5sh4/files/output.log +230 -0
- dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/wandb/run-20260410_020501-sjem5sh4/files/requirements.txt +205 -0
- dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/wandb/run-20260410_020501-sjem5sh4/files/wandb-metadata.json +129 -0
- dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/wandb/run-20260410_020501-sjem5sh4/files/wandb-summary.json +1 -0
- dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/wandb/run-20260410_020501-sjem5sh4/logs/debug-core.log +19 -0
- dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/wandb/run-20260410_020501-sjem5sh4/logs/debug-internal.log +0 -0
- dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/wandb/run-20260410_020501-sjem5sh4/logs/debug.log +25 -0
- dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/wandb/run-20260410_020501-sjem5sh4/run-sjem5sh4.wandb +3 -0
- dp/diffusion-policy-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/argv.txt +37 -0
- dp/diffusion-policy-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_40000/ema_net.pth +3 -0
- dp/diffusion-policy-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_40000/model.safetensors +3 -0
- dp/diffusion-policy-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_40000/optimizer.bin +3 -0
- dp/diffusion-policy-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_40000/random_states_0.pkl +3 -0
- dp/diffusion-policy-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_40000/random_states_1.pkl +3 -0
.gitattributes
CHANGED
|
@@ -2106,3 +2106,9 @@ act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-
|
|
| 2106 |
act/act-g1-sim/g1wholebodytabletopgrasp-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603190700/wandb/run-20260319_070047-kgnkklsw/run-kgnkklsw.wandb filter=lfs diff=lfs merge=lfs -text
|
| 2107 |
act/act-g1-sim/g1wholebodyxmovebendpickteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604091356/wandb/run-20260409_135628-skni4cjp/run-skni4cjp.wandb filter=lfs diff=lfs merge=lfs -text
|
| 2108 |
act/act-g1-sim/g1wholebodyxmovepick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604040237/wandb/run-20260404_023728-qq4fojro/run-qq4fojro.wandb filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2106 |
act/act-g1-sim/g1wholebodytabletopgrasp-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603190700/wandb/run-20260319_070047-kgnkklsw/run-kgnkklsw.wandb filter=lfs diff=lfs merge=lfs -text
|
| 2107 |
act/act-g1-sim/g1wholebodyxmovebendpickteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604091356/wandb/run-20260409_135628-skni4cjp/run-skni4cjp.wandb filter=lfs diff=lfs merge=lfs -text
|
| 2108 |
act/act-g1-sim/g1wholebodyxmovepick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604040237/wandb/run-20260404_023728-qq4fojro/run-qq4fojro.wandb filter=lfs diff=lfs merge=lfs -text
|
| 2109 |
+
dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142630-q4pxxlgy/run-q4pxxlgy.wandb filter=lfs diff=lfs merge=lfs -text
|
| 2110 |
+
dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/wandb/run-20260410_020501-sjem5sh4/run-sjem5sh4.wandb filter=lfs diff=lfs merge=lfs -text
|
| 2111 |
+
dp/diffusion-policy-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/wandb/run-20260408_155038-072o1sgn/run-072o1sgn.wandb filter=lfs diff=lfs merge=lfs -text
|
| 2112 |
+
dp/diffusion-policy-g1-sim/g1wholebodytabletopgrasp-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603190701/wandb/run-20260319_070107-xj1s7wqu/run-xj1s7wqu.wandb filter=lfs diff=lfs merge=lfs -text
|
| 2113 |
+
dp/diffusion-policy-g1-sim/g1wholebodyxmovebendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604020758/wandb/run-20260402_075844-zpz3nbw9/run-zpz3nbw9.wandb filter=lfs diff=lfs merge=lfs -text
|
| 2114 |
+
dp/diffusion-policy-g1-sim/g1wholebodyxmovepick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604040957/wandb/run-20260404_095754-ei6sek0s/run-ei6sek0s.wandb filter=lfs diff=lfs merge=lfs -text
|
dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/argv.txt
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
scripts/train.py
|
| 2 |
+
simple_dp_config
|
| 3 |
+
--seed=2026
|
| 4 |
+
--exp=g1wholebodybendpick-v0
|
| 5 |
+
--train.name=diffusion-policy-g1
|
| 6 |
+
--log.report-to=wandb
|
| 7 |
+
--train.data_parallel=ddp
|
| 8 |
+
--train.mixed_precision=bf16
|
| 9 |
+
--train.train-batch-size=32
|
| 10 |
+
--train.warmup-steps=1000
|
| 11 |
+
--train.warmup-ratio=None
|
| 12 |
+
--train.checkpointing-steps=5000
|
| 13 |
+
--train.validation_steps=500
|
| 14 |
+
--train.val_num_batches=20
|
| 15 |
+
--train.gradient_accumulation_steps=1
|
| 16 |
+
--train.max-training-steps=40000
|
| 17 |
+
--train.learning-rate=1e-4
|
| 18 |
+
--train.max-grad-norm=1.0
|
| 19 |
+
--train.lr_scheduler_kwargs.weight_decay=1e-6
|
| 20 |
+
--train.lr_scheduler_kwargs.betas 0.95 0.999
|
| 21 |
+
--train.lr_scheduler_type=cosine
|
| 22 |
+
--data.root_dir=/data/jliu/data
|
| 23 |
+
--data.train-repo-ids=G1WholebodyBendPick-v0
|
| 24 |
+
--data.transform.repack.action_chunk_size=16
|
| 25 |
+
--data.transform.repack.pad-action-dim=36
|
| 26 |
+
--data.transform.repack.pad-state-dim=36
|
| 27 |
+
--data.transform.field.stat-path=meta/stats_psi0.json
|
| 28 |
+
--data.transform.field.stat-action-key=action
|
| 29 |
+
--data.transform.field.stat-state-key=states
|
| 30 |
+
--data.transform.field.normalize-state
|
| 31 |
+
--data.transform.field.action-norm-type=bounds
|
| 32 |
+
--data.transform.field.pad-action-dim=36
|
| 33 |
+
--data.transform.field.pad-state-dim=36
|
| 34 |
+
--data.transform.model.img-aug
|
| 35 |
+
--model.action-chunk-size=16
|
| 36 |
+
--model.action-dim=36
|
| 37 |
+
--model.obs-dim=36
|
dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000/ema_net.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8d8c4727a1ec7c89a0f0820a0f468b9b1d58f93a71ce3f35cf1ec667030f06d4
|
| 3 |
+
size 337302879
|
dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c55c264b9f098847a7d1b86f93c4ca548105d5f164df7e7897528b882572b264
|
| 3 |
+
size 337238328
|
dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000/optimizer.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:10ec543c4a0c00c14e86eb72965ea25f3b1fa217ba8601cbe1bc6f410606fa05
|
| 3 |
+
size 674602251
|
dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000/random_states_0.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:faf49f02df00df10b1ca449b17a5d7c25fab7cf942cab6edc6724e07bc59630e
|
| 3 |
+
size 15473
|
dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000/random_states_1.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1ee0d940b68d363b86689ce83b318a89f6f1b260c687312fcd2dfbf60616727b
|
| 3 |
+
size 15473
|
dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000/random_states_2.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ec7310301652e67dc740290e1c03e472c6851bc8553321fa426c5dec4d2f0a35
|
| 3 |
+
size 15473
|
dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000/random_states_3.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1caa7eab048227d902a2778748c4f035dcca5f9902327bc5b7eab7cebc0e0645
|
| 3 |
+
size 15473
|
dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000/scheduler.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d3ee6860bb6aecdd232f5d24bba08e626d7d56119a5f05ba8c2ad8eecf56fe38
|
| 3 |
+
size 1465
|
dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/envs.txt
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
OMP_NUM_THREADS=32
|
| 2 |
+
HF_HOME=/data/cache
|
| 3 |
+
TORCH_HOME=/data/cache
|
| 4 |
+
HF_TOKEN=hf_...TiKa
|
| 5 |
+
HF_LEROBOT_HOME=/data/data/lerobot
|
| 6 |
+
WE_HOME=Not Set
|
| 7 |
+
DATA_HOME=/data/data
|
| 8 |
+
UV_CACHE_DIR=/data/cache
|
| 9 |
+
WANDB_API_KEY=90e...5c06
|
| 10 |
+
PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
|
| 11 |
+
CUDA_VISIBLE_DEVICES=0,1,2,3
|
| 12 |
+
WORLD_SIZE=4
|
| 13 |
+
LOCAL_WORLD_SIZE=4
|
| 14 |
+
RANK=0
|
| 15 |
+
LOCAL_RANK=0
|
| 16 |
+
MASTER_ADDR=nebula99
|
| 17 |
+
MASTER_PORT=41289
|
dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/run_config.json
ADDED
|
@@ -0,0 +1,300 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"exp": "g1wholebodybendpick-v0",
|
| 3 |
+
"seed": 2026,
|
| 4 |
+
"auto_tag_run": false,
|
| 5 |
+
"eval": false,
|
| 6 |
+
"debug": false,
|
| 7 |
+
"timestamp": "2603181426",
|
| 8 |
+
"log": {
|
| 9 |
+
"logging_dir": "logs",
|
| 10 |
+
"report_to": "wandb",
|
| 11 |
+
"log_freq": 100
|
| 12 |
+
},
|
| 13 |
+
"wandb": {
|
| 14 |
+
"project": "psi",
|
| 15 |
+
"entity": "jliu530-soochow-university",
|
| 16 |
+
"group": "diffusion-policy-g1",
|
| 17 |
+
"id": "q4pxxlgy",
|
| 18 |
+
"name": "g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426",
|
| 19 |
+
"resume": "allow"
|
| 20 |
+
},
|
| 21 |
+
"train": {
|
| 22 |
+
"num_workers": 8,
|
| 23 |
+
"overfit_single_batch": false,
|
| 24 |
+
"name": "diffusion-policy-g1",
|
| 25 |
+
"resume_from_checkpoint": null,
|
| 26 |
+
"skip_resumed_steps": false,
|
| 27 |
+
"hf_token": ".hf_token",
|
| 28 |
+
"lora": false,
|
| 29 |
+
"output_dir": ".runs",
|
| 30 |
+
"gradient_accumulation_steps": 1,
|
| 31 |
+
"mixed_precision": "bf16",
|
| 32 |
+
"max_grad_norm": 1.0,
|
| 33 |
+
"optimizer_foreach": null,
|
| 34 |
+
"train_batch_size": 32,
|
| 35 |
+
"val_batch_size": 16,
|
| 36 |
+
"val_num_batches": 20,
|
| 37 |
+
"checkpointing_steps": 5000,
|
| 38 |
+
"max_checkpoints_to_keep": null,
|
| 39 |
+
"validation_steps": 500,
|
| 40 |
+
"learning_rate": 0.0001,
|
| 41 |
+
"lr_scheduler_type": "cosine",
|
| 42 |
+
"lr_scheduler_kwargs": {
|
| 43 |
+
"betas": [
|
| 44 |
+
0.95,
|
| 45 |
+
0.999
|
| 46 |
+
],
|
| 47 |
+
"weight_decay": 1e-6,
|
| 48 |
+
"eps": 1e-8
|
| 49 |
+
},
|
| 50 |
+
"scheduler_specific_kwargs": {},
|
| 51 |
+
"data_parallel": "ddp",
|
| 52 |
+
"sharding_strategy": "full-shard",
|
| 53 |
+
"deepspeed_config": "/data/jliu/psi/src/InternVLA/config/deepseeds/zero3.json",
|
| 54 |
+
"enable_gradient_checkpointing": true,
|
| 55 |
+
"enable_mixed_precision_training": true,
|
| 56 |
+
"reduce_in_full_precision": true,
|
| 57 |
+
"max_training_steps": 40000,
|
| 58 |
+
"num_train_epochs": null,
|
| 59 |
+
"warmup_steps": 1000,
|
| 60 |
+
"warmup_ratio": null
|
| 61 |
+
},
|
| 62 |
+
"data": {
|
| 63 |
+
"transform": {
|
| 64 |
+
"repack": {
|
| 65 |
+
"dataset_name": "simple",
|
| 66 |
+
"num_past_frames": 0,
|
| 67 |
+
"action_chunk_size": 16,
|
| 68 |
+
"pad_action_dim": 36,
|
| 69 |
+
"pad_state_dim": 36
|
| 70 |
+
},
|
| 71 |
+
"model": {
|
| 72 |
+
"resize": {
|
| 73 |
+
"size": [
|
| 74 |
+
256,
|
| 75 |
+
480
|
| 76 |
+
]
|
| 77 |
+
},
|
| 78 |
+
"color_jitter": {
|
| 79 |
+
"brightness": 0.2,
|
| 80 |
+
"contrast": [
|
| 81 |
+
0.8,
|
| 82 |
+
1.2
|
| 83 |
+
],
|
| 84 |
+
"saturation": [
|
| 85 |
+
0.8,
|
| 86 |
+
1.2
|
| 87 |
+
],
|
| 88 |
+
"hue": 0.05
|
| 89 |
+
},
|
| 90 |
+
"center_crop": {
|
| 91 |
+
"size": [
|
| 92 |
+
224,
|
| 93 |
+
224
|
| 94 |
+
]
|
| 95 |
+
},
|
| 96 |
+
"normalize": {
|
| 97 |
+
"mean": [
|
| 98 |
+
0.485,
|
| 99 |
+
0.456,
|
| 100 |
+
0.406
|
| 101 |
+
],
|
| 102 |
+
"std": [
|
| 103 |
+
0.229,
|
| 104 |
+
0.224,
|
| 105 |
+
0.225
|
| 106 |
+
]
|
| 107 |
+
},
|
| 108 |
+
"img_aug": true
|
| 109 |
+
},
|
| 110 |
+
"field": {
|
| 111 |
+
"stat_path": "meta/stats_psi0.json",
|
| 112 |
+
"action_norm_type": "bounds",
|
| 113 |
+
"stat_action_key": "action",
|
| 114 |
+
"stat_state_key": "states",
|
| 115 |
+
"use_norm_mask": false,
|
| 116 |
+
"action_norm_masks": [
|
| 117 |
+
true,
|
| 118 |
+
true,
|
| 119 |
+
true,
|
| 120 |
+
true,
|
| 121 |
+
true,
|
| 122 |
+
true,
|
| 123 |
+
false
|
| 124 |
+
],
|
| 125 |
+
"action_min": [
|
| 126 |
+
-0.13059291243553162,
|
| 127 |
+
-0.09108058363199234,
|
| 128 |
+
-0.0024844733998179436,
|
| 129 |
+
-0.20733775198459625,
|
| 130 |
+
-0.15850023925304413,
|
| 131 |
+
-0.17450474202632904,
|
| 132 |
+
-0.2997315526008606,
|
| 133 |
+
-0.015391111373901367,
|
| 134 |
+
-0.34571564197540283,
|
| 135 |
+
-0.4991437792778015,
|
| 136 |
+
0.0,
|
| 137 |
+
0.0,
|
| 138 |
+
0.0,
|
| 139 |
+
0.0,
|
| 140 |
+
-0.1015840545296669,
|
| 141 |
+
-0.06647031009197235,
|
| 142 |
+
-0.16578954458236694,
|
| 143 |
+
-0.14477218687534332,
|
| 144 |
+
-0.3665394186973572,
|
| 145 |
+
-0.28364259004592896,
|
| 146 |
+
-0.1775387078523636,
|
| 147 |
+
-0.48419490456581116,
|
| 148 |
+
-0.7551082968711853,
|
| 149 |
+
-0.2692946195602417,
|
| 150 |
+
-0.03164339065551758,
|
| 151 |
+
-0.00003876500704791397,
|
| 152 |
+
-0.3909206688404083,
|
| 153 |
+
0.0,
|
| 154 |
+
-0.04351663216948509,
|
| 155 |
+
-0.014203650876879692,
|
| 156 |
+
-0.049649015069007874,
|
| 157 |
+
0.44999998807907104,
|
| 158 |
+
0.0,
|
| 159 |
+
0.0,
|
| 160 |
+
0.0,
|
| 161 |
+
0.0
|
| 162 |
+
],
|
| 163 |
+
"action_max": [
|
| 164 |
+
0.08620641380548477,
|
| 165 |
+
0.13058121502399445,
|
| 166 |
+
0.22948147356510162,
|
| 167 |
+
0.020551620051264763,
|
| 168 |
+
0.005824880674481392,
|
| 169 |
+
0.010019193403422832,
|
| 170 |
+
8.43817247186962e-7,
|
| 171 |
+
0.39566752314567566,
|
| 172 |
+
0.0,
|
| 173 |
+
0.0,
|
| 174 |
+
0.4860266447067261,
|
| 175 |
+
1.0467392206192017,
|
| 176 |
+
0.6470075845718384,
|
| 177 |
+
0.8298009037971497,
|
| 178 |
+
0.03516175225377083,
|
| 179 |
+
0.11019192636013031,
|
| 180 |
+
0.04779902100563049,
|
| 181 |
+
0.12850724160671234,
|
| 182 |
+
0.000038688118365826085,
|
| 183 |
+
0.0012142359046265483,
|
| 184 |
+
0.000033343669201713055,
|
| 185 |
+
0.002679983852431178,
|
| 186 |
+
0.00041063950629904866,
|
| 187 |
+
0.1973484456539154,
|
| 188 |
+
0.2633756697177887,
|
| 189 |
+
0.34943076968193054,
|
| 190 |
+
0.0012102096807211637,
|
| 191 |
+
0.8342975974082947,
|
| 192 |
+
0.31870752573013306,
|
| 193 |
+
0.45533719658851624,
|
| 194 |
+
0.15729404985904694,
|
| 195 |
+
0.75,
|
| 196 |
+
0.0,
|
| 197 |
+
0.0,
|
| 198 |
+
0.0,
|
| 199 |
+
0.0
|
| 200 |
+
],
|
| 201 |
+
"state_min": [
|
| 202 |
+
-0.13899999856948853,
|
| 203 |
+
-0.09099991619586945,
|
| 204 |
+
-5.989517215532203e-11,
|
| 205 |
+
-0.20900000631809235,
|
| 206 |
+
-0.1589999943971634,
|
| 207 |
+
-0.210999995470047,
|
| 208 |
+
-0.3009999990463257,
|
| 209 |
+
-0.01600000075995922,
|
| 210 |
+
-0.1860000044107437,
|
| 211 |
+
-0.6940000057220459,
|
| 212 |
+
0.0,
|
| 213 |
+
0.0,
|
| 214 |
+
0.0,
|
| 215 |
+
0.0,
|
| 216 |
+
-0.1019991859793663,
|
| 217 |
+
-0.06899992376565933,
|
| 218 |
+
-0.16899999976158142,
|
| 219 |
+
-0.14499999582767487,
|
| 220 |
+
-0.3709999918937683,
|
| 221 |
+
-0.28700000047683716,
|
| 222 |
+
-0.17800045013427734,
|
| 223 |
+
-0.4869999885559082,
|
| 224 |
+
-0.7599999904632568,
|
| 225 |
+
-0.27300000190734863,
|
| 226 |
+
-0.029999999329447746,
|
| 227 |
+
0.0,
|
| 228 |
+
-0.39100033044815063,
|
| 229 |
+
-0.0010000000474974513,
|
| 230 |
+
0.0,
|
| 231 |
+
-0.15000000596046448,
|
| 232 |
+
0.0,
|
| 233 |
+
0.44999998807907104,
|
| 234 |
+
0.0,
|
| 235 |
+
0.0,
|
| 236 |
+
0.0,
|
| 237 |
+
0.0
|
| 238 |
+
],
|
| 239 |
+
"state_max": [
|
| 240 |
+
0.0860000029206276,
|
| 241 |
+
0.2720000147819519,
|
| 242 |
+
0.23100000619888306,
|
| 243 |
+
0.0,
|
| 244 |
+
6.510182259944486e-8,
|
| 245 |
+
0.0,
|
| 246 |
+
0.0,
|
| 247 |
+
0.5550000071525574,
|
| 248 |
+
0.02100004442036152,
|
| 249 |
+
0.0,
|
| 250 |
+
0.5429999828338623,
|
| 251 |
+
1.13100004196167,
|
| 252 |
+
0.5770000219345093,
|
| 253 |
+
0.9580000042915344,
|
| 254 |
+
0.07141251862049103,
|
| 255 |
+
0.10899999737739563,
|
| 256 |
+
0.04699999839067459,
|
| 257 |
+
0.13600000739097595,
|
| 258 |
+
0.0,
|
| 259 |
+
0.003000000026077032,
|
| 260 |
+
0.0,
|
| 261 |
+
0.009999999776482582,
|
| 262 |
+
0.0020000000949949026,
|
| 263 |
+
0.2029999941587448,
|
| 264 |
+
0.2759999930858612,
|
| 265 |
+
0.3499999940395355,
|
| 266 |
+
0.003000000026077032,
|
| 267 |
+
0.8370000123977661,
|
| 268 |
+
0.0,
|
| 269 |
+
0.0,
|
| 270 |
+
0.0,
|
| 271 |
+
0.75,
|
| 272 |
+
0.0,
|
| 273 |
+
0.0,
|
| 274 |
+
0.0,
|
| 275 |
+
0.0
|
| 276 |
+
],
|
| 277 |
+
"normalize_state": true,
|
| 278 |
+
"pad_action_dim": 36,
|
| 279 |
+
"pad_state_dim": 36
|
| 280 |
+
}
|
| 281 |
+
},
|
| 282 |
+
"root_dir": "/data/jliu/data",
|
| 283 |
+
"train_repo_ids": [
|
| 284 |
+
"G1WholebodyBendPick-v0"
|
| 285 |
+
],
|
| 286 |
+
"val_repo_ids": [
|
| 287 |
+
"G1WholebodyBendPick-v0"
|
| 288 |
+
]
|
| 289 |
+
},
|
| 290 |
+
"model": {
|
| 291 |
+
"num_diffusion_iters": 100,
|
| 292 |
+
"action_chunk_size": 16,
|
| 293 |
+
"num_cameras": 1,
|
| 294 |
+
"share_vision_encoder": false,
|
| 295 |
+
"obs_horizon": 1,
|
| 296 |
+
"action_exec_horizon": 6,
|
| 297 |
+
"action_dim": 36,
|
| 298 |
+
"obs_dim": 36
|
| 299 |
+
}
|
| 300 |
+
}
|
dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/debug.log
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-03-18 14:26:30,764 INFO MainThread:279364 [wandb_setup.py:_flush():81] Current SDK version is 0.25.1
|
| 2 |
+
2026-03-18 14:26:30,764 INFO MainThread:279364 [wandb_setup.py:_flush():81] Configure stats pid to 279364
|
| 3 |
+
2026-03-18 14:26:30,764 INFO MainThread:279364 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-03-18 14:26:30,765 INFO MainThread:279364 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data/jliu/psi/.runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142630-q4pxxlgy/logs/debug.log
|
| 5 |
+
2026-03-18 14:26:30,765 INFO MainThread:279364 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data/jliu/psi/.runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142630-q4pxxlgy/logs/debug-internal.log
|
| 6 |
+
2026-03-18 14:26:30,765 INFO MainThread:279364 [wandb_init.py:init():844] calling init triggers
|
| 7 |
+
2026-03-18 14:26:30,765 INFO MainThread:279364 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'_wandb': {}}
|
| 9 |
+
2026-03-18 14:26:30,765 INFO MainThread:279364 [wandb_init.py:init():892] starting backend
|
| 10 |
+
2026-03-18 14:26:31,018 INFO MainThread:279364 [wandb_init.py:init():895] sending inform_init request
|
| 11 |
+
2026-03-18 14:26:31,028 INFO MainThread:279364 [wandb_init.py:init():903] backend started and connected
|
| 12 |
+
2026-03-18 14:26:31,029 INFO MainThread:279364 [wandb_init.py:init():973] updated telemetry
|
| 13 |
+
2026-03-18 14:26:31,035 INFO MainThread:279364 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-03-18 14:26:32,251 INFO MainThread:279364 [wandb_init.py:init():1042] starting run threads in backend
|
| 15 |
+
2026-03-18 14:26:32,404 INFO MainThread:279364 [wandb_run.py:_console_start():2524] atexit reg
|
| 16 |
+
2026-03-18 14:26:32,404 INFO MainThread:279364 [wandb_run.py:_redirect():2373] redirect: wrap_raw
|
| 17 |
+
2026-03-18 14:26:32,404 INFO MainThread:279364 [wandb_run.py:_redirect():2442] Wrapping output streams.
|
| 18 |
+
2026-03-18 14:26:32,404 INFO MainThread:279364 [wandb_run.py:_redirect():2465] Redirects installed.
|
| 19 |
+
2026-03-18 14:26:32,409 INFO MainThread:279364 [wandb_init.py:init():1082] run started, returning control to user process
|
| 20 |
+
2026-03-18 14:26:32,411 INFO MainThread:279364 [wandb_run.py:_config_callback():1403] config_cb None None {'exp': 'g1wholebodybendpick-v0', 'seed': 2026, 'auto_tag_run': False, 'eval': False, 'debug': False, 'timestamp': '2603181426', 'log': {'logging_dir': 'logs', 'report_to': 'wandb', 'log_freq': 100}, 'wandb': {'project': 'psi', 'entity': 'jliu530-soochow-university', 'group': None, 'id': None, 'name': None, 'resume': 'allow'}, 'train': {'num_workers': 8, 'overfit_single_batch': False, 'name': 'diffusion-policy-g1', 'resume_from_checkpoint': None, 'skip_resumed_steps': False, 'hf_token': '.hf_token', 'lora': False, 'output_dir': '.runs', 'gradient_accumulation_steps': 1, 'mixed_precision': 'bf16', 'max_grad_norm': 1.0, 'optimizer_foreach': None, 'train_batch_size': 32, 'val_batch_size': 16, 'val_num_batches': 20, 'checkpointing_steps': 5000, 'max_checkpoints_to_keep': None, 'validation_steps': 500, 'learning_rate': 0.0001, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {'betas': [0.95, 0.999], 'weight_decay': 1e-06, 'eps': 1e-08}, 'scheduler_specific_kwargs': {}, 'data_parallel': 'ddp', 'sharding_strategy': 'full-shard', 'deepspeed_config': '/data/jliu/psi/src/InternVLA/config/deepseeds/zero3.json', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': True, 'max_training_steps': 40000, 'num_train_epochs': None, 'warmup_steps': 1000, 'warmup_ratio': None}, 'data': {'transform': {'repack': {'dataset_name': 'simple', 'num_past_frames': 0, 'action_chunk_size': 16, 'pad_action_dim': 36, 'pad_state_dim': 36}, 'model': {'resize': {'size': [256, 480]}, 'color_jitter': {'brightness': 0.2, 'contrast': [0.8, 1.2], 'saturation': [0.8, 1.2], 'hue': 0.05}, 'center_crop': {'size': [224, 224]}, 'normalize': {'mean': [0.485, 0.456, 0.406], 'std': [0.229, 0.224, 0.225]}, 'img_aug': True}, 'field': {'stat_path': 'meta/stats_psi0.json', 'action_norm_type': 'bounds', 'stat_action_key': 'action', 'stat_state_key': 'states', 'use_norm_mask': False, 'action_norm_masks': [True, True, True, True, True, True, False], 'action_min': [-0.13059291243553162, -0.09108058363199234, -0.0024844733998179436, -0.20733775198459625, -0.15850023925304413, -0.17450474202632904, -0.2997315526008606, -0.015391111373901367, -0.34571564197540283, -0.4991437792778015, 0.0, 0.0, 0.0, 0.0, -0.1015840545296669, -0.06647031009197235, -0.16578954458236694, -0.14477218687534332, -0.3665394186973572, -0.28364259004592896, -0.1775387078523636, -0.48419490456581116, -0.7551082968711853, -0.2692946195602417, -0.03164339065551758, -3.876500704791397e-05, -0.3909206688404083, 0.0, -0.04351663216948509, -0.014203650876879692, -0.049649015069007874, 0.44999998807907104, 0.0, 0.0, 0.0, 0.0], 'action_max': [0.08620641380548477, 0.13058121502399445, 0.22948147356510162, 0.020551620051264763, 0.005824880674481392, 0.010019193403422832, 8.43817247186962e-07, 0.39566752314567566, 0.0, 0.0, 0.4860266447067261, 1.0467392206192017, 0.6470075845718384, 0.8298009037971497, 0.03516175225377083, 0.11019192636013031, 0.04779902100563049, 0.12850724160671234, 3.8688118365826085e-05, 0.0012142359046265483, 3.3343669201713055e-05, 0.002679983852431178, 0.00041063950629904866, 0.1973484456539154, 0.2633756697177887, 0.34943076968193054, 0.0012102096807211637, 0.8342975974082947, 0.31870752573013306, 0.45533719658851624, 0.15729404985904694, 0.75, 0.0, 0.0, 0.0, 0.0], 'state_min': [-0.13899999856948853, -0.09099991619586945, -5.989517215532203e-11, -0.20900000631809235, -0.1589999943971634, -0.210999995470047, -0.3009999990463257, -0.01600000075995922, -0.1860000044107437, -0.6940000057220459, 0.0, 0.0, 0.0, 0.0, -0.1019991859793663, -0.06899992376565933, -0.16899999976158142, -0.14499999582767487, -0.3709999918937683, -0.28700000047683716, -0.17800045013427734, -0.4869999885559082, -0.7599999904632568, -0.27300000190734863, -0.029999999329447746, 0.0, -0.39100033044815063, -0.0010000000474974513, 0.0, -0.15000000596046448, 0.0, 0.44999998807907104, 0.0, 0.0, 0.0, 0.0], 'state_max': [0.0860000029206276, 0.2720000147819519, 0.23100000619888306, 0.0, 6.510182259944486e-08, 0.0, 0.0, 0.5550000071525574, 0.02100004442036152, 0.0, 0.5429999828338623, 1.13100004196167, 0.5770000219345093, 0.9580000042915344, 0.07141251862049103, 0.10899999737739563, 0.04699999839067459, 0.13600000739097595, 0.0, 0.003000000026077032, 0.0, 0.009999999776482582, 0.0020000000949949026, 0.2029999941587448, 0.2759999930858612, 0.3499999940395355, 0.003000000026077032, 0.8370000123977661, 0.0, 0.0, 0.0, 0.75, 0.0, 0.0, 0.0, 0.0], 'normalize_state': True, 'pad_action_dim': 36, 'pad_state_dim': 36}}, 'root_dir': '/data/jliu/data', 'train_repo_ids': ['G1WholebodyBendPick-v0'], 'val_repo_ids': ['G1WholebodyBendPick-v0']}, 'model': {'num_diffusion_iters': 100, 'action_chunk_size': 16, 'num_cameras': 1, 'share_vision_encoder': False, 'obs_horizon': 1, 'action_exec_horizon': 6, 'action_dim': 36, 'obs_dim': 36}, 'environment_variables': {'OMP_NUM_THREADS': '32', 'HF_HOME': '/data/cache', 'TORCH_HOME': '/data/cache', 'HF_TOKEN': 'hf_...TiKa', 'HF_LEROBOT_HOME': '/data/data/lerobot', 'WE_HOME': 'Not Set', 'DATA_HOME': '/data/data', 'UV_CACHE_DIR': '/data/cache', 'WANDB_API_KEY': '90e...5c06', 'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION': 'python', 'CUDA_VISIBLE_DEVICES': '0,1,2,3', 'WORLD_SIZE': '4', 'LOCAL_WORLD_SIZE': '4', 'RANK': '0', 'LOCAL_RANK': '0', 'MASTER_ADDR': 'nebula99', 'MASTER_PORT': '41289'}}
|
| 21 |
+
2026-03-18 20:24:38,632 INFO MainThread:279364 [wandb_run.py:_finish():2291] finishing run jliu530-soochow-university/psi/q4pxxlgy
|
| 22 |
+
2026-03-18 20:24:38,634 INFO MainThread:279364 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0
|
| 23 |
+
2026-03-18 20:24:38,634 INFO MainThread:279364 [wandb_run.py:_restore():2472] restore
|
| 24 |
+
2026-03-18 20:24:38,635 INFO MainThread:279364 [wandb_run.py:_restore():2478] restore done
|
| 25 |
+
2026-03-18 20:24:39,775 INFO MainThread:279364 [wandb_run.py:_footer_sync_info():3868] logging synced files
|
dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142630-q4pxxlgy/files/config.yaml
ADDED
|
@@ -0,0 +1,436 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
_wandb:
|
| 2 |
+
value:
|
| 3 |
+
cli_version: 0.25.1
|
| 4 |
+
e:
|
| 5 |
+
9to9pktd3ndtvl9vuv59f4rwekbfrlue:
|
| 6 |
+
args:
|
| 7 |
+
- simple_dp_config
|
| 8 |
+
- --seed=2026
|
| 9 |
+
- --exp=g1wholebodybendpick-v0
|
| 10 |
+
- --train.name=diffusion-policy-g1
|
| 11 |
+
- --log.report-to=wandb
|
| 12 |
+
- --train.data_parallel=ddp
|
| 13 |
+
- --train.mixed_precision=bf16
|
| 14 |
+
- --train.train-batch-size=32
|
| 15 |
+
- --train.warmup-steps=1000
|
| 16 |
+
- --train.warmup-ratio=None
|
| 17 |
+
- --train.checkpointing-steps=5000
|
| 18 |
+
- --train.validation_steps=500
|
| 19 |
+
- --train.val_num_batches=20
|
| 20 |
+
- --train.gradient_accumulation_steps=1
|
| 21 |
+
- --train.max-training-steps=40000
|
| 22 |
+
- --train.learning-rate=1e-4
|
| 23 |
+
- --train.max-grad-norm=1.0
|
| 24 |
+
- --train.lr_scheduler_kwargs.weight_decay=1e-6
|
| 25 |
+
- --train.lr_scheduler_kwargs.betas
|
| 26 |
+
- "0.95"
|
| 27 |
+
- "0.999"
|
| 28 |
+
- --train.lr_scheduler_type=cosine
|
| 29 |
+
- --data.root_dir=/data/jliu/data
|
| 30 |
+
- --data.train-repo-ids=G1WholebodyBendPick-v0
|
| 31 |
+
- --data.transform.repack.action_chunk_size=16
|
| 32 |
+
- --data.transform.repack.pad-action-dim=36
|
| 33 |
+
- --data.transform.repack.pad-state-dim=36
|
| 34 |
+
- --data.transform.field.stat-path=meta/stats_psi0.json
|
| 35 |
+
- --data.transform.field.stat-action-key=action
|
| 36 |
+
- --data.transform.field.stat-state-key=states
|
| 37 |
+
- --data.transform.field.normalize-state
|
| 38 |
+
- --data.transform.field.action-norm-type=bounds
|
| 39 |
+
- --data.transform.field.pad-action-dim=36
|
| 40 |
+
- --data.transform.field.pad-state-dim=36
|
| 41 |
+
- --data.transform.model.img-aug
|
| 42 |
+
- --model.action-chunk-size=16
|
| 43 |
+
- --model.action-dim=36
|
| 44 |
+
- --model.obs-dim=36
|
| 45 |
+
codePath: scripts/train.py
|
| 46 |
+
codePathLocal: scripts/train.py
|
| 47 |
+
cpu_count: 128
|
| 48 |
+
cpu_count_logical: 128
|
| 49 |
+
cudaVersion: "12.9"
|
| 50 |
+
disk:
|
| 51 |
+
/:
|
| 52 |
+
total: "105089261568"
|
| 53 |
+
used: "99538145280"
|
| 54 |
+
email: jliu530@163.com
|
| 55 |
+
executable: /data/jliu/psi/.venv-dp/bin/python3
|
| 56 |
+
git:
|
| 57 |
+
commit: 04614628ecb677f5f278e2d31c1103aed8127f26
|
| 58 |
+
remote: https://github.com/songlin/psi.git
|
| 59 |
+
gpu: NVIDIA A100-SXM4-80GB
|
| 60 |
+
gpu_count: 8
|
| 61 |
+
gpu_nvidia:
|
| 62 |
+
- architecture: Ampere
|
| 63 |
+
cudaCores: 6912
|
| 64 |
+
memoryTotal: "85899345920"
|
| 65 |
+
name: NVIDIA A100-SXM4-80GB
|
| 66 |
+
uuid: GPU-37370f00-90f5-4e03-f763-7a35649e6783
|
| 67 |
+
- architecture: Ampere
|
| 68 |
+
cudaCores: 6912
|
| 69 |
+
memoryTotal: "85899345920"
|
| 70 |
+
name: NVIDIA A100-SXM4-80GB
|
| 71 |
+
uuid: GPU-d4b40383-188b-cc16-9180-20c1a71a777f
|
| 72 |
+
- architecture: Ampere
|
| 73 |
+
cudaCores: 6912
|
| 74 |
+
memoryTotal: "85899345920"
|
| 75 |
+
name: NVIDIA A100-SXM4-80GB
|
| 76 |
+
uuid: GPU-0f3d827c-66cf-04da-b182-4ed0414a2549
|
| 77 |
+
- architecture: Ampere
|
| 78 |
+
cudaCores: 6912
|
| 79 |
+
memoryTotal: "85899345920"
|
| 80 |
+
name: NVIDIA A100-SXM4-80GB
|
| 81 |
+
uuid: GPU-164a49bb-43d3-2250-59e2-1b40eee0757f
|
| 82 |
+
- architecture: Ampere
|
| 83 |
+
cudaCores: 6912
|
| 84 |
+
memoryTotal: "85899345920"
|
| 85 |
+
name: NVIDIA A100-SXM4-80GB
|
| 86 |
+
uuid: GPU-c0471f40-ae19-5371-a1af-da406ca30f83
|
| 87 |
+
- architecture: Ampere
|
| 88 |
+
cudaCores: 6912
|
| 89 |
+
memoryTotal: "85899345920"
|
| 90 |
+
name: NVIDIA A100-SXM4-80GB
|
| 91 |
+
uuid: GPU-d9040f59-13d3-6c37-4ea0-c6c751424527
|
| 92 |
+
- architecture: Ampere
|
| 93 |
+
cudaCores: 6912
|
| 94 |
+
memoryTotal: "85899345920"
|
| 95 |
+
name: NVIDIA A100-SXM4-80GB
|
| 96 |
+
uuid: GPU-f834d0e8-a118-24c7-5461-30fce111bea6
|
| 97 |
+
- architecture: Ampere
|
| 98 |
+
cudaCores: 6912
|
| 99 |
+
memoryTotal: "85899345920"
|
| 100 |
+
name: NVIDIA A100-SXM4-80GB
|
| 101 |
+
uuid: GPU-6e687915-3247-735c-6641-544540804e79
|
| 102 |
+
host: nebula99
|
| 103 |
+
memory:
|
| 104 |
+
total: "1623177744384"
|
| 105 |
+
os: Linux-6.8.0-100-generic-x86_64-with-glibc2.39
|
| 106 |
+
program: /data/jliu/psi/scripts/train.py
|
| 107 |
+
python: CPython 3.10.20
|
| 108 |
+
root: /data/jliu/psi/.runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426
|
| 109 |
+
startedAt: "2026-03-18T14:26:30.762673Z"
|
| 110 |
+
writerId: 9to9pktd3ndtvl9vuv59f4rwekbfrlue
|
| 111 |
+
m: []
|
| 112 |
+
python_version: 3.10.20
|
| 113 |
+
t:
|
| 114 |
+
"1":
|
| 115 |
+
- 1
|
| 116 |
+
- 11
|
| 117 |
+
- 41
|
| 118 |
+
- 49
|
| 119 |
+
- 71
|
| 120 |
+
- 83
|
| 121 |
+
"2":
|
| 122 |
+
- 1
|
| 123 |
+
- 11
|
| 124 |
+
- 41
|
| 125 |
+
- 49
|
| 126 |
+
- 51
|
| 127 |
+
- 71
|
| 128 |
+
- 83
|
| 129 |
+
"3":
|
| 130 |
+
- 2
|
| 131 |
+
- 13
|
| 132 |
+
- 61
|
| 133 |
+
"4": 3.10.20
|
| 134 |
+
"5": 0.25.1
|
| 135 |
+
"6": 4.44.2
|
| 136 |
+
"12": 0.25.1
|
| 137 |
+
"13": linux-x86_64
|
| 138 |
+
auto_tag_run:
|
| 139 |
+
value: false
|
| 140 |
+
data:
|
| 141 |
+
value:
|
| 142 |
+
root_dir: /data/jliu/data
|
| 143 |
+
train_repo_ids:
|
| 144 |
+
- G1WholebodyBendPick-v0
|
| 145 |
+
transform:
|
| 146 |
+
field:
|
| 147 |
+
action_max:
|
| 148 |
+
- 0.08620641380548477
|
| 149 |
+
- 0.13058121502399445
|
| 150 |
+
- 0.22948147356510162
|
| 151 |
+
- 0.020551620051264763
|
| 152 |
+
- 0.005824880674481392
|
| 153 |
+
- 0.010019193403422832
|
| 154 |
+
- 8.43817247186962e-07
|
| 155 |
+
- 0.39566752314567566
|
| 156 |
+
- 0
|
| 157 |
+
- 0
|
| 158 |
+
- 0.4860266447067261
|
| 159 |
+
- 1.0467392206192017
|
| 160 |
+
- 0.6470075845718384
|
| 161 |
+
- 0.8298009037971497
|
| 162 |
+
- 0.03516175225377083
|
| 163 |
+
- 0.11019192636013031
|
| 164 |
+
- 0.04779902100563049
|
| 165 |
+
- 0.12850724160671234
|
| 166 |
+
- 3.8688118365826085e-05
|
| 167 |
+
- 0.0012142359046265483
|
| 168 |
+
- 3.3343669201713055e-05
|
| 169 |
+
- 0.002679983852431178
|
| 170 |
+
- 0.00041063950629904866
|
| 171 |
+
- 0.1973484456539154
|
| 172 |
+
- 0.2633756697177887
|
| 173 |
+
- 0.34943076968193054
|
| 174 |
+
- 0.0012102096807211637
|
| 175 |
+
- 0.8342975974082947
|
| 176 |
+
- 0.31870752573013306
|
| 177 |
+
- 0.45533719658851624
|
| 178 |
+
- 0.15729404985904694
|
| 179 |
+
- 0.75
|
| 180 |
+
- 0
|
| 181 |
+
- 0
|
| 182 |
+
- 0
|
| 183 |
+
- 0
|
| 184 |
+
action_min:
|
| 185 |
+
- -0.13059291243553162
|
| 186 |
+
- -0.09108058363199234
|
| 187 |
+
- -0.0024844733998179436
|
| 188 |
+
- -0.20733775198459625
|
| 189 |
+
- -0.15850023925304413
|
| 190 |
+
- -0.17450474202632904
|
| 191 |
+
- -0.2997315526008606
|
| 192 |
+
- -0.015391111373901367
|
| 193 |
+
- -0.34571564197540283
|
| 194 |
+
- -0.4991437792778015
|
| 195 |
+
- 0
|
| 196 |
+
- 0
|
| 197 |
+
- 0
|
| 198 |
+
- 0
|
| 199 |
+
- -0.1015840545296669
|
| 200 |
+
- -0.06647031009197235
|
| 201 |
+
- -0.16578954458236694
|
| 202 |
+
- -0.14477218687534332
|
| 203 |
+
- -0.3665394186973572
|
| 204 |
+
- -0.28364259004592896
|
| 205 |
+
- -0.1775387078523636
|
| 206 |
+
- -0.48419490456581116
|
| 207 |
+
- -0.7551082968711853
|
| 208 |
+
- -0.2692946195602417
|
| 209 |
+
- -0.03164339065551758
|
| 210 |
+
- -3.876500704791397e-05
|
| 211 |
+
- -0.3909206688404083
|
| 212 |
+
- 0
|
| 213 |
+
- -0.04351663216948509
|
| 214 |
+
- -0.014203650876879692
|
| 215 |
+
- -0.049649015069007874
|
| 216 |
+
- 0.44999998807907104
|
| 217 |
+
- 0
|
| 218 |
+
- 0
|
| 219 |
+
- 0
|
| 220 |
+
- 0
|
| 221 |
+
action_norm_masks:
|
| 222 |
+
- true
|
| 223 |
+
- true
|
| 224 |
+
- true
|
| 225 |
+
- true
|
| 226 |
+
- true
|
| 227 |
+
- true
|
| 228 |
+
- false
|
| 229 |
+
action_norm_type: bounds
|
| 230 |
+
normalize_state: true
|
| 231 |
+
pad_action_dim: 36
|
| 232 |
+
pad_state_dim: 36
|
| 233 |
+
stat_action_key: action
|
| 234 |
+
stat_path: meta/stats_psi0.json
|
| 235 |
+
stat_state_key: states
|
| 236 |
+
state_max:
|
| 237 |
+
- 0.0860000029206276
|
| 238 |
+
- 0.2720000147819519
|
| 239 |
+
- 0.23100000619888306
|
| 240 |
+
- 0
|
| 241 |
+
- 6.510182259944486e-08
|
| 242 |
+
- 0
|
| 243 |
+
- 0
|
| 244 |
+
- 0.5550000071525574
|
| 245 |
+
- 0.02100004442036152
|
| 246 |
+
- 0
|
| 247 |
+
- 0.5429999828338623
|
| 248 |
+
- 1.13100004196167
|
| 249 |
+
- 0.5770000219345093
|
| 250 |
+
- 0.9580000042915344
|
| 251 |
+
- 0.07141251862049103
|
| 252 |
+
- 0.10899999737739563
|
| 253 |
+
- 0.04699999839067459
|
| 254 |
+
- 0.13600000739097595
|
| 255 |
+
- 0
|
| 256 |
+
- 0.003000000026077032
|
| 257 |
+
- 0
|
| 258 |
+
- 0.009999999776482582
|
| 259 |
+
- 0.0020000000949949026
|
| 260 |
+
- 0.2029999941587448
|
| 261 |
+
- 0.2759999930858612
|
| 262 |
+
- 0.3499999940395355
|
| 263 |
+
- 0.003000000026077032
|
| 264 |
+
- 0.8370000123977661
|
| 265 |
+
- 0
|
| 266 |
+
- 0
|
| 267 |
+
- 0
|
| 268 |
+
- 0.75
|
| 269 |
+
- 0
|
| 270 |
+
- 0
|
| 271 |
+
- 0
|
| 272 |
+
- 0
|
| 273 |
+
state_min:
|
| 274 |
+
- -0.13899999856948853
|
| 275 |
+
- -0.09099991619586945
|
| 276 |
+
- -5.989517215532203e-11
|
| 277 |
+
- -0.20900000631809235
|
| 278 |
+
- -0.1589999943971634
|
| 279 |
+
- -0.210999995470047
|
| 280 |
+
- -0.3009999990463257
|
| 281 |
+
- -0.01600000075995922
|
| 282 |
+
- -0.1860000044107437
|
| 283 |
+
- -0.6940000057220459
|
| 284 |
+
- 0
|
| 285 |
+
- 0
|
| 286 |
+
- 0
|
| 287 |
+
- 0
|
| 288 |
+
- -0.1019991859793663
|
| 289 |
+
- -0.06899992376565933
|
| 290 |
+
- -0.16899999976158142
|
| 291 |
+
- -0.14499999582767487
|
| 292 |
+
- -0.3709999918937683
|
| 293 |
+
- -0.28700000047683716
|
| 294 |
+
- -0.17800045013427734
|
| 295 |
+
- -0.4869999885559082
|
| 296 |
+
- -0.7599999904632568
|
| 297 |
+
- -0.27300000190734863
|
| 298 |
+
- -0.029999999329447746
|
| 299 |
+
- 0
|
| 300 |
+
- -0.39100033044815063
|
| 301 |
+
- -0.0010000000474974513
|
| 302 |
+
- 0
|
| 303 |
+
- -0.15000000596046448
|
| 304 |
+
- 0
|
| 305 |
+
- 0.44999998807907104
|
| 306 |
+
- 0
|
| 307 |
+
- 0
|
| 308 |
+
- 0
|
| 309 |
+
- 0
|
| 310 |
+
use_norm_mask: false
|
| 311 |
+
model:
|
| 312 |
+
center_crop:
|
| 313 |
+
size:
|
| 314 |
+
- 224
|
| 315 |
+
- 224
|
| 316 |
+
color_jitter:
|
| 317 |
+
brightness: 0.2
|
| 318 |
+
contrast:
|
| 319 |
+
- 0.8
|
| 320 |
+
- 1.2
|
| 321 |
+
hue: 0.05
|
| 322 |
+
saturation:
|
| 323 |
+
- 0.8
|
| 324 |
+
- 1.2
|
| 325 |
+
img_aug: true
|
| 326 |
+
normalize:
|
| 327 |
+
mean:
|
| 328 |
+
- 0.485
|
| 329 |
+
- 0.456
|
| 330 |
+
- 0.406
|
| 331 |
+
std:
|
| 332 |
+
- 0.229
|
| 333 |
+
- 0.224
|
| 334 |
+
- 0.225
|
| 335 |
+
resize:
|
| 336 |
+
size:
|
| 337 |
+
- 256
|
| 338 |
+
- 480
|
| 339 |
+
repack:
|
| 340 |
+
action_chunk_size: 16
|
| 341 |
+
dataset_name: simple
|
| 342 |
+
num_past_frames: 0
|
| 343 |
+
pad_action_dim: 36
|
| 344 |
+
pad_state_dim: 36
|
| 345 |
+
val_repo_ids:
|
| 346 |
+
- G1WholebodyBendPick-v0
|
| 347 |
+
debug:
|
| 348 |
+
value: false
|
| 349 |
+
environment_variables:
|
| 350 |
+
value:
|
| 351 |
+
CUDA_VISIBLE_DEVICES: 0,1,2,3
|
| 352 |
+
DATA_HOME: /data/data
|
| 353 |
+
HF_HOME: /data/cache
|
| 354 |
+
HF_LEROBOT_HOME: /data/data/lerobot
|
| 355 |
+
HF_TOKEN: hf_...TiKa
|
| 356 |
+
LOCAL_RANK: "0"
|
| 357 |
+
LOCAL_WORLD_SIZE: "4"
|
| 358 |
+
MASTER_ADDR: nebula99
|
| 359 |
+
MASTER_PORT: "41289"
|
| 360 |
+
OMP_NUM_THREADS: "32"
|
| 361 |
+
PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python
|
| 362 |
+
RANK: "0"
|
| 363 |
+
TORCH_HOME: /data/cache
|
| 364 |
+
UV_CACHE_DIR: /data/cache
|
| 365 |
+
WANDB_API_KEY: 90e...5c06
|
| 366 |
+
WE_HOME: Not Set
|
| 367 |
+
WORLD_SIZE: "4"
|
| 368 |
+
eval:
|
| 369 |
+
value: false
|
| 370 |
+
exp:
|
| 371 |
+
value: g1wholebodybendpick-v0
|
| 372 |
+
log:
|
| 373 |
+
value:
|
| 374 |
+
log_freq: 100
|
| 375 |
+
logging_dir: logs
|
| 376 |
+
report_to: wandb
|
| 377 |
+
model:
|
| 378 |
+
value:
|
| 379 |
+
action_chunk_size: 16
|
| 380 |
+
action_dim: 36
|
| 381 |
+
action_exec_horizon: 6
|
| 382 |
+
num_cameras: 1
|
| 383 |
+
num_diffusion_iters: 100
|
| 384 |
+
obs_dim: 36
|
| 385 |
+
obs_horizon: 1
|
| 386 |
+
share_vision_encoder: false
|
| 387 |
+
seed:
|
| 388 |
+
value: 2026
|
| 389 |
+
timestamp:
|
| 390 |
+
value: "2603181426"
|
| 391 |
+
train:
|
| 392 |
+
value:
|
| 393 |
+
checkpointing_steps: 5000
|
| 394 |
+
data_parallel: ddp
|
| 395 |
+
deepspeed_config: /data/jliu/psi/src/InternVLA/config/deepseeds/zero3.json
|
| 396 |
+
enable_gradient_checkpointing: true
|
| 397 |
+
enable_mixed_precision_training: true
|
| 398 |
+
gradient_accumulation_steps: 1
|
| 399 |
+
hf_token: .hf_token
|
| 400 |
+
learning_rate: 0.0001
|
| 401 |
+
lora: false
|
| 402 |
+
lr_scheduler_kwargs:
|
| 403 |
+
betas:
|
| 404 |
+
- 0.95
|
| 405 |
+
- 0.999
|
| 406 |
+
eps: 1e-08
|
| 407 |
+
weight_decay: 1e-06
|
| 408 |
+
lr_scheduler_type: cosine
|
| 409 |
+
max_checkpoints_to_keep: null
|
| 410 |
+
max_grad_norm: 1
|
| 411 |
+
max_training_steps: 40000
|
| 412 |
+
mixed_precision: bf16
|
| 413 |
+
name: diffusion-policy-g1
|
| 414 |
+
num_train_epochs: null
|
| 415 |
+
num_workers: 8
|
| 416 |
+
optimizer_foreach: null
|
| 417 |
+
output_dir: .runs
|
| 418 |
+
overfit_single_batch: false
|
| 419 |
+
reduce_in_full_precision: true
|
| 420 |
+
resume_from_checkpoint: null
|
| 421 |
+
sharding_strategy: full-shard
|
| 422 |
+
skip_resumed_steps: false
|
| 423 |
+
train_batch_size: 32
|
| 424 |
+
val_batch_size: 16
|
| 425 |
+
val_num_batches: 20
|
| 426 |
+
validation_steps: 500
|
| 427 |
+
warmup_ratio: null
|
| 428 |
+
warmup_steps: 1000
|
| 429 |
+
wandb:
|
| 430 |
+
value:
|
| 431 |
+
entity: jliu530-soochow-university
|
| 432 |
+
group: null
|
| 433 |
+
id: null
|
| 434 |
+
name: null
|
| 435 |
+
project: psi
|
| 436 |
+
resume: allow
|
dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142630-q4pxxlgy/files/output.log
ADDED
|
@@ -0,0 +1,224 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[2;36m[14:26:32 03/18][0m[2;36m [0m[34mINFO [0m | >> [1m[[0m*[1m][0m Saved configuration to .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m ]8;id=805427;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=514750;file:///data/jliu/psi/scripts/train.py#128\[2m128[0m]8;;\
|
| 2 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0m*[1m][0m Training configurations: ]8;id=2657;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=644039;file:///data/jliu/psi/scripts/train.py#181\[2m181[0m]8;;\
|
| 3 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> training task: [32m'diffusion-policy-g1'[0m ]8;id=471497;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=12046;file:///data/jliu/psi/scripts/train.py#182\[2m182[0m]8;;\
|
| 4 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> run name: g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m ]8;id=329497;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=220436;file:///data/jliu/psi/scripts/train.py#183\[2m183[0m]8;;\
|
| 5 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> seed: [1;36m2026[0m ]8;id=844246;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=394625;file:///data/jliu/psi/scripts/train.py#184\[2m184[0m]8;;\
|
| 6 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> mixed precision: torch.bfloat16 ]8;id=356929;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=94020;file:///data/jliu/psi/scripts/train.py#185\[2m185[0m]8;;\
|
| 7 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> warmup steps: [1;36m1000[0m ]8;id=147732;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=919050;file:///data/jliu/psi/scripts/train.py#186\[2m186[0m]8;;\
|
| 8 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> validation steps: [1;36m500[0m ]8;id=25941;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=888976;file:///data/jliu/psi/scripts/train.py#187\[2m187[0m]8;;\
|
| 9 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> checkpoint steps: [1;36m5000[0m ]8;id=95089;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=418144;file:///data/jliu/psi/scripts/train.py#188\[2m188[0m]8;;\
|
| 10 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> max gradient norm: [1;36m1.0[0m ]8;id=122104;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=446109;file:///data/jliu/psi/scripts/train.py#189\[2m189[0m]8;;\
|
| 11 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> PyTorch version [1;36m2.8[0m.[1;36m0[0m available. ]8;id=514883;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/datasets/config.py\[2mconfig.py[0m]8;;\[2m:[0m]8;id=417488;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/datasets/config.py#54\[2m54[0m]8;;\
|
| 12 |
+
Resolving data files: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████| 100/100 [00:00<00:00, 188001.08it/s]
|
| 13 |
+
Resolving data files: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████| 100/100 [00:00<00:00, 152797.96it/s]
|
| 14 |
+
[2;36m[14:26:33 03/18][0m[2;36m [0m[34mINFO [0m | >> |=> Num training samples: ]8;id=591396;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=971225;file:///data/jliu/psi/scripts/train.py#192\[2m192[0m]8;;\
|
| 15 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Training dataset size: [1;36m15[0m,[1;36m509[0m ]8;id=603472;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=235431;file:///data/jliu/psi/scripts/train.py#193\[2m193[0m]8;;\
|
| 16 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Val dataset size: [1;36m15[0m,[1;36m509[0m ]8;id=548632;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=929903;file:///data/jliu/psi/scripts/train.py#195\[2m195[0m]8;;\
|
| 17 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0m*[1m][0m Initialize optimizers and schedulers[33m...[0m ]8;id=492873;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=149330;file:///data/jliu/psi/scripts/train.py#198\[2m198[0m]8;;\
|
| 18 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0m*[1m][0m ***** Running training ***** ]8;id=127940;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=657111;file:///data/jliu/psi/scripts/train.py#202\[2m202[0m]8;;\
|
| 19 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Num training examples = [1;36m15509[0m ]8;id=974614;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=572136;file:///data/jliu/psi/scripts/train.py#203\[2m203[0m]8;;\
|
| 20 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Max training Epochs = [1;36m328[0m ]8;id=574775;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=838653;file:///data/jliu/psi/scripts/train.py#204\[2m204[0m]8;;\
|
| 21 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Total optimization steps = [1;36m40000[0m ]8;id=927413;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=421473;file:///data/jliu/psi/scripts/train.py#205\[2m205[0m]8;;\
|
| 22 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Num steps Per Epoch = [1;36m122[0m ]8;id=9727;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=34717;file:///data/jliu/psi/scripts/train.py#206\[2m206[0m]8;;\
|
| 23 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Effective training epochs = [1;36m1311.48[0m ]8;id=842580;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=548288;file:///data/jliu/psi/scripts/train.py#207\[2m207[0m]8;;\
|
| 24 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Global train batch size [1m([0mw. parallel, distributed & accumulation[1m)[0m = [1;36m128[0m ]8;id=272745;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=726293;file:///data/jliu/psi/scripts/train.py#208\[2m208[0m]8;;\
|
| 25 |
+
[2;36m[14:26:34 03/18][0m[2;36m [0m[34mINFO [0m | >> |=> Device train batch size = [1;36m32[0m ]8;id=521307;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=847206;file:///data/jliu/psi/scripts/train.py#209\[2m209[0m]8;;\
|
| 26 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Gradient Accumulation steps = [1;36m1[0m ]8;id=871789;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=304159;file:///data/jliu/psi/scripts/train.py#210\[2m210[0m]8;;\
|
| 27 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Num processes [1m([0mGPUs[1m)[0m = [1;36m4[0m ]8;id=459749;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=722818;file:///data/jliu/psi/scripts/train.py#211\[2m211[0m]8;;\
|
| 28 |
+
[2;36m[14:26:35 03/18][0m[2;36m [0m[34mINFO [0m | >> [1m[[0m*[1m][0m Accelerator runs in: .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m ]8;id=966756;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=191940;file:///data/jliu/psi/scripts/train.py#218\[2m218[0m]8;;\
|
| 29 |
+
Traing steps: 0%| | 0/40000 [00:00<?, ?it/s]/data/jliu/psi/.venv-dp/lib/python3.10/site-packages/transformers/deepspeed.py:24: FutureWarning: transformers.deepspeed module is deprecated and will be removed in a future version. Please import deepspeed modules directly from transformers.integrations
|
| 30 |
+
warnings.warn(
|
| 31 |
+
Traing steps: 25%|██████████████████████▍ | 9999/40000 [1:35:02<2:29:49, 3.34it/s, loss=0.0148, lr=8.7e-05]
|
| 32 |
+
|
| 33 |
+
[2;36m[15:14:00 03/18][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Saving current state to ]8;id=680052;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=279409;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/accelerator.py#3632\[2m3632[0m]8;;\
|
| 34 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_5000 [2m [0m
|
| 35 |
+
[2;36m[15:14:03 03/18][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Model weights saved in ]8;id=898249;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=431290;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 36 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_5000/ [2m [0m
|
| 37 |
+
[2;36m [0m model.safetensors [2m [0m
|
| 38 |
+
[2;36m[15:14:04 03/18][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Optimizer state saved in ]8;id=224;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=473149;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 39 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_5000/ [2m [0m
|
| 40 |
+
[2;36m [0m optimizer.bin [2m [0m
|
| 41 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Scheduler state saved in ]8;id=639771;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=832328;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#129\[2m129[0m]8;;\
|
| 42 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_5000/ [2m [0m
|
| 43 |
+
[2;36m [0m scheduler.bin [2m [0m
|
| 44 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Sampler state for dataloader [1;36m0[0m saved in ]8;id=788377;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=572992;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#146\[2m146[0m]8;;\
|
| 45 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_5000/ [2m [0m
|
| 46 |
+
[2;36m [0m sampler.bin [2m [0m
|
| 47 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Sampler state for dataloader [1;36m1[0m saved in ]8;id=374604;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=162278;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#146\[2m146[0m]8;;\
|
| 48 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_5000/ [2m [0m
|
| 49 |
+
[2;36m [0m sampler_1.bin [2m [0m
|
| 50 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Random states saved in ]8;id=806908;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=971194;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#179\[2m179[0m]8;;\
|
| 51 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_5000/ [2m [0m
|
| 52 |
+
[2;36m [0m random_states_0.pkl [2m [0m
|
| 53 |
+
Saved state to .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_5000
|
| 54 |
+
[2;36m[16:01:38 03/18][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Saving current state to ]8;id=755470;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=983054;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/accelerator.py#3632\[2m3632[0m]8;;\
|
| 55 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_10000 [2m [0m
|
| 56 |
+
[2;36m[16:01:40 03/18][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Model weights saved in ]8;id=616494;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=137590;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 57 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_10000 [2m [0m
|
| 58 |
+
[2;36m [0m /model.safetensors [2m [0m
|
| 59 |
+
[2;36m[16:01:42 03/18][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Optimizer state saved in ]8;id=304800;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=88534;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 60 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_10000 [2m [0m
|
| 61 |
+
[2;36m [0m /optimizer.bin [2m [0m
|
| 62 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Scheduler state saved in ]8;id=141920;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=821494;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#129\[2m129[0m]8;;\
|
| 63 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_10000 [2m [0m
|
| 64 |
+
[2;36m [0m /scheduler.bin [2m [0m
|
| 65 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Sampler state for dataloader [1;36m0[0m saved in ]8;id=532364;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=432054;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#146\[2m146[0m]8;;\
|
| 66 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_10000 [2m [0m
|
| 67 |
+
[2;36m [0m /sampler.bin [2m [0m
|
| 68 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Sampler state for dataloader [1;36m1[0m saved in ]8;id=846104;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=206251;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#146\[2m146[0m]8;;\
|
| 69 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_10000 [2m [0m
|
| 70 |
+
[2;36m [0m /sampler_1.bin [2m [0m
|
| 71 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Random states saved in ]8;id=930895;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=77000;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#179\[2m179[0m]8;;\
|
| 72 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_10000 [2m [0m
|
| 73 |
+
[2;36m [0m /random_states_0.pkl [2m [0m
|
| 74 |
+
Saved state to .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_10000
|
| 75 |
+
[2;36m[16:47:50 03/18][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Saving current state to ]8;id=370569;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=979940;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/accelerator.py#3632\[2m3632[0m]8;;\
|
| 76 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_15000 [2m [0m
|
| 77 |
+
[2;36m[16:47:51 03/18][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Model weights saved in ]8;id=535180;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=499147;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 78 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_15000 [2m [0m
|
| 79 |
+
[2;36m [0m /model.safetensors [2m [0m
|
| 80 |
+
[2;36m[16:47:52 03/18][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Optimizer state saved in ]8;id=341880;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=958353;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 81 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_15000 [2m [0m
|
| 82 |
+
[2;36m [0m /optimizer.bin [2m [0m
|
| 83 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Scheduler state saved in ]8;id=599385;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=648655;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#129\[2m129[0m]8;;\
|
| 84 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_15000 [2m [0m
|
| 85 |
+
[2;36m [0m /scheduler.bin [2m [0m
|
| 86 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Sampler state for dataloader [1;36m0[0m saved in ]8;id=544854;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=217040;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#146\[2m146[0m]8;;\
|
| 87 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_15000 [2m [0m
|
| 88 |
+
[2;36m [0m /sampler.bin [2m [0m
|
| 89 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Sampler state for dataloader [1;36m1[0m saved in ]8;id=627278;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=244660;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#146\[2m146[0m]8;;\
|
| 90 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_15000 [2m [0m
|
| 91 |
+
[2;36m [0m /sampler_1.bin [2m [0m
|
| 92 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Random states saved in ]8;id=418546;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=35811;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#179\[2m179[0m]8;;\
|
| 93 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_15000 [2m [0m
|
| 94 |
+
[2;36m [0m /random_states_0.pkl [2m [0m
|
| 95 |
+
Saved state to .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_15000
|
| 96 |
+
[2;36m[17:33:53 03/18][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Saving current state to ]8;id=47434;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=679719;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/accelerator.py#3632\[2m3632[0m]8;;\
|
| 97 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_20000 [2m [0m
|
| 98 |
+
[2;36m[17:33:55 03/18][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Model weights saved in ]8;id=473768;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=910263;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 99 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_20000 [2m [0m
|
| 100 |
+
[2;36m [0m /model.safetensors [2m [0m
|
| 101 |
+
[2;36m[17:33:56 03/18][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Optimizer state saved in ]8;id=745480;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=711599;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 102 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_20000 [2m [0m
|
| 103 |
+
[2;36m [0m /optimizer.bin [2m [0m
|
| 104 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Scheduler state saved in ]8;id=555678;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=229348;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#129\[2m129[0m]8;;\
|
| 105 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_20000 [2m [0m
|
| 106 |
+
[2;36m [0m /scheduler.bin [2m [0m
|
| 107 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Sampler state for dataloader [1;36m0[0m saved in ]8;id=144411;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=168181;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#146\[2m146[0m]8;;\
|
| 108 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_20000 [2m [0m
|
| 109 |
+
[2;36m [0m /sampler.bin [2m [0m
|
| 110 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Sampler state for dataloader [1;36m1[0m saved in ]8;id=286058;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=446016;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#146\[2m146[0m]8;;\
|
| 111 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_20000 [2m [0m
|
| 112 |
+
[2;36m [0m /sampler_1.bin [2m [0m
|
| 113 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Random states saved in ]8;id=401798;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=319720;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#179\[2m179[0m]8;;\
|
| 114 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_20000 [2m [0m
|
| 115 |
+
[2;36m [0m /random_states_0.pkl [2m [0m
|
| 116 |
+
Saved state to .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_20000
|
| 117 |
+
[2;36m[18:19:28 03/18][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Saving current state to ]8;id=486450;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=521773;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/accelerator.py#3632\[2m3632[0m]8;;\
|
| 118 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_25000 [2m [0m
|
| 119 |
+
[2;36m[18:19:30 03/18][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Model weights saved in ]8;id=490419;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=185558;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 120 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_25000 [2m [0m
|
| 121 |
+
[2;36m [0m /model.safetensors [2m [0m
|
| 122 |
+
[2;36m[18:19:31 03/18][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Optimizer state saved in ]8;id=125939;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=552863;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 123 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_25000 [2m [0m
|
| 124 |
+
[2;36m [0m /optimizer.bin [2m [0m
|
| 125 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Scheduler state saved in ]8;id=855797;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=99159;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#129\[2m129[0m]8;;\
|
| 126 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_25000 [2m [0m
|
| 127 |
+
[2;36m [0m /scheduler.bin [2m [0m
|
| 128 |
+
[2;36m[18:19:32 03/18][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Sampler state for dataloader [1;36m0[0m saved in ]8;id=999999;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=97642;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#146\[2m146[0m]8;;\
|
| 129 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_25000 [2m [0m
|
| 130 |
+
[2;36m [0m /sampler.bin [2m [0m
|
| 131 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Sampler state for dataloader [1;36m1[0m saved in ]8;id=427821;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=571536;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#146\[2m146[0m]8;;\
|
| 132 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_25000 [2m [0m
|
| 133 |
+
[2;36m [0m /sampler_1.bin [2m [0m
|
| 134 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Random states saved in ]8;id=702964;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=191314;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#179\[2m179[0m]8;;\
|
| 135 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_25000 [2m [0m
|
| 136 |
+
[2;36m [0m /random_states_0.pkl [2m [0m
|
| 137 |
+
Saved state to .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_25000
|
| 138 |
+
[2;36m[19:04:51 03/18][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Saving current state to ]8;id=368785;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=589302;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/accelerator.py#3632\[2m3632[0m]8;;\
|
| 139 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_30000 [2m [0m
|
| 140 |
+
[2;36m[19:04:52 03/18][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Model weights saved in ]8;id=227061;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=115021;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 141 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_30000 [2m [0m
|
| 142 |
+
[2;36m [0m /model.safetensors [2m [0m
|
| 143 |
+
[2;36m[19:04:53 03/18][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Optimizer state saved in ]8;id=787738;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=595634;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 144 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_30000 [2m [0m
|
| 145 |
+
[2;36m [0m /optimizer.bin [2m [0m
|
| 146 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Scheduler state saved in ]8;id=590171;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=281123;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#129\[2m129[0m]8;;\
|
| 147 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_30000 [2m [0m
|
| 148 |
+
[2;36m [0m /scheduler.bin [2m [0m
|
| 149 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Sampler state for dataloader [1;36m0[0m saved in ]8;id=819504;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=276522;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#146\[2m146[0m]8;;\
|
| 150 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_30000 [2m [0m
|
| 151 |
+
[2;36m [0m /sampler.bin [2m [0m
|
| 152 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Sampler state for dataloader [1;36m1[0m saved in ]8;id=717799;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=63361;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#146\[2m146[0m]8;;\
|
| 153 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_30000 [2m [0m
|
| 154 |
+
[2;36m [0m /sampler_1.bin [2m [0m
|
| 155 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Random states saved in ]8;id=194505;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=677321;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#179\[2m179[0m]8;;\
|
| 156 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_30000 [2m [0m
|
| 157 |
+
[2;36m [0m /random_states_0.pkl [2m [0m
|
| 158 |
+
Saved state to .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_30000
|
| 159 |
+
[2;36m[19:49:58 03/18][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Saving current state to ]8;id=290231;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=616676;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/accelerator.py#3632\[2m3632[0m]8;;\
|
| 160 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_35000 [2m [0m
|
| 161 |
+
[2;36m[19:50:00 03/18][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Model weights saved in ]8;id=329883;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=169230;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 162 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_35000 [2m [0m
|
| 163 |
+
[2;36m [0m /model.safetensors [2m [0m
|
| 164 |
+
[2;36m[19:50:01 03/18][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Optimizer state saved in ]8;id=431638;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=908584;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 165 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_35000 [2m [0m
|
| 166 |
+
[2;36m [0m /optimizer.bin [2m [0m
|
| 167 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Scheduler state saved in ]8;id=434664;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=16179;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#129\[2m129[0m]8;;\
|
| 168 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_35000 [2m [0m
|
| 169 |
+
[2;36m [0m /scheduler.bin [2m [0m
|
| 170 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Sampler state for dataloader [1;36m0[0m saved in ]8;id=324973;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=128996;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#146\[2m146[0m]8;;\
|
| 171 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_35000 [2m [0m
|
| 172 |
+
[2;36m [0m /sampler.bin [2m [0m
|
| 173 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Sampler state for dataloader [1;36m1[0m saved in ]8;id=981583;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=899091;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#146\[2m146[0m]8;;\
|
| 174 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_35000 [2m [0m
|
| 175 |
+
[2;36m [0m /sampler_1.bin [2m [0m
|
| 176 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Random states saved in ]8;id=636753;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=656123;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#179\[2m179[0m]8;;\
|
| 177 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_35000 [2m [0m
|
| 178 |
+
[2;36m [0m /random_states_0.pkl [2m [0m
|
| 179 |
+
Saved state to .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_35000
|
| 180 |
+
[2;36m[20:24:02 03/18][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Saving current state to ]8;id=969664;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=423824;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/accelerator.py#3632\[2m3632[0m]8;;\
|
| 181 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_40000 [2m [0m
|
| 182 |
+
[2;36m[20:24:04 03/18][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Model weights saved in ]8;id=485144;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=446823;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 183 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_40000 [2m [0m
|
| 184 |
+
[2;36m [0m /model.safetensors [2m [0m
|
| 185 |
+
[2;36m[20:24:05 03/18][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Optimizer state saved in ]8;id=392321;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=903139;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 186 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_40000 [2m [0m
|
| 187 |
+
[2;36m [0m /optimizer.bin [2m [0m
|
| 188 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Scheduler state saved in ]8;id=407945;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=306059;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#129\[2m129[0m]8;;\
|
| 189 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_40000 [2m [0m
|
| 190 |
+
[2;36m [0m /scheduler.bin [2m [0m
|
| 191 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Sampler state for dataloader [1;36m0[0m saved in ]8;id=771109;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=749912;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#146\[2m146[0m]8;;\
|
| 192 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_40000 [2m [0m
|
| 193 |
+
[2;36m [0m /sampler.bin [2m [0m
|
| 194 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Sampler state for dataloader [1;36m1[0m saved in ]8;id=729075;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=346604;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#146\[2m146[0m]8;;\
|
| 195 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_40000 [2m [0m
|
| 196 |
+
[2;36m [0m /sampler_1.bin [2m [0m
|
| 197 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Random states saved in ]8;id=960350;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=128256;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#179\[2m179[0m]8;;\
|
| 198 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_40000 [2m [0m
|
| 199 |
+
[2;36m [0m /random_states_0.pkl [2m [0m
|
| 200 |
+
Saved state to .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000
|
| 201 |
+
Training has reached maximum steps.
|
| 202 |
+
[2;36m[20:24:35 03/18][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Saving current state to ]8;id=614052;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=946822;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/accelerator.py#3632\[2m3632[0m]8;;\
|
| 203 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_39999 [2m [0m
|
| 204 |
+
[2;36m[20:24:37 03/18][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Model weights saved in ]8;id=588107;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=58872;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 205 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_39999 [2m [0m
|
| 206 |
+
[2;36m [0m /model.safetensors [2m [0m
|
| 207 |
+
[2;36m[20:24:38 03/18][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Optimizer state saved in ]8;id=128644;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=124028;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 208 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_39999 [2m [0m
|
| 209 |
+
[2;36m [0m /optimizer.bin [2m [0m
|
| 210 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Scheduler state saved in ]8;id=625950;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=679365;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#129\[2m129[0m]8;;\
|
| 211 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_39999 [2m [0m
|
| 212 |
+
[2;36m [0m /scheduler.bin [2m [0m
|
| 213 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Sampler state for dataloader [1;36m0[0m saved in ]8;id=606210;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=132432;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#146\[2m146[0m]8;;\
|
| 214 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_39999 [2m [0m
|
| 215 |
+
[2;36m [0m /sampler.bin [2m [0m
|
| 216 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Sampler state for dataloader [1;36m1[0m saved in ]8;id=250930;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=814323;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#146\[2m146[0m]8;;\
|
| 217 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_39999 [2m [0m
|
| 218 |
+
[2;36m [0m /sampler_1.bin [2m [0m
|
| 219 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Random states saved in ]8;id=131290;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=192379;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#179\[2m179[0m]8;;\
|
| 220 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2603181426[0m/checkpoints/ckpt_39999 [2m [0m
|
| 221 |
+
[2;36m [0m /random_states_0.pkl [2m [0m
|
| 222 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0m*[1m][0m Finalized DP Trainer. Epoch losses: [1m[[0m[1;36m0.0[0m, [1;36m1.0039703845977783[0m, [1;36m0.8921335339546204[0m, ]8;id=567626;file:///data/jliu/psi/src/psi/trainers/diffusion_policy_g1.py\[2mdiffusion_policy_g1.py[0m]8;;\[2m:[0m]8;id=716966;file:///data/jliu/psi/src/psi/trainers/diffusion_policy_g1.py#360\[2m360[0m]8;;\
|
| 223 |
+
[2;36m [0m [1;36m0.6254487037658691[0m, [1;36m0.4844936728477478[0m, [1;36m0.3499990701675415[0m[1m][0m [2m [0m
|
| 224 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0m*[1m][0m Happy Ending! ]8;id=154857;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=239343;file:///data/jliu/psi/scripts/train.py#310\[2m310[0m]8;;\
|
dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142630-q4pxxlgy/files/requirements.txt
ADDED
|
@@ -0,0 +1,205 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
tifffile==2025.5.10
|
| 2 |
+
mpmath==1.3.0
|
| 3 |
+
pynput==1.8.1
|
| 4 |
+
llvmlite==0.46.0
|
| 5 |
+
exceptiongroup==1.3.0
|
| 6 |
+
sympy==1.14.0
|
| 7 |
+
aiohttp-cors==0.8.1
|
| 8 |
+
soupsieve==2.8
|
| 9 |
+
Farama-Notifications==0.0.4
|
| 10 |
+
numpydantic==1.6.7
|
| 11 |
+
uvicorn==0.38.0
|
| 12 |
+
waterbear==2.6.8
|
| 13 |
+
nvidia-cudnn-cu12==9.10.2.21
|
| 14 |
+
transformers==4.44.2
|
| 15 |
+
cmake==4.2.3
|
| 16 |
+
xxhash==3.5.0
|
| 17 |
+
annotated-doc==0.0.4
|
| 18 |
+
jsonlines==4.0.0
|
| 19 |
+
nvidia-cuda-nvrtc-cu12==12.8.93
|
| 20 |
+
pydantic==2.10.6
|
| 21 |
+
triton==3.4.0
|
| 22 |
+
nvidia-cufft-cu12==11.3.3.83
|
| 23 |
+
multidict==6.6.4
|
| 24 |
+
setuptools==82.0.1
|
| 25 |
+
shtab==1.7.2
|
| 26 |
+
pycollada==0.9.2
|
| 27 |
+
beautifulsoup4==4.14.2
|
| 28 |
+
msgspec==0.19.0
|
| 29 |
+
huggingface-hub==0.35.3
|
| 30 |
+
pydantic_core==2.27.2
|
| 31 |
+
iniconfig==2.3.0
|
| 32 |
+
zarr==2.18.3
|
| 33 |
+
pytz==2025.2
|
| 34 |
+
attrs==25.3.0
|
| 35 |
+
requests==2.32.5
|
| 36 |
+
hf_transfer==0.1.9
|
| 37 |
+
numba==0.64.0
|
| 38 |
+
einx==0.3.0
|
| 39 |
+
scipy==1.15.3
|
| 40 |
+
transforms3d==0.4.2
|
| 41 |
+
pyzmq==27.1.0
|
| 42 |
+
aiohappyeyeballs==2.6.1
|
| 43 |
+
pymunk==7.2.0
|
| 44 |
+
six==1.17.0
|
| 45 |
+
expandvars==1.1.2
|
| 46 |
+
prompt_toolkit==3.0.52
|
| 47 |
+
wcwidth==0.6.0
|
| 48 |
+
tomli==2.4.0
|
| 49 |
+
ruamel.yaml.clib==0.2.14
|
| 50 |
+
multiprocess==0.70.16
|
| 51 |
+
vhacdx==0.0.9
|
| 52 |
+
nvidia-cusparse-cu12==12.5.8.93
|
| 53 |
+
aiosignal==1.4.0
|
| 54 |
+
itsdangerous==2.2.0
|
| 55 |
+
av==17.0.0
|
| 56 |
+
nvidia-nccl-cu12==2.27.3
|
| 57 |
+
httpx==0.28.1
|
| 58 |
+
asciitree==0.3.3
|
| 59 |
+
platformdirs==4.4.0
|
| 60 |
+
sentry-sdk==2.39.0
|
| 61 |
+
draccus==0.11.5
|
| 62 |
+
nvidia-curand-cu12==10.3.9.90
|
| 63 |
+
plotly==6.2.0
|
| 64 |
+
jsonschema==4.25.1
|
| 65 |
+
fastapi==0.119.1
|
| 66 |
+
nvidia-nvjitlink-cu12==12.8.93
|
| 67 |
+
fsspec==2025.3.0
|
| 68 |
+
MarkupSafe==3.0.3
|
| 69 |
+
pydantic-yaml==1.6.0
|
| 70 |
+
h11==0.16.0
|
| 71 |
+
typer==0.24.1
|
| 72 |
+
mypy_extensions==1.1.0
|
| 73 |
+
pytest==9.0.2
|
| 74 |
+
h5py==3.14.0
|
| 75 |
+
python-xlib==0.33
|
| 76 |
+
lazy_loader==0.4
|
| 77 |
+
einops==0.8.1
|
| 78 |
+
albumentations==1.4.18
|
| 79 |
+
params_proto==2.13.2
|
| 80 |
+
fasteners==0.20
|
| 81 |
+
psutil==7.1.0
|
| 82 |
+
starlette==0.48.0
|
| 83 |
+
anyio==4.11.0
|
| 84 |
+
gdown==5.2.0
|
| 85 |
+
charset-normalizer==3.4.3
|
| 86 |
+
tyro==0.9.32
|
| 87 |
+
filelock==3.19.1
|
| 88 |
+
websockets==15.0.1
|
| 89 |
+
accelerate==1.13.0
|
| 90 |
+
python-dotenv==1.2.1
|
| 91 |
+
orderly-set==5.5.0
|
| 92 |
+
inquirerpy==0.3.4
|
| 93 |
+
urllib3==2.5.0
|
| 94 |
+
PyYAML==6.0.3
|
| 95 |
+
rerun-sdk==0.23.1
|
| 96 |
+
antlr4-python3-runtime==4.9.3
|
| 97 |
+
mdurl==0.1.2
|
| 98 |
+
omegaconf==2.3.0
|
| 99 |
+
referencing==0.37.0
|
| 100 |
+
pycparser==3.0
|
| 101 |
+
pluggy==1.6.0
|
| 102 |
+
torchvision==0.23.0
|
| 103 |
+
docstring_parser==0.17.0
|
| 104 |
+
wandb==0.25.1
|
| 105 |
+
numpy==1.26.4
|
| 106 |
+
GitPython==3.1.45
|
| 107 |
+
opencv-python-headless==4.11.0.86
|
| 108 |
+
cffi==2.0.0
|
| 109 |
+
yourdfpy==0.0.58
|
| 110 |
+
async-timeout==4.0.3
|
| 111 |
+
shapely==2.1.2
|
| 112 |
+
frozenlist==1.7.0
|
| 113 |
+
simplejpeg==1.9.0
|
| 114 |
+
nvidia-cuda-cupti-cu12==12.8.90
|
| 115 |
+
Pygments==2.19.2
|
| 116 |
+
rtree==1.4.1
|
| 117 |
+
gitdb==4.0.12
|
| 118 |
+
colorlog==6.10.1
|
| 119 |
+
Werkzeug==3.1.6
|
| 120 |
+
zipp==3.23.0
|
| 121 |
+
albucore==0.0.17
|
| 122 |
+
imageio==2.34.2
|
| 123 |
+
frozendict==2.4.6
|
| 124 |
+
jsonschema-specifications==2025.9.1
|
| 125 |
+
rpds-py==0.28.0
|
| 126 |
+
tokenizers==0.19.1
|
| 127 |
+
blinker==1.9.0
|
| 128 |
+
argparse==1.4.0
|
| 129 |
+
nvidia-nvtx-cu12==12.8.90
|
| 130 |
+
msgpack==1.1.1
|
| 131 |
+
smmap==5.0.2
|
| 132 |
+
packaging==25.0
|
| 133 |
+
embreex==2.17.7.post7
|
| 134 |
+
cloudpickle==3.1.2
|
| 135 |
+
vuer==0.0.68
|
| 136 |
+
propcache==0.3.2
|
| 137 |
+
torchaudio==2.8.0
|
| 138 |
+
pfzy==0.3.4
|
| 139 |
+
dm-tree==0.1.8
|
| 140 |
+
nvidia-cuda-runtime-cu12==12.8.90
|
| 141 |
+
datasets==3.6.0
|
| 142 |
+
numcodecs==0.13.1
|
| 143 |
+
sniffio==1.3.1
|
| 144 |
+
Jinja2==3.1.6
|
| 145 |
+
narwhals==2.6.0
|
| 146 |
+
gymnasium==0.29.1
|
| 147 |
+
yarl==1.20.1
|
| 148 |
+
typeguard==4.4.4
|
| 149 |
+
termcolor==3.1.0
|
| 150 |
+
importlib_metadata==8.7.1
|
| 151 |
+
eval_type_backport==0.2.2
|
| 152 |
+
certifi==2025.8.3
|
| 153 |
+
tzdata==2025.2
|
| 154 |
+
typing_extensions==4.15.0
|
| 155 |
+
viser==1.0.15
|
| 156 |
+
typing-inspect==0.9.0
|
| 157 |
+
torch==2.8.0
|
| 158 |
+
annotated-types==0.7.0
|
| 159 |
+
svg.path==7.0
|
| 160 |
+
mergedeep==1.3.4
|
| 161 |
+
psi==0.0.0
|
| 162 |
+
deepdiff==8.6.1
|
| 163 |
+
nvidia-cufile-cu12==1.13.1.3
|
| 164 |
+
toml==0.10.2
|
| 165 |
+
click==8.3.0
|
| 166 |
+
diffusers==0.30.2
|
| 167 |
+
dotenv==0.9.9
|
| 168 |
+
trimesh==4.8.3
|
| 169 |
+
nvidia-cusparselt-cu12==0.7.1
|
| 170 |
+
pyyaml-include==1.4.1
|
| 171 |
+
scikit-image==0.25.2
|
| 172 |
+
PySocks==1.7.1
|
| 173 |
+
aiohttp==3.10.5
|
| 174 |
+
regex==2026.2.28
|
| 175 |
+
opencv-python==4.11.0.86
|
| 176 |
+
nodeenv==1.9.1
|
| 177 |
+
torchcodec==0.6.0
|
| 178 |
+
protobuf==6.33.2
|
| 179 |
+
pillow==11.3.0
|
| 180 |
+
nvidia-cusolver-cu12==11.7.3.90
|
| 181 |
+
idna==3.10
|
| 182 |
+
Flask==3.1.3
|
| 183 |
+
lerobot==0.1.0
|
| 184 |
+
safetensors==0.7.0
|
| 185 |
+
killport==1.2.0
|
| 186 |
+
hf-xet==1.1.10
|
| 187 |
+
nvidia-cublas-cu12==12.8.4.1
|
| 188 |
+
imageio-ffmpeg==0.6.0
|
| 189 |
+
httpcore==1.0.9
|
| 190 |
+
networkx==3.4.2
|
| 191 |
+
ruamel.yaml==0.18.15
|
| 192 |
+
manifold3d==3.2.1
|
| 193 |
+
shellingham==1.5.4
|
| 194 |
+
mapbox_earcut==1.0.3
|
| 195 |
+
pyarrow==21.0.0
|
| 196 |
+
dill==0.3.8
|
| 197 |
+
markdown-it-py==4.0.0
|
| 198 |
+
tqdm==4.67.1
|
| 199 |
+
argcomplete==3.6.2
|
| 200 |
+
pandas==2.3.3
|
| 201 |
+
lxml==6.0.2
|
| 202 |
+
python-dateutil==2.9.0.post0
|
| 203 |
+
evdev==1.9.3
|
| 204 |
+
rich==14.1.0
|
| 205 |
+
psi==0.0.0
|
dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142630-q4pxxlgy/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-6.8.0-100-generic-x86_64-with-glibc2.39",
|
| 3 |
+
"python": "CPython 3.10.20",
|
| 4 |
+
"startedAt": "2026-03-18T14:26:30.762673Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"simple_dp_config",
|
| 7 |
+
"--seed=2026",
|
| 8 |
+
"--exp=g1wholebodybendpick-v0",
|
| 9 |
+
"--train.name=diffusion-policy-g1",
|
| 10 |
+
"--log.report-to=wandb",
|
| 11 |
+
"--train.data_parallel=ddp",
|
| 12 |
+
"--train.mixed_precision=bf16",
|
| 13 |
+
"--train.train-batch-size=32",
|
| 14 |
+
"--train.warmup-steps=1000",
|
| 15 |
+
"--train.warmup-ratio=None",
|
| 16 |
+
"--train.checkpointing-steps=5000",
|
| 17 |
+
"--train.validation_steps=500",
|
| 18 |
+
"--train.val_num_batches=20",
|
| 19 |
+
"--train.gradient_accumulation_steps=1",
|
| 20 |
+
"--train.max-training-steps=40000",
|
| 21 |
+
"--train.learning-rate=1e-4",
|
| 22 |
+
"--train.max-grad-norm=1.0",
|
| 23 |
+
"--train.lr_scheduler_kwargs.weight_decay=1e-6",
|
| 24 |
+
"--train.lr_scheduler_kwargs.betas",
|
| 25 |
+
"0.95",
|
| 26 |
+
"0.999",
|
| 27 |
+
"--train.lr_scheduler_type=cosine",
|
| 28 |
+
"--data.root_dir=/data/jliu/data",
|
| 29 |
+
"--data.train-repo-ids=G1WholebodyBendPick-v0",
|
| 30 |
+
"--data.transform.repack.action_chunk_size=16",
|
| 31 |
+
"--data.transform.repack.pad-action-dim=36",
|
| 32 |
+
"--data.transform.repack.pad-state-dim=36",
|
| 33 |
+
"--data.transform.field.stat-path=meta/stats_psi0.json",
|
| 34 |
+
"--data.transform.field.stat-action-key=action",
|
| 35 |
+
"--data.transform.field.stat-state-key=states",
|
| 36 |
+
"--data.transform.field.normalize-state",
|
| 37 |
+
"--data.transform.field.action-norm-type=bounds",
|
| 38 |
+
"--data.transform.field.pad-action-dim=36",
|
| 39 |
+
"--data.transform.field.pad-state-dim=36",
|
| 40 |
+
"--data.transform.model.img-aug",
|
| 41 |
+
"--model.action-chunk-size=16",
|
| 42 |
+
"--model.action-dim=36",
|
| 43 |
+
"--model.obs-dim=36"
|
| 44 |
+
],
|
| 45 |
+
"program": "/data/jliu/psi/scripts/train.py",
|
| 46 |
+
"codePath": "scripts/train.py",
|
| 47 |
+
"codePathLocal": "scripts/train.py",
|
| 48 |
+
"git": {
|
| 49 |
+
"remote": "https://github.com/songlin/psi.git",
|
| 50 |
+
"commit": "04614628ecb677f5f278e2d31c1103aed8127f26"
|
| 51 |
+
},
|
| 52 |
+
"email": "jliu530@163.com",
|
| 53 |
+
"root": "/data/jliu/psi/.runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426",
|
| 54 |
+
"host": "nebula99",
|
| 55 |
+
"executable": "/data/jliu/psi/.venv-dp/bin/python3",
|
| 56 |
+
"cpu_count": 128,
|
| 57 |
+
"cpu_count_logical": 128,
|
| 58 |
+
"gpu": "NVIDIA A100-SXM4-80GB",
|
| 59 |
+
"gpu_count": 8,
|
| 60 |
+
"disk": {
|
| 61 |
+
"/": {
|
| 62 |
+
"total": "105089261568",
|
| 63 |
+
"used": "99538145280"
|
| 64 |
+
}
|
| 65 |
+
},
|
| 66 |
+
"memory": {
|
| 67 |
+
"total": "1623177744384"
|
| 68 |
+
},
|
| 69 |
+
"gpu_nvidia": [
|
| 70 |
+
{
|
| 71 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 72 |
+
"memoryTotal": "85899345920",
|
| 73 |
+
"cudaCores": 6912,
|
| 74 |
+
"architecture": "Ampere",
|
| 75 |
+
"uuid": "GPU-37370f00-90f5-4e03-f763-7a35649e6783"
|
| 76 |
+
},
|
| 77 |
+
{
|
| 78 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 79 |
+
"memoryTotal": "85899345920",
|
| 80 |
+
"cudaCores": 6912,
|
| 81 |
+
"architecture": "Ampere",
|
| 82 |
+
"uuid": "GPU-d4b40383-188b-cc16-9180-20c1a71a777f"
|
| 83 |
+
},
|
| 84 |
+
{
|
| 85 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 86 |
+
"memoryTotal": "85899345920",
|
| 87 |
+
"cudaCores": 6912,
|
| 88 |
+
"architecture": "Ampere",
|
| 89 |
+
"uuid": "GPU-0f3d827c-66cf-04da-b182-4ed0414a2549"
|
| 90 |
+
},
|
| 91 |
+
{
|
| 92 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 93 |
+
"memoryTotal": "85899345920",
|
| 94 |
+
"cudaCores": 6912,
|
| 95 |
+
"architecture": "Ampere",
|
| 96 |
+
"uuid": "GPU-164a49bb-43d3-2250-59e2-1b40eee0757f"
|
| 97 |
+
},
|
| 98 |
+
{
|
| 99 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 100 |
+
"memoryTotal": "85899345920",
|
| 101 |
+
"cudaCores": 6912,
|
| 102 |
+
"architecture": "Ampere",
|
| 103 |
+
"uuid": "GPU-c0471f40-ae19-5371-a1af-da406ca30f83"
|
| 104 |
+
},
|
| 105 |
+
{
|
| 106 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 107 |
+
"memoryTotal": "85899345920",
|
| 108 |
+
"cudaCores": 6912,
|
| 109 |
+
"architecture": "Ampere",
|
| 110 |
+
"uuid": "GPU-d9040f59-13d3-6c37-4ea0-c6c751424527"
|
| 111 |
+
},
|
| 112 |
+
{
|
| 113 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 114 |
+
"memoryTotal": "85899345920",
|
| 115 |
+
"cudaCores": 6912,
|
| 116 |
+
"architecture": "Ampere",
|
| 117 |
+
"uuid": "GPU-f834d0e8-a118-24c7-5461-30fce111bea6"
|
| 118 |
+
},
|
| 119 |
+
{
|
| 120 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 121 |
+
"memoryTotal": "85899345920",
|
| 122 |
+
"cudaCores": 6912,
|
| 123 |
+
"architecture": "Ampere",
|
| 124 |
+
"uuid": "GPU-6e687915-3247-735c-6641-544540804e79"
|
| 125 |
+
}
|
| 126 |
+
],
|
| 127 |
+
"cudaVersion": "12.9",
|
| 128 |
+
"writerId": "9to9pktd3ndtvl9vuv59f4rwekbfrlue"
|
| 129 |
+
}
|
dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142630-q4pxxlgy/files/wandb-summary.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"_step":40000,"val/denorm_err_l1_hand_joints":0.0031825141049921513,"val/denorm_err_l1_arm_joints":0.001931339385919273,"train/loss":0.0009651134605519474,"val/bc_loss":0.0009418176487088203,"_wandb":{"runtime":21486},"train/epoch":330,"val/denorm_err_l1_torso_vyaw":0,"val/denorm_err_l1_height":0.0013415103312581778,"_runtime":21486.385705629,"val/denorm_err_l1_rpy":0.0008791955769993365,"val/denorm_err_l1_torso_vx":0,"val/denorm_err_l1_torso_vy":0,"_timestamp":1.7738654759254735e+09,"val/denorm_err_l1_torso_target_yaw":0}
|
dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142630-q4pxxlgy/logs/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142630-q4pxxlgy/logs/debug.log
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-03-18 14:26:30,764 INFO MainThread:279364 [wandb_setup.py:_flush():81] Current SDK version is 0.25.1
|
| 2 |
+
2026-03-18 14:26:30,764 INFO MainThread:279364 [wandb_setup.py:_flush():81] Configure stats pid to 279364
|
| 3 |
+
2026-03-18 14:26:30,764 INFO MainThread:279364 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-03-18 14:26:30,765 INFO MainThread:279364 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data/jliu/psi/.runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142630-q4pxxlgy/logs/debug.log
|
| 5 |
+
2026-03-18 14:26:30,765 INFO MainThread:279364 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data/jliu/psi/.runs/diffusion-policy-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142630-q4pxxlgy/logs/debug-internal.log
|
| 6 |
+
2026-03-18 14:26:30,765 INFO MainThread:279364 [wandb_init.py:init():844] calling init triggers
|
| 7 |
+
2026-03-18 14:26:30,765 INFO MainThread:279364 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'_wandb': {}}
|
| 9 |
+
2026-03-18 14:26:30,765 INFO MainThread:279364 [wandb_init.py:init():892] starting backend
|
| 10 |
+
2026-03-18 14:26:31,018 INFO MainThread:279364 [wandb_init.py:init():895] sending inform_init request
|
| 11 |
+
2026-03-18 14:26:31,028 INFO MainThread:279364 [wandb_init.py:init():903] backend started and connected
|
| 12 |
+
2026-03-18 14:26:31,029 INFO MainThread:279364 [wandb_init.py:init():973] updated telemetry
|
| 13 |
+
2026-03-18 14:26:31,035 INFO MainThread:279364 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-03-18 14:26:32,251 INFO MainThread:279364 [wandb_init.py:init():1042] starting run threads in backend
|
| 15 |
+
2026-03-18 14:26:32,404 INFO MainThread:279364 [wandb_run.py:_console_start():2524] atexit reg
|
| 16 |
+
2026-03-18 14:26:32,404 INFO MainThread:279364 [wandb_run.py:_redirect():2373] redirect: wrap_raw
|
| 17 |
+
2026-03-18 14:26:32,404 INFO MainThread:279364 [wandb_run.py:_redirect():2442] Wrapping output streams.
|
| 18 |
+
2026-03-18 14:26:32,404 INFO MainThread:279364 [wandb_run.py:_redirect():2465] Redirects installed.
|
| 19 |
+
2026-03-18 14:26:32,409 INFO MainThread:279364 [wandb_init.py:init():1082] run started, returning control to user process
|
| 20 |
+
2026-03-18 14:26:32,411 INFO MainThread:279364 [wandb_run.py:_config_callback():1403] config_cb None None {'exp': 'g1wholebodybendpick-v0', 'seed': 2026, 'auto_tag_run': False, 'eval': False, 'debug': False, 'timestamp': '2603181426', 'log': {'logging_dir': 'logs', 'report_to': 'wandb', 'log_freq': 100}, 'wandb': {'project': 'psi', 'entity': 'jliu530-soochow-university', 'group': None, 'id': None, 'name': None, 'resume': 'allow'}, 'train': {'num_workers': 8, 'overfit_single_batch': False, 'name': 'diffusion-policy-g1', 'resume_from_checkpoint': None, 'skip_resumed_steps': False, 'hf_token': '.hf_token', 'lora': False, 'output_dir': '.runs', 'gradient_accumulation_steps': 1, 'mixed_precision': 'bf16', 'max_grad_norm': 1.0, 'optimizer_foreach': None, 'train_batch_size': 32, 'val_batch_size': 16, 'val_num_batches': 20, 'checkpointing_steps': 5000, 'max_checkpoints_to_keep': None, 'validation_steps': 500, 'learning_rate': 0.0001, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {'betas': [0.95, 0.999], 'weight_decay': 1e-06, 'eps': 1e-08}, 'scheduler_specific_kwargs': {}, 'data_parallel': 'ddp', 'sharding_strategy': 'full-shard', 'deepspeed_config': '/data/jliu/psi/src/InternVLA/config/deepseeds/zero3.json', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': True, 'max_training_steps': 40000, 'num_train_epochs': None, 'warmup_steps': 1000, 'warmup_ratio': None}, 'data': {'transform': {'repack': {'dataset_name': 'simple', 'num_past_frames': 0, 'action_chunk_size': 16, 'pad_action_dim': 36, 'pad_state_dim': 36}, 'model': {'resize': {'size': [256, 480]}, 'color_jitter': {'brightness': 0.2, 'contrast': [0.8, 1.2], 'saturation': [0.8, 1.2], 'hue': 0.05}, 'center_crop': {'size': [224, 224]}, 'normalize': {'mean': [0.485, 0.456, 0.406], 'std': [0.229, 0.224, 0.225]}, 'img_aug': True}, 'field': {'stat_path': 'meta/stats_psi0.json', 'action_norm_type': 'bounds', 'stat_action_key': 'action', 'stat_state_key': 'states', 'use_norm_mask': False, 'action_norm_masks': [True, True, True, True, True, True, False], 'action_min': [-0.13059291243553162, -0.09108058363199234, -0.0024844733998179436, -0.20733775198459625, -0.15850023925304413, -0.17450474202632904, -0.2997315526008606, -0.015391111373901367, -0.34571564197540283, -0.4991437792778015, 0.0, 0.0, 0.0, 0.0, -0.1015840545296669, -0.06647031009197235, -0.16578954458236694, -0.14477218687534332, -0.3665394186973572, -0.28364259004592896, -0.1775387078523636, -0.48419490456581116, -0.7551082968711853, -0.2692946195602417, -0.03164339065551758, -3.876500704791397e-05, -0.3909206688404083, 0.0, -0.04351663216948509, -0.014203650876879692, -0.049649015069007874, 0.44999998807907104, 0.0, 0.0, 0.0, 0.0], 'action_max': [0.08620641380548477, 0.13058121502399445, 0.22948147356510162, 0.020551620051264763, 0.005824880674481392, 0.010019193403422832, 8.43817247186962e-07, 0.39566752314567566, 0.0, 0.0, 0.4860266447067261, 1.0467392206192017, 0.6470075845718384, 0.8298009037971497, 0.03516175225377083, 0.11019192636013031, 0.04779902100563049, 0.12850724160671234, 3.8688118365826085e-05, 0.0012142359046265483, 3.3343669201713055e-05, 0.002679983852431178, 0.00041063950629904866, 0.1973484456539154, 0.2633756697177887, 0.34943076968193054, 0.0012102096807211637, 0.8342975974082947, 0.31870752573013306, 0.45533719658851624, 0.15729404985904694, 0.75, 0.0, 0.0, 0.0, 0.0], 'state_min': [-0.13899999856948853, -0.09099991619586945, -5.989517215532203e-11, -0.20900000631809235, -0.1589999943971634, -0.210999995470047, -0.3009999990463257, -0.01600000075995922, -0.1860000044107437, -0.6940000057220459, 0.0, 0.0, 0.0, 0.0, -0.1019991859793663, -0.06899992376565933, -0.16899999976158142, -0.14499999582767487, -0.3709999918937683, -0.28700000047683716, -0.17800045013427734, -0.4869999885559082, -0.7599999904632568, -0.27300000190734863, -0.029999999329447746, 0.0, -0.39100033044815063, -0.0010000000474974513, 0.0, -0.15000000596046448, 0.0, 0.44999998807907104, 0.0, 0.0, 0.0, 0.0], 'state_max': [0.0860000029206276, 0.2720000147819519, 0.23100000619888306, 0.0, 6.510182259944486e-08, 0.0, 0.0, 0.5550000071525574, 0.02100004442036152, 0.0, 0.5429999828338623, 1.13100004196167, 0.5770000219345093, 0.9580000042915344, 0.07141251862049103, 0.10899999737739563, 0.04699999839067459, 0.13600000739097595, 0.0, 0.003000000026077032, 0.0, 0.009999999776482582, 0.0020000000949949026, 0.2029999941587448, 0.2759999930858612, 0.3499999940395355, 0.003000000026077032, 0.8370000123977661, 0.0, 0.0, 0.0, 0.75, 0.0, 0.0, 0.0, 0.0], 'normalize_state': True, 'pad_action_dim': 36, 'pad_state_dim': 36}}, 'root_dir': '/data/jliu/data', 'train_repo_ids': ['G1WholebodyBendPick-v0'], 'val_repo_ids': ['G1WholebodyBendPick-v0']}, 'model': {'num_diffusion_iters': 100, 'action_chunk_size': 16, 'num_cameras': 1, 'share_vision_encoder': False, 'obs_horizon': 1, 'action_exec_horizon': 6, 'action_dim': 36, 'obs_dim': 36}, 'environment_variables': {'OMP_NUM_THREADS': '32', 'HF_HOME': '/data/cache', 'TORCH_HOME': '/data/cache', 'HF_TOKEN': 'hf_...TiKa', 'HF_LEROBOT_HOME': '/data/data/lerobot', 'WE_HOME': 'Not Set', 'DATA_HOME': '/data/data', 'UV_CACHE_DIR': '/data/cache', 'WANDB_API_KEY': '90e...5c06', 'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION': 'python', 'CUDA_VISIBLE_DEVICES': '0,1,2,3', 'WORLD_SIZE': '4', 'LOCAL_WORLD_SIZE': '4', 'RANK': '0', 'LOCAL_RANK': '0', 'MASTER_ADDR': 'nebula99', 'MASTER_PORT': '41289'}}
|
| 21 |
+
2026-03-18 20:24:38,632 INFO MainThread:279364 [wandb_run.py:_finish():2291] finishing run jliu530-soochow-university/psi/q4pxxlgy
|
| 22 |
+
2026-03-18 20:24:38,634 INFO MainThread:279364 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0
|
| 23 |
+
2026-03-18 20:24:38,634 INFO MainThread:279364 [wandb_run.py:_restore():2472] restore
|
| 24 |
+
2026-03-18 20:24:38,635 INFO MainThread:279364 [wandb_run.py:_restore():2478] restore done
|
| 25 |
+
2026-03-18 20:24:39,775 INFO MainThread:279364 [wandb_run.py:_footer_sync_info():3868] logging synced files
|
dp/diffusion-policy-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142630-q4pxxlgy/run-q4pxxlgy.wandb
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a15c7ce564f713397371a85e04f4c936fd2f9ef9099ffa03bfcbecfad655d1b9
|
| 3 |
+
size 46453684
|
dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/argv.txt
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
scripts/train.py
|
| 2 |
+
simple_dp_config
|
| 3 |
+
--seed=2026
|
| 4 |
+
--exp=g1wholebodyhandover-v0
|
| 5 |
+
--train.name=diffusion-policy-g1
|
| 6 |
+
--log.report-to=wandb
|
| 7 |
+
--train.data_parallel=ddp
|
| 8 |
+
--train.mixed_precision=bf16
|
| 9 |
+
--train.train-batch-size=32
|
| 10 |
+
--train.warmup-steps=1000
|
| 11 |
+
--train.warmup-ratio=None
|
| 12 |
+
--train.checkpointing-steps=5000
|
| 13 |
+
--train.validation_steps=500
|
| 14 |
+
--train.val_num_batches=20
|
| 15 |
+
--train.gradient_accumulation_steps=1
|
| 16 |
+
--train.max-training-steps=40000
|
| 17 |
+
--train.learning-rate=1e-4
|
| 18 |
+
--train.max-grad-norm=1.0
|
| 19 |
+
--train.lr_scheduler_kwargs.weight_decay=1e-6
|
| 20 |
+
--train.lr_scheduler_kwargs.betas 0.95 0.999
|
| 21 |
+
--train.lr_scheduler_type=cosine
|
| 22 |
+
--data.root_dir=/data/jliu/data
|
| 23 |
+
--data.train-repo-ids=G1WholebodyHandover-v0
|
| 24 |
+
--data.transform.repack.action_chunk_size=16
|
| 25 |
+
--data.transform.repack.pad-action-dim=36
|
| 26 |
+
--data.transform.repack.pad-state-dim=36
|
| 27 |
+
--data.transform.field.stat-path=meta/stats_psi0.json
|
| 28 |
+
--data.transform.field.stat-action-key=action
|
| 29 |
+
--data.transform.field.stat-state-key=states
|
| 30 |
+
--data.transform.field.normalize-state
|
| 31 |
+
--data.transform.field.action-norm-type=bounds
|
| 32 |
+
--data.transform.field.pad-action-dim=36
|
| 33 |
+
--data.transform.field.pad-state-dim=36
|
| 34 |
+
--data.transform.model.img-aug
|
| 35 |
+
--model.action-chunk-size=16
|
| 36 |
+
--model.action-dim=36
|
| 37 |
+
--model.obs-dim=36
|
dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/checkpoints/ckpt_40000/ema_net.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c269fd0fba8f8419c012e3156fbcfe9dc0d718e76fe4bcce8ba6545074a01964
|
| 3 |
+
size 337302879
|
dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/checkpoints/ckpt_40000/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7e86eecf6975eb87fe7ae20127197f7b6d077b7a5cc4f01e1a6504a159b58c49
|
| 3 |
+
size 337238328
|
dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/checkpoints/ckpt_40000/optimizer.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:bed6316f01241eca5edb869101e6b126cf9dddfdc0a43588f6b761f19c9ca548
|
| 3 |
+
size 674602251
|
dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/checkpoints/ckpt_40000/random_states_0.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:bff892a7232cd4085a6a278617f383a57a337b8bd891ee607759e541b30875a6
|
| 3 |
+
size 15473
|
dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/checkpoints/ckpt_40000/random_states_1.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e712a43aee1ee2dda8edf287dd5d639562a2d9bdf3b9fd6d3b08ed07a27c1f5a
|
| 3 |
+
size 15473
|
dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/checkpoints/ckpt_40000/random_states_2.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a0289ed06080a05ed79ad2a7dbdd7fcf4e445e3adc604187a629769f203b3d5e
|
| 3 |
+
size 15473
|
dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/checkpoints/ckpt_40000/random_states_3.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b032d451de6e355991083548432d64f2eba42c38d119dcde8404311bf3f0ab54
|
| 3 |
+
size 15473
|
dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/checkpoints/ckpt_40000/scheduler.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d3ee6860bb6aecdd232f5d24bba08e626d7d56119a5f05ba8c2ad8eecf56fe38
|
| 3 |
+
size 1465
|
dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/envs.txt
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
OMP_NUM_THREADS=32
|
| 2 |
+
HF_HOME=/data/cache
|
| 3 |
+
TORCH_HOME=/data/cache
|
| 4 |
+
HF_TOKEN=hf_...TiKa
|
| 5 |
+
HF_LEROBOT_HOME=/data/data/lerobot
|
| 6 |
+
WE_HOME=Not Set
|
| 7 |
+
DATA_HOME=/data/data
|
| 8 |
+
UV_CACHE_DIR=/data/cache
|
| 9 |
+
WANDB_API_KEY=90e...5c06
|
| 10 |
+
PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
|
| 11 |
+
CUDA_VISIBLE_DEVICES=4,5,6,7
|
| 12 |
+
WORLD_SIZE=4
|
| 13 |
+
LOCAL_WORLD_SIZE=4
|
| 14 |
+
RANK=0
|
| 15 |
+
LOCAL_RANK=0
|
| 16 |
+
MASTER_ADDR=nebula99
|
| 17 |
+
MASTER_PORT=38477
|
dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/run_config.json
ADDED
|
@@ -0,0 +1,300 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"exp": "g1wholebodyhandover-v0",
|
| 3 |
+
"seed": 2026,
|
| 4 |
+
"auto_tag_run": false,
|
| 5 |
+
"eval": false,
|
| 6 |
+
"debug": false,
|
| 7 |
+
"timestamp": "2604100204",
|
| 8 |
+
"log": {
|
| 9 |
+
"logging_dir": "logs",
|
| 10 |
+
"report_to": "wandb",
|
| 11 |
+
"log_freq": 100
|
| 12 |
+
},
|
| 13 |
+
"wandb": {
|
| 14 |
+
"project": "psi",
|
| 15 |
+
"entity": "jliu530-soochow-university",
|
| 16 |
+
"group": "diffusion-policy-g1",
|
| 17 |
+
"id": "sjem5sh4",
|
| 18 |
+
"name": "g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204",
|
| 19 |
+
"resume": "allow"
|
| 20 |
+
},
|
| 21 |
+
"train": {
|
| 22 |
+
"num_workers": 8,
|
| 23 |
+
"overfit_single_batch": false,
|
| 24 |
+
"name": "diffusion-policy-g1",
|
| 25 |
+
"resume_from_checkpoint": null,
|
| 26 |
+
"skip_resumed_steps": false,
|
| 27 |
+
"hf_token": ".hf_token",
|
| 28 |
+
"lora": false,
|
| 29 |
+
"output_dir": ".runs",
|
| 30 |
+
"gradient_accumulation_steps": 1,
|
| 31 |
+
"mixed_precision": "bf16",
|
| 32 |
+
"max_grad_norm": 1.0,
|
| 33 |
+
"optimizer_foreach": null,
|
| 34 |
+
"train_batch_size": 32,
|
| 35 |
+
"val_batch_size": 16,
|
| 36 |
+
"val_num_batches": 20,
|
| 37 |
+
"checkpointing_steps": 5000,
|
| 38 |
+
"max_checkpoints_to_keep": null,
|
| 39 |
+
"validation_steps": 500,
|
| 40 |
+
"learning_rate": 0.0001,
|
| 41 |
+
"lr_scheduler_type": "cosine",
|
| 42 |
+
"lr_scheduler_kwargs": {
|
| 43 |
+
"betas": [
|
| 44 |
+
0.95,
|
| 45 |
+
0.999
|
| 46 |
+
],
|
| 47 |
+
"weight_decay": 1e-6,
|
| 48 |
+
"eps": 1e-8
|
| 49 |
+
},
|
| 50 |
+
"scheduler_specific_kwargs": {},
|
| 51 |
+
"data_parallel": "ddp",
|
| 52 |
+
"sharding_strategy": "full-shard",
|
| 53 |
+
"deepspeed_config": "/data/jliu/psi/src/InternVLA/config/deepseeds/zero3.json",
|
| 54 |
+
"enable_gradient_checkpointing": true,
|
| 55 |
+
"enable_mixed_precision_training": true,
|
| 56 |
+
"reduce_in_full_precision": true,
|
| 57 |
+
"max_training_steps": 40000,
|
| 58 |
+
"num_train_epochs": null,
|
| 59 |
+
"warmup_steps": 1000,
|
| 60 |
+
"warmup_ratio": null
|
| 61 |
+
},
|
| 62 |
+
"data": {
|
| 63 |
+
"transform": {
|
| 64 |
+
"repack": {
|
| 65 |
+
"dataset_name": "simple",
|
| 66 |
+
"num_past_frames": 0,
|
| 67 |
+
"action_chunk_size": 16,
|
| 68 |
+
"pad_action_dim": 36,
|
| 69 |
+
"pad_state_dim": 36
|
| 70 |
+
},
|
| 71 |
+
"model": {
|
| 72 |
+
"resize": {
|
| 73 |
+
"size": [
|
| 74 |
+
256,
|
| 75 |
+
480
|
| 76 |
+
]
|
| 77 |
+
},
|
| 78 |
+
"color_jitter": {
|
| 79 |
+
"brightness": 0.2,
|
| 80 |
+
"contrast": [
|
| 81 |
+
0.8,
|
| 82 |
+
1.2
|
| 83 |
+
],
|
| 84 |
+
"saturation": [
|
| 85 |
+
0.8,
|
| 86 |
+
1.2
|
| 87 |
+
],
|
| 88 |
+
"hue": 0.05
|
| 89 |
+
},
|
| 90 |
+
"center_crop": {
|
| 91 |
+
"size": [
|
| 92 |
+
224,
|
| 93 |
+
224
|
| 94 |
+
]
|
| 95 |
+
},
|
| 96 |
+
"normalize": {
|
| 97 |
+
"mean": [
|
| 98 |
+
0.485,
|
| 99 |
+
0.456,
|
| 100 |
+
0.406
|
| 101 |
+
],
|
| 102 |
+
"std": [
|
| 103 |
+
0.229,
|
| 104 |
+
0.224,
|
| 105 |
+
0.225
|
| 106 |
+
]
|
| 107 |
+
},
|
| 108 |
+
"img_aug": true
|
| 109 |
+
},
|
| 110 |
+
"field": {
|
| 111 |
+
"stat_path": "meta/stats_psi0.json",
|
| 112 |
+
"action_norm_type": "bounds",
|
| 113 |
+
"stat_action_key": "action",
|
| 114 |
+
"stat_state_key": "states",
|
| 115 |
+
"use_norm_mask": false,
|
| 116 |
+
"action_norm_masks": [
|
| 117 |
+
true,
|
| 118 |
+
true,
|
| 119 |
+
true,
|
| 120 |
+
true,
|
| 121 |
+
true,
|
| 122 |
+
true,
|
| 123 |
+
false
|
| 124 |
+
],
|
| 125 |
+
"action_min": [
|
| 126 |
+
-0.5,
|
| 127 |
+
-1.1095792134107943e-16,
|
| 128 |
+
-1.1095792134107943e-16,
|
| 129 |
+
-1.5,
|
| 130 |
+
-1.5,
|
| 131 |
+
-1.5,
|
| 132 |
+
-1.5,
|
| 133 |
+
-0.5,
|
| 134 |
+
-0.699999988079071,
|
| 135 |
+
-0.699999988079071,
|
| 136 |
+
-2.2166350627321588e-16,
|
| 137 |
+
-2.2166350627321588e-16,
|
| 138 |
+
0.0,
|
| 139 |
+
-2.2166350627321588e-16,
|
| 140 |
+
-0.47567468881607056,
|
| 141 |
+
0.1900009959936142,
|
| 142 |
+
-0.512170135974884,
|
| 143 |
+
-0.6265152096748352,
|
| 144 |
+
-0.5008617043495178,
|
| 145 |
+
-0.8220608830451965,
|
| 146 |
+
-0.9223371148109436,
|
| 147 |
+
-0.49507391452789307,
|
| 148 |
+
-0.3437551259994507,
|
| 149 |
+
-0.6871383190155029,
|
| 150 |
+
-0.7637607455253601,
|
| 151 |
+
-0.7568023204803467,
|
| 152 |
+
-0.576077401638031,
|
| 153 |
+
-0.4588268995285034,
|
| 154 |
+
-0.13876836001873016,
|
| 155 |
+
-0.10360867530107498,
|
| 156 |
+
-0.47856518626213074,
|
| 157 |
+
0.7400000095367432,
|
| 158 |
+
-0.5,
|
| 159 |
+
-0.5,
|
| 160 |
+
-0.26161932945251465,
|
| 161 |
+
-0.06718750298023224
|
| 162 |
+
],
|
| 163 |
+
"action_max": [
|
| 164 |
+
0.5,
|
| 165 |
+
0.699999988079071,
|
| 166 |
+
0.699999988079071,
|
| 167 |
+
0.0,
|
| 168 |
+
2.2146225653890418e-16,
|
| 169 |
+
2.2146225653890418e-16,
|
| 170 |
+
2.2146225653890418e-16,
|
| 171 |
+
1.2266071310501902e-19,
|
| 172 |
+
1.1078670818917075e-16,
|
| 173 |
+
1.1078670818917075e-16,
|
| 174 |
+
1.5,
|
| 175 |
+
1.5,
|
| 176 |
+
0.6000000238418579,
|
| 177 |
+
1.5,
|
| 178 |
+
0.2472410947084427,
|
| 179 |
+
0.7092280983924866,
|
| 180 |
+
1.2571598291397095,
|
| 181 |
+
0.42311304807662964,
|
| 182 |
+
0.8564174771308899,
|
| 183 |
+
0.5002086162567139,
|
| 184 |
+
0.5172277092933655,
|
| 185 |
+
0.16140148043632507,
|
| 186 |
+
-0.1900009959936142,
|
| 187 |
+
0.5362864136695862,
|
| 188 |
+
0.5715147256851196,
|
| 189 |
+
0.5002322196960449,
|
| 190 |
+
0.566592276096344,
|
| 191 |
+
0.6392397880554199,
|
| 192 |
+
0.1580466777086258,
|
| 193 |
+
0.2233395129442215,
|
| 194 |
+
0.2582152187824249,
|
| 195 |
+
0.7400000095367432,
|
| 196 |
+
0.5,
|
| 197 |
+
0.5,
|
| 198 |
+
0.3454970121383667,
|
| 199 |
+
0.2899305522441864
|
| 200 |
+
],
|
| 201 |
+
"state_min": [
|
| 202 |
+
-0.5564982891082764,
|
| 203 |
+
-0.48307520151138306,
|
| 204 |
+
-0.0005447770818136632,
|
| 205 |
+
-0.8388738632202148,
|
| 206 |
+
-1.3970016241073608,
|
| 207 |
+
-0.8296014666557312,
|
| 208 |
+
-1.4599460363388062,
|
| 209 |
+
-0.5806806683540344,
|
| 210 |
+
-0.5149835348129272,
|
| 211 |
+
-0.6775947213172913,
|
| 212 |
+
-0.001480442238971591,
|
| 213 |
+
-0.0002713006397243589,
|
| 214 |
+
-0.000914653530344367,
|
| 215 |
+
-0.00019419840828049928,
|
| 216 |
+
-0.4206617772579193,
|
| 217 |
+
0.13972464203834534,
|
| 218 |
+
-0.546251654624939,
|
| 219 |
+
-0.5596316456794739,
|
| 220 |
+
-0.4764360189437866,
|
| 221 |
+
-0.7253566384315491,
|
| 222 |
+
-0.9443663954734802,
|
| 223 |
+
-0.4381798803806305,
|
| 224 |
+
-0.3338131606578827,
|
| 225 |
+
-0.667724072933197,
|
| 226 |
+
-0.6881827116012573,
|
| 227 |
+
-0.7544379830360413,
|
| 228 |
+
-0.5189417600631714,
|
| 229 |
+
-0.4484957158565521,
|
| 230 |
+
-0.13709338009357452,
|
| 231 |
+
-0.07360810041427612,
|
| 232 |
+
-0.4748336970806122,
|
| 233 |
+
0.7400000095367432,
|
| 234 |
+
0.0,
|
| 235 |
+
0.0,
|
| 236 |
+
0.0,
|
| 237 |
+
0.0
|
| 238 |
+
],
|
| 239 |
+
"state_max": [
|
| 240 |
+
0.43566983938217163,
|
| 241 |
+
0.3739710748195648,
|
| 242 |
+
0.6575677990913391,
|
| 243 |
+
0.004060761071741581,
|
| 244 |
+
0.0005700877518393099,
|
| 245 |
+
0.0004725759499706328,
|
| 246 |
+
0.00010080631182063371,
|
| 247 |
+
0.00001310737025050912,
|
| 248 |
+
0.21882089972496033,
|
| 249 |
+
0.0005271440604701638,
|
| 250 |
+
0.530737042427063,
|
| 251 |
+
1.4406861066818237,
|
| 252 |
+
1.4605127573013306,
|
| 253 |
+
1.4595911502838135,
|
| 254 |
+
0.2663630545139313,
|
| 255 |
+
0.657910943031311,
|
| 256 |
+
1.2515853643417358,
|
| 257 |
+
0.502498209476471,
|
| 258 |
+
0.8292973637580872,
|
| 259 |
+
0.5248894095420837,
|
| 260 |
+
0.4653257131576538,
|
| 261 |
+
0.18638382852077484,
|
| 262 |
+
-0.16696421802043915,
|
| 263 |
+
0.49318820238113403,
|
| 264 |
+
0.6363148093223572,
|
| 265 |
+
0.45773962140083313,
|
| 266 |
+
0.6238265037536621,
|
| 267 |
+
0.653800904750824,
|
| 268 |
+
0.1436084657907486,
|
| 269 |
+
0.25937986373901367,
|
| 270 |
+
0.26422709226608276,
|
| 271 |
+
0.7400000095367432,
|
| 272 |
+
0.0,
|
| 273 |
+
0.0,
|
| 274 |
+
0.0,
|
| 275 |
+
0.0
|
| 276 |
+
],
|
| 277 |
+
"normalize_state": true,
|
| 278 |
+
"pad_action_dim": 36,
|
| 279 |
+
"pad_state_dim": 36
|
| 280 |
+
}
|
| 281 |
+
},
|
| 282 |
+
"root_dir": "/data/jliu/data",
|
| 283 |
+
"train_repo_ids": [
|
| 284 |
+
"G1WholebodyHandover-v0"
|
| 285 |
+
],
|
| 286 |
+
"val_repo_ids": [
|
| 287 |
+
"G1WholebodyHandover-v0"
|
| 288 |
+
]
|
| 289 |
+
},
|
| 290 |
+
"model": {
|
| 291 |
+
"num_diffusion_iters": 100,
|
| 292 |
+
"action_chunk_size": 16,
|
| 293 |
+
"num_cameras": 1,
|
| 294 |
+
"share_vision_encoder": false,
|
| 295 |
+
"obs_horizon": 1,
|
| 296 |
+
"action_exec_horizon": 6,
|
| 297 |
+
"action_dim": 36,
|
| 298 |
+
"obs_dim": 36
|
| 299 |
+
}
|
| 300 |
+
}
|
dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/wandb/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/wandb/debug.log
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-04-10 02:05:01,007 INFO MainThread:1404689 [wandb_setup.py:_flush():81] Current SDK version is 0.25.1
|
| 2 |
+
2026-04-10 02:05:01,007 INFO MainThread:1404689 [wandb_setup.py:_flush():81] Configure stats pid to 1404689
|
| 3 |
+
2026-04-10 02:05:01,007 INFO MainThread:1404689 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-04-10 02:05:01,007 INFO MainThread:1404689 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data/jliu/psi/.runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/wandb/run-20260410_020501-sjem5sh4/logs/debug.log
|
| 5 |
+
2026-04-10 02:05:01,007 INFO MainThread:1404689 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data/jliu/psi/.runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/wandb/run-20260410_020501-sjem5sh4/logs/debug-internal.log
|
| 6 |
+
2026-04-10 02:05:01,007 INFO MainThread:1404689 [wandb_init.py:init():844] calling init triggers
|
| 7 |
+
2026-04-10 02:05:01,007 INFO MainThread:1404689 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'_wandb': {}}
|
| 9 |
+
2026-04-10 02:05:01,007 INFO MainThread:1404689 [wandb_init.py:init():892] starting backend
|
| 10 |
+
2026-04-10 02:05:01,257 INFO MainThread:1404689 [wandb_init.py:init():895] sending inform_init request
|
| 11 |
+
2026-04-10 02:05:01,265 INFO MainThread:1404689 [wandb_init.py:init():903] backend started and connected
|
| 12 |
+
2026-04-10 02:05:01,266 INFO MainThread:1404689 [wandb_init.py:init():973] updated telemetry
|
| 13 |
+
2026-04-10 02:05:01,272 INFO MainThread:1404689 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-04-10 02:05:02,182 INFO MainThread:1404689 [wandb_init.py:init():1042] starting run threads in backend
|
| 15 |
+
2026-04-10 02:05:02,323 INFO MainThread:1404689 [wandb_run.py:_console_start():2524] atexit reg
|
| 16 |
+
2026-04-10 02:05:02,323 INFO MainThread:1404689 [wandb_run.py:_redirect():2373] redirect: wrap_raw
|
| 17 |
+
2026-04-10 02:05:02,323 INFO MainThread:1404689 [wandb_run.py:_redirect():2442] Wrapping output streams.
|
| 18 |
+
2026-04-10 02:05:02,323 INFO MainThread:1404689 [wandb_run.py:_redirect():2465] Redirects installed.
|
| 19 |
+
2026-04-10 02:05:02,328 INFO MainThread:1404689 [wandb_init.py:init():1082] run started, returning control to user process
|
| 20 |
+
2026-04-10 02:05:02,329 INFO MainThread:1404689 [wandb_run.py:_config_callback():1403] config_cb None None {'exp': 'g1wholebodyhandover-v0', 'seed': 2026, 'auto_tag_run': False, 'eval': False, 'debug': False, 'timestamp': '2604100204', 'log': {'logging_dir': 'logs', 'report_to': 'wandb', 'log_freq': 100}, 'wandb': {'project': 'psi', 'entity': 'jliu530-soochow-university', 'group': None, 'id': None, 'name': None, 'resume': 'allow'}, 'train': {'num_workers': 8, 'overfit_single_batch': False, 'name': 'diffusion-policy-g1', 'resume_from_checkpoint': None, 'skip_resumed_steps': False, 'hf_token': '.hf_token', 'lora': False, 'output_dir': '.runs', 'gradient_accumulation_steps': 1, 'mixed_precision': 'bf16', 'max_grad_norm': 1.0, 'optimizer_foreach': None, 'train_batch_size': 32, 'val_batch_size': 16, 'val_num_batches': 20, 'checkpointing_steps': 5000, 'max_checkpoints_to_keep': None, 'validation_steps': 500, 'learning_rate': 0.0001, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {'betas': [0.95, 0.999], 'weight_decay': 1e-06, 'eps': 1e-08}, 'scheduler_specific_kwargs': {}, 'data_parallel': 'ddp', 'sharding_strategy': 'full-shard', 'deepspeed_config': '/data/jliu/psi/src/InternVLA/config/deepseeds/zero3.json', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': True, 'max_training_steps': 40000, 'num_train_epochs': None, 'warmup_steps': 1000, 'warmup_ratio': None}, 'data': {'transform': {'repack': {'dataset_name': 'simple', 'num_past_frames': 0, 'action_chunk_size': 16, 'pad_action_dim': 36, 'pad_state_dim': 36}, 'model': {'resize': {'size': [256, 480]}, 'color_jitter': {'brightness': 0.2, 'contrast': [0.8, 1.2], 'saturation': [0.8, 1.2], 'hue': 0.05}, 'center_crop': {'size': [224, 224]}, 'normalize': {'mean': [0.485, 0.456, 0.406], 'std': [0.229, 0.224, 0.225]}, 'img_aug': True}, 'field': {'stat_path': 'meta/stats_psi0.json', 'action_norm_type': 'bounds', 'stat_action_key': 'action', 'stat_state_key': 'states', 'use_norm_mask': False, 'action_norm_masks': [True, True, True, True, True, True, False], 'action_min': [-0.5, -1.1095792134107943e-16, -1.1095792134107943e-16, -1.5, -1.5, -1.5, -1.5, -0.5, -0.699999988079071, -0.699999988079071, -2.2166350627321588e-16, -2.2166350627321588e-16, 0.0, -2.2166350627321588e-16, -0.47567468881607056, 0.1900009959936142, -0.512170135974884, -0.6265152096748352, -0.5008617043495178, -0.8220608830451965, -0.9223371148109436, -0.49507391452789307, -0.3437551259994507, -0.6871383190155029, -0.7637607455253601, -0.7568023204803467, -0.576077401638031, -0.4588268995285034, -0.13876836001873016, -0.10360867530107498, -0.47856518626213074, 0.7400000095367432, -0.5, -0.5, -0.26161932945251465, -0.06718750298023224], 'action_max': [0.5, 0.699999988079071, 0.699999988079071, 0.0, 2.2146225653890418e-16, 2.2146225653890418e-16, 2.2146225653890418e-16, 1.2266071310501902e-19, 1.1078670818917075e-16, 1.1078670818917075e-16, 1.5, 1.5, 0.6000000238418579, 1.5, 0.2472410947084427, 0.7092280983924866, 1.2571598291397095, 0.42311304807662964, 0.8564174771308899, 0.5002086162567139, 0.5172277092933655, 0.16140148043632507, -0.1900009959936142, 0.5362864136695862, 0.5715147256851196, 0.5002322196960449, 0.566592276096344, 0.6392397880554199, 0.1580466777086258, 0.2233395129442215, 0.2582152187824249, 0.7400000095367432, 0.5, 0.5, 0.3454970121383667, 0.2899305522441864], 'state_min': [-0.5564982891082764, -0.48307520151138306, -0.0005447770818136632, -0.8388738632202148, -1.3970016241073608, -0.8296014666557312, -1.4599460363388062, -0.5806806683540344, -0.5149835348129272, -0.6775947213172913, -0.001480442238971591, -0.0002713006397243589, -0.000914653530344367, -0.00019419840828049928, -0.4206617772579193, 0.13972464203834534, -0.546251654624939, -0.5596316456794739, -0.4764360189437866, -0.7253566384315491, -0.9443663954734802, -0.4381798803806305, -0.3338131606578827, -0.667724072933197, -0.6881827116012573, -0.7544379830360413, -0.5189417600631714, -0.4484957158565521, -0.13709338009357452, -0.07360810041427612, -0.4748336970806122, 0.7400000095367432, 0.0, 0.0, 0.0, 0.0], 'state_max': [0.43566983938217163, 0.3739710748195648, 0.6575677990913391, 0.004060761071741581, 0.0005700877518393099, 0.0004725759499706328, 0.00010080631182063371, 1.310737025050912e-05, 0.21882089972496033, 0.0005271440604701638, 0.530737042427063, 1.4406861066818237, 1.4605127573013306, 1.4595911502838135, 0.2663630545139313, 0.657910943031311, 1.2515853643417358, 0.502498209476471, 0.8292973637580872, 0.5248894095420837, 0.4653257131576538, 0.18638382852077484, -0.16696421802043915, 0.49318820238113403, 0.6363148093223572, 0.45773962140083313, 0.6238265037536621, 0.653800904750824, 0.1436084657907486, 0.25937986373901367, 0.26422709226608276, 0.7400000095367432, 0.0, 0.0, 0.0, 0.0], 'normalize_state': True, 'pad_action_dim': 36, 'pad_state_dim': 36}}, 'root_dir': '/data/jliu/data', 'train_repo_ids': ['G1WholebodyHandover-v0'], 'val_repo_ids': ['G1WholebodyHandover-v0']}, 'model': {'num_diffusion_iters': 100, 'action_chunk_size': 16, 'num_cameras': 1, 'share_vision_encoder': False, 'obs_horizon': 1, 'action_exec_horizon': 6, 'action_dim': 36, 'obs_dim': 36}, 'environment_variables': {'OMP_NUM_THREADS': '32', 'HF_HOME': '/data/cache', 'TORCH_HOME': '/data/cache', 'HF_TOKEN': 'hf_...TiKa', 'HF_LEROBOT_HOME': '/data/data/lerobot', 'WE_HOME': 'Not Set', 'DATA_HOME': '/data/data', 'UV_CACHE_DIR': '/data/cache', 'WANDB_API_KEY': '90e...5c06', 'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION': 'python', 'CUDA_VISIBLE_DEVICES': '4,5,6,7', 'WORLD_SIZE': '4', 'LOCAL_WORLD_SIZE': '4', 'RANK': '0', 'LOCAL_RANK': '0', 'MASTER_ADDR': 'nebula99', 'MASTER_PORT': '38477'}}
|
| 21 |
+
2026-04-10 05:25:47,354 INFO MainThread:1404689 [wandb_run.py:_finish():2291] finishing run jliu530-soochow-university/psi/sjem5sh4
|
| 22 |
+
2026-04-10 05:25:47,355 INFO MainThread:1404689 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0
|
| 23 |
+
2026-04-10 05:25:47,356 INFO MainThread:1404689 [wandb_run.py:_restore():2472] restore
|
| 24 |
+
2026-04-10 05:25:47,356 INFO MainThread:1404689 [wandb_run.py:_restore():2478] restore done
|
| 25 |
+
2026-04-10 05:25:48,206 INFO MainThread:1404689 [wandb_run.py:_footer_sync_info():3868] logging synced files
|
dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/wandb/run-20260410_020501-sjem5sh4/files/config.yaml
ADDED
|
@@ -0,0 +1,436 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
_wandb:
|
| 2 |
+
value:
|
| 3 |
+
cli_version: 0.25.1
|
| 4 |
+
e:
|
| 5 |
+
6ptlrch3t1wewxljiv5hkmtr0grgv5w6:
|
| 6 |
+
args:
|
| 7 |
+
- simple_dp_config
|
| 8 |
+
- --seed=2026
|
| 9 |
+
- --exp=g1wholebodyhandover-v0
|
| 10 |
+
- --train.name=diffusion-policy-g1
|
| 11 |
+
- --log.report-to=wandb
|
| 12 |
+
- --train.data_parallel=ddp
|
| 13 |
+
- --train.mixed_precision=bf16
|
| 14 |
+
- --train.train-batch-size=32
|
| 15 |
+
- --train.warmup-steps=1000
|
| 16 |
+
- --train.warmup-ratio=None
|
| 17 |
+
- --train.checkpointing-steps=5000
|
| 18 |
+
- --train.validation_steps=500
|
| 19 |
+
- --train.val_num_batches=20
|
| 20 |
+
- --train.gradient_accumulation_steps=1
|
| 21 |
+
- --train.max-training-steps=40000
|
| 22 |
+
- --train.learning-rate=1e-4
|
| 23 |
+
- --train.max-grad-norm=1.0
|
| 24 |
+
- --train.lr_scheduler_kwargs.weight_decay=1e-6
|
| 25 |
+
- --train.lr_scheduler_kwargs.betas
|
| 26 |
+
- "0.95"
|
| 27 |
+
- "0.999"
|
| 28 |
+
- --train.lr_scheduler_type=cosine
|
| 29 |
+
- --data.root_dir=/data/jliu/data
|
| 30 |
+
- --data.train-repo-ids=G1WholebodyHandover-v0
|
| 31 |
+
- --data.transform.repack.action_chunk_size=16
|
| 32 |
+
- --data.transform.repack.pad-action-dim=36
|
| 33 |
+
- --data.transform.repack.pad-state-dim=36
|
| 34 |
+
- --data.transform.field.stat-path=meta/stats_psi0.json
|
| 35 |
+
- --data.transform.field.stat-action-key=action
|
| 36 |
+
- --data.transform.field.stat-state-key=states
|
| 37 |
+
- --data.transform.field.normalize-state
|
| 38 |
+
- --data.transform.field.action-norm-type=bounds
|
| 39 |
+
- --data.transform.field.pad-action-dim=36
|
| 40 |
+
- --data.transform.field.pad-state-dim=36
|
| 41 |
+
- --data.transform.model.img-aug
|
| 42 |
+
- --model.action-chunk-size=16
|
| 43 |
+
- --model.action-dim=36
|
| 44 |
+
- --model.obs-dim=36
|
| 45 |
+
codePath: scripts/train.py
|
| 46 |
+
codePathLocal: scripts/train.py
|
| 47 |
+
cpu_count: 128
|
| 48 |
+
cpu_count_logical: 128
|
| 49 |
+
cudaVersion: "12.9"
|
| 50 |
+
disk:
|
| 51 |
+
/:
|
| 52 |
+
total: "105089261568"
|
| 53 |
+
used: "82290688000"
|
| 54 |
+
email: jliu530@163.com
|
| 55 |
+
executable: /data/jliu/psi/.venv-dp/bin/python3
|
| 56 |
+
git:
|
| 57 |
+
commit: 04614628ecb677f5f278e2d31c1103aed8127f26
|
| 58 |
+
remote: https://github.com/songlin/psi.git
|
| 59 |
+
gpu: NVIDIA A100-SXM4-80GB
|
| 60 |
+
gpu_count: 8
|
| 61 |
+
gpu_nvidia:
|
| 62 |
+
- architecture: Ampere
|
| 63 |
+
cudaCores: 6912
|
| 64 |
+
memoryTotal: "85899345920"
|
| 65 |
+
name: NVIDIA A100-SXM4-80GB
|
| 66 |
+
uuid: GPU-02b51758-2aec-8c71-c751-92567fdb15ad
|
| 67 |
+
- architecture: Ampere
|
| 68 |
+
cudaCores: 6912
|
| 69 |
+
memoryTotal: "85899345920"
|
| 70 |
+
name: NVIDIA A100-SXM4-80GB
|
| 71 |
+
uuid: GPU-be645bb4-1ef0-15da-fc6b-53c5378650df
|
| 72 |
+
- architecture: Ampere
|
| 73 |
+
cudaCores: 6912
|
| 74 |
+
memoryTotal: "85899345920"
|
| 75 |
+
name: NVIDIA A100-SXM4-80GB
|
| 76 |
+
uuid: GPU-44868ccd-5809-61e2-f8cc-1dff8efc7eb1
|
| 77 |
+
- architecture: Ampere
|
| 78 |
+
cudaCores: 6912
|
| 79 |
+
memoryTotal: "85899345920"
|
| 80 |
+
name: NVIDIA A100-SXM4-80GB
|
| 81 |
+
uuid: GPU-a12b8de3-fb12-2901-c3a2-1e0e2b050b7b
|
| 82 |
+
- architecture: Ampere
|
| 83 |
+
cudaCores: 6912
|
| 84 |
+
memoryTotal: "85899345920"
|
| 85 |
+
name: NVIDIA A100-SXM4-80GB
|
| 86 |
+
uuid: GPU-57787183-4904-b2c9-cf52-1c1af976b365
|
| 87 |
+
- architecture: Ampere
|
| 88 |
+
cudaCores: 6912
|
| 89 |
+
memoryTotal: "85899345920"
|
| 90 |
+
name: NVIDIA A100-SXM4-80GB
|
| 91 |
+
uuid: GPU-e82d0fd0-9dcb-3d7c-9451-c4770ad6ddc3
|
| 92 |
+
- architecture: Ampere
|
| 93 |
+
cudaCores: 6912
|
| 94 |
+
memoryTotal: "85899345920"
|
| 95 |
+
name: NVIDIA A100-SXM4-80GB
|
| 96 |
+
uuid: GPU-9a825fa3-60d0-3acf-6c79-67e9230ae24c
|
| 97 |
+
- architecture: Ampere
|
| 98 |
+
cudaCores: 6912
|
| 99 |
+
memoryTotal: "85899345920"
|
| 100 |
+
name: NVIDIA A100-SXM4-80GB
|
| 101 |
+
uuid: GPU-6fe02e5e-b886-9c02-e433-6237f3897dd8
|
| 102 |
+
host: nebula99
|
| 103 |
+
memory:
|
| 104 |
+
total: "1623177445376"
|
| 105 |
+
os: Linux-6.8.0-106-generic-x86_64-with-glibc2.39
|
| 106 |
+
program: /data/jliu/psi/scripts/train.py
|
| 107 |
+
python: CPython 3.10.20
|
| 108 |
+
root: /data/jliu/psi/.runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204
|
| 109 |
+
startedAt: "2026-04-10T02:05:01.005456Z"
|
| 110 |
+
writerId: 6ptlrch3t1wewxljiv5hkmtr0grgv5w6
|
| 111 |
+
m: []
|
| 112 |
+
python_version: 3.10.20
|
| 113 |
+
t:
|
| 114 |
+
"1":
|
| 115 |
+
- 1
|
| 116 |
+
- 11
|
| 117 |
+
- 41
|
| 118 |
+
- 49
|
| 119 |
+
- 71
|
| 120 |
+
- 83
|
| 121 |
+
"2":
|
| 122 |
+
- 1
|
| 123 |
+
- 11
|
| 124 |
+
- 41
|
| 125 |
+
- 49
|
| 126 |
+
- 51
|
| 127 |
+
- 71
|
| 128 |
+
- 83
|
| 129 |
+
"3":
|
| 130 |
+
- 2
|
| 131 |
+
- 13
|
| 132 |
+
- 61
|
| 133 |
+
"4": 3.10.20
|
| 134 |
+
"5": 0.25.1
|
| 135 |
+
"6": 4.44.2
|
| 136 |
+
"12": 0.25.1
|
| 137 |
+
"13": linux-x86_64
|
| 138 |
+
auto_tag_run:
|
| 139 |
+
value: false
|
| 140 |
+
data:
|
| 141 |
+
value:
|
| 142 |
+
root_dir: /data/jliu/data
|
| 143 |
+
train_repo_ids:
|
| 144 |
+
- G1WholebodyHandover-v0
|
| 145 |
+
transform:
|
| 146 |
+
field:
|
| 147 |
+
action_max:
|
| 148 |
+
- 0.5
|
| 149 |
+
- 0.699999988079071
|
| 150 |
+
- 0.699999988079071
|
| 151 |
+
- 0
|
| 152 |
+
- 2.2146225653890418e-16
|
| 153 |
+
- 2.2146225653890418e-16
|
| 154 |
+
- 2.2146225653890418e-16
|
| 155 |
+
- 1.2266071310501902e-19
|
| 156 |
+
- 1.1078670818917075e-16
|
| 157 |
+
- 1.1078670818917075e-16
|
| 158 |
+
- 1.5
|
| 159 |
+
- 1.5
|
| 160 |
+
- 0.6000000238418579
|
| 161 |
+
- 1.5
|
| 162 |
+
- 0.2472410947084427
|
| 163 |
+
- 0.7092280983924866
|
| 164 |
+
- 1.2571598291397095
|
| 165 |
+
- 0.42311304807662964
|
| 166 |
+
- 0.8564174771308899
|
| 167 |
+
- 0.5002086162567139
|
| 168 |
+
- 0.5172277092933655
|
| 169 |
+
- 0.16140148043632507
|
| 170 |
+
- -0.1900009959936142
|
| 171 |
+
- 0.5362864136695862
|
| 172 |
+
- 0.5715147256851196
|
| 173 |
+
- 0.5002322196960449
|
| 174 |
+
- 0.566592276096344
|
| 175 |
+
- 0.6392397880554199
|
| 176 |
+
- 0.1580466777086258
|
| 177 |
+
- 0.2233395129442215
|
| 178 |
+
- 0.2582152187824249
|
| 179 |
+
- 0.7400000095367432
|
| 180 |
+
- 0.5
|
| 181 |
+
- 0.5
|
| 182 |
+
- 0.3454970121383667
|
| 183 |
+
- 0.2899305522441864
|
| 184 |
+
action_min:
|
| 185 |
+
- -0.5
|
| 186 |
+
- -1.1095792134107943e-16
|
| 187 |
+
- -1.1095792134107943e-16
|
| 188 |
+
- -1.5
|
| 189 |
+
- -1.5
|
| 190 |
+
- -1.5
|
| 191 |
+
- -1.5
|
| 192 |
+
- -0.5
|
| 193 |
+
- -0.699999988079071
|
| 194 |
+
- -0.699999988079071
|
| 195 |
+
- -2.2166350627321588e-16
|
| 196 |
+
- -2.2166350627321588e-16
|
| 197 |
+
- 0
|
| 198 |
+
- -2.2166350627321588e-16
|
| 199 |
+
- -0.47567468881607056
|
| 200 |
+
- 0.1900009959936142
|
| 201 |
+
- -0.512170135974884
|
| 202 |
+
- -0.6265152096748352
|
| 203 |
+
- -0.5008617043495178
|
| 204 |
+
- -0.8220608830451965
|
| 205 |
+
- -0.9223371148109436
|
| 206 |
+
- -0.49507391452789307
|
| 207 |
+
- -0.3437551259994507
|
| 208 |
+
- -0.6871383190155029
|
| 209 |
+
- -0.7637607455253601
|
| 210 |
+
- -0.7568023204803467
|
| 211 |
+
- -0.576077401638031
|
| 212 |
+
- -0.4588268995285034
|
| 213 |
+
- -0.13876836001873016
|
| 214 |
+
- -0.10360867530107498
|
| 215 |
+
- -0.47856518626213074
|
| 216 |
+
- 0.7400000095367432
|
| 217 |
+
- -0.5
|
| 218 |
+
- -0.5
|
| 219 |
+
- -0.26161932945251465
|
| 220 |
+
- -0.06718750298023224
|
| 221 |
+
action_norm_masks:
|
| 222 |
+
- true
|
| 223 |
+
- true
|
| 224 |
+
- true
|
| 225 |
+
- true
|
| 226 |
+
- true
|
| 227 |
+
- true
|
| 228 |
+
- false
|
| 229 |
+
action_norm_type: bounds
|
| 230 |
+
normalize_state: true
|
| 231 |
+
pad_action_dim: 36
|
| 232 |
+
pad_state_dim: 36
|
| 233 |
+
stat_action_key: action
|
| 234 |
+
stat_path: meta/stats_psi0.json
|
| 235 |
+
stat_state_key: states
|
| 236 |
+
state_max:
|
| 237 |
+
- 0.43566983938217163
|
| 238 |
+
- 0.3739710748195648
|
| 239 |
+
- 0.6575677990913391
|
| 240 |
+
- 0.004060761071741581
|
| 241 |
+
- 0.0005700877518393099
|
| 242 |
+
- 0.0004725759499706328
|
| 243 |
+
- 0.00010080631182063371
|
| 244 |
+
- 1.310737025050912e-05
|
| 245 |
+
- 0.21882089972496033
|
| 246 |
+
- 0.0005271440604701638
|
| 247 |
+
- 0.530737042427063
|
| 248 |
+
- 1.4406861066818237
|
| 249 |
+
- 1.4605127573013306
|
| 250 |
+
- 1.4595911502838135
|
| 251 |
+
- 0.2663630545139313
|
| 252 |
+
- 0.657910943031311
|
| 253 |
+
- 1.2515853643417358
|
| 254 |
+
- 0.502498209476471
|
| 255 |
+
- 0.8292973637580872
|
| 256 |
+
- 0.5248894095420837
|
| 257 |
+
- 0.4653257131576538
|
| 258 |
+
- 0.18638382852077484
|
| 259 |
+
- -0.16696421802043915
|
| 260 |
+
- 0.49318820238113403
|
| 261 |
+
- 0.6363148093223572
|
| 262 |
+
- 0.45773962140083313
|
| 263 |
+
- 0.6238265037536621
|
| 264 |
+
- 0.653800904750824
|
| 265 |
+
- 0.1436084657907486
|
| 266 |
+
- 0.25937986373901367
|
| 267 |
+
- 0.26422709226608276
|
| 268 |
+
- 0.7400000095367432
|
| 269 |
+
- 0
|
| 270 |
+
- 0
|
| 271 |
+
- 0
|
| 272 |
+
- 0
|
| 273 |
+
state_min:
|
| 274 |
+
- -0.5564982891082764
|
| 275 |
+
- -0.48307520151138306
|
| 276 |
+
- -0.0005447770818136632
|
| 277 |
+
- -0.8388738632202148
|
| 278 |
+
- -1.3970016241073608
|
| 279 |
+
- -0.8296014666557312
|
| 280 |
+
- -1.4599460363388062
|
| 281 |
+
- -0.5806806683540344
|
| 282 |
+
- -0.5149835348129272
|
| 283 |
+
- -0.6775947213172913
|
| 284 |
+
- -0.001480442238971591
|
| 285 |
+
- -0.0002713006397243589
|
| 286 |
+
- -0.000914653530344367
|
| 287 |
+
- -0.00019419840828049928
|
| 288 |
+
- -0.4206617772579193
|
| 289 |
+
- 0.13972464203834534
|
| 290 |
+
- -0.546251654624939
|
| 291 |
+
- -0.5596316456794739
|
| 292 |
+
- -0.4764360189437866
|
| 293 |
+
- -0.7253566384315491
|
| 294 |
+
- -0.9443663954734802
|
| 295 |
+
- -0.4381798803806305
|
| 296 |
+
- -0.3338131606578827
|
| 297 |
+
- -0.667724072933197
|
| 298 |
+
- -0.6881827116012573
|
| 299 |
+
- -0.7544379830360413
|
| 300 |
+
- -0.5189417600631714
|
| 301 |
+
- -0.4484957158565521
|
| 302 |
+
- -0.13709338009357452
|
| 303 |
+
- -0.07360810041427612
|
| 304 |
+
- -0.4748336970806122
|
| 305 |
+
- 0.7400000095367432
|
| 306 |
+
- 0
|
| 307 |
+
- 0
|
| 308 |
+
- 0
|
| 309 |
+
- 0
|
| 310 |
+
use_norm_mask: false
|
| 311 |
+
model:
|
| 312 |
+
center_crop:
|
| 313 |
+
size:
|
| 314 |
+
- 224
|
| 315 |
+
- 224
|
| 316 |
+
color_jitter:
|
| 317 |
+
brightness: 0.2
|
| 318 |
+
contrast:
|
| 319 |
+
- 0.8
|
| 320 |
+
- 1.2
|
| 321 |
+
hue: 0.05
|
| 322 |
+
saturation:
|
| 323 |
+
- 0.8
|
| 324 |
+
- 1.2
|
| 325 |
+
img_aug: true
|
| 326 |
+
normalize:
|
| 327 |
+
mean:
|
| 328 |
+
- 0.485
|
| 329 |
+
- 0.456
|
| 330 |
+
- 0.406
|
| 331 |
+
std:
|
| 332 |
+
- 0.229
|
| 333 |
+
- 0.224
|
| 334 |
+
- 0.225
|
| 335 |
+
resize:
|
| 336 |
+
size:
|
| 337 |
+
- 256
|
| 338 |
+
- 480
|
| 339 |
+
repack:
|
| 340 |
+
action_chunk_size: 16
|
| 341 |
+
dataset_name: simple
|
| 342 |
+
num_past_frames: 0
|
| 343 |
+
pad_action_dim: 36
|
| 344 |
+
pad_state_dim: 36
|
| 345 |
+
val_repo_ids:
|
| 346 |
+
- G1WholebodyHandover-v0
|
| 347 |
+
debug:
|
| 348 |
+
value: false
|
| 349 |
+
environment_variables:
|
| 350 |
+
value:
|
| 351 |
+
CUDA_VISIBLE_DEVICES: 4,5,6,7
|
| 352 |
+
DATA_HOME: /data/data
|
| 353 |
+
HF_HOME: /data/cache
|
| 354 |
+
HF_LEROBOT_HOME: /data/data/lerobot
|
| 355 |
+
HF_TOKEN: hf_...TiKa
|
| 356 |
+
LOCAL_RANK: "0"
|
| 357 |
+
LOCAL_WORLD_SIZE: "4"
|
| 358 |
+
MASTER_ADDR: nebula99
|
| 359 |
+
MASTER_PORT: "38477"
|
| 360 |
+
OMP_NUM_THREADS: "32"
|
| 361 |
+
PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python
|
| 362 |
+
RANK: "0"
|
| 363 |
+
TORCH_HOME: /data/cache
|
| 364 |
+
UV_CACHE_DIR: /data/cache
|
| 365 |
+
WANDB_API_KEY: 90e...5c06
|
| 366 |
+
WE_HOME: Not Set
|
| 367 |
+
WORLD_SIZE: "4"
|
| 368 |
+
eval:
|
| 369 |
+
value: false
|
| 370 |
+
exp:
|
| 371 |
+
value: g1wholebodyhandover-v0
|
| 372 |
+
log:
|
| 373 |
+
value:
|
| 374 |
+
log_freq: 100
|
| 375 |
+
logging_dir: logs
|
| 376 |
+
report_to: wandb
|
| 377 |
+
model:
|
| 378 |
+
value:
|
| 379 |
+
action_chunk_size: 16
|
| 380 |
+
action_dim: 36
|
| 381 |
+
action_exec_horizon: 6
|
| 382 |
+
num_cameras: 1
|
| 383 |
+
num_diffusion_iters: 100
|
| 384 |
+
obs_dim: 36
|
| 385 |
+
obs_horizon: 1
|
| 386 |
+
share_vision_encoder: false
|
| 387 |
+
seed:
|
| 388 |
+
value: 2026
|
| 389 |
+
timestamp:
|
| 390 |
+
value: "2604100204"
|
| 391 |
+
train:
|
| 392 |
+
value:
|
| 393 |
+
checkpointing_steps: 5000
|
| 394 |
+
data_parallel: ddp
|
| 395 |
+
deepspeed_config: /data/jliu/psi/src/InternVLA/config/deepseeds/zero3.json
|
| 396 |
+
enable_gradient_checkpointing: true
|
| 397 |
+
enable_mixed_precision_training: true
|
| 398 |
+
gradient_accumulation_steps: 1
|
| 399 |
+
hf_token: .hf_token
|
| 400 |
+
learning_rate: 0.0001
|
| 401 |
+
lora: false
|
| 402 |
+
lr_scheduler_kwargs:
|
| 403 |
+
betas:
|
| 404 |
+
- 0.95
|
| 405 |
+
- 0.999
|
| 406 |
+
eps: 1e-08
|
| 407 |
+
weight_decay: 1e-06
|
| 408 |
+
lr_scheduler_type: cosine
|
| 409 |
+
max_checkpoints_to_keep: null
|
| 410 |
+
max_grad_norm: 1
|
| 411 |
+
max_training_steps: 40000
|
| 412 |
+
mixed_precision: bf16
|
| 413 |
+
name: diffusion-policy-g1
|
| 414 |
+
num_train_epochs: null
|
| 415 |
+
num_workers: 8
|
| 416 |
+
optimizer_foreach: null
|
| 417 |
+
output_dir: .runs
|
| 418 |
+
overfit_single_batch: false
|
| 419 |
+
reduce_in_full_precision: true
|
| 420 |
+
resume_from_checkpoint: null
|
| 421 |
+
sharding_strategy: full-shard
|
| 422 |
+
skip_resumed_steps: false
|
| 423 |
+
train_batch_size: 32
|
| 424 |
+
val_batch_size: 16
|
| 425 |
+
val_num_batches: 20
|
| 426 |
+
validation_steps: 500
|
| 427 |
+
warmup_ratio: null
|
| 428 |
+
warmup_steps: 1000
|
| 429 |
+
wandb:
|
| 430 |
+
value:
|
| 431 |
+
entity: jliu530-soochow-university
|
| 432 |
+
group: null
|
| 433 |
+
id: null
|
| 434 |
+
name: null
|
| 435 |
+
project: psi
|
| 436 |
+
resume: allow
|
dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/wandb/run-20260410_020501-sjem5sh4/files/output.log
ADDED
|
@@ -0,0 +1,230 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[2;36m[02:05:02 04/10][0m[2;36m [0m[34mINFO [0m | >> [1m[[0m*[1m][0m Saved configuration to ]8;id=805427;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=514750;file:///data/jliu/psi/scripts/train.py#128\[2m128[0m]8;;\
|
| 2 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m [2m [0m
|
| 3 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0m*[1m][0m Training configurations: ]8;id=2657;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=644039;file:///data/jliu/psi/scripts/train.py#181\[2m181[0m]8;;\
|
| 4 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> training task: [32m'diffusion-policy-g1'[0m ]8;id=471497;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=12046;file:///data/jliu/psi/scripts/train.py#182\[2m182[0m]8;;\
|
| 5 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> run name: g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m ]8;id=329497;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=220436;file:///data/jliu/psi/scripts/train.py#183\[2m183[0m]8;;\
|
| 6 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> seed: [1;36m2026[0m ]8;id=844246;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=394625;file:///data/jliu/psi/scripts/train.py#184\[2m184[0m]8;;\
|
| 7 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> mixed precision: torch.bfloat16 ]8;id=356929;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=94020;file:///data/jliu/psi/scripts/train.py#185\[2m185[0m]8;;\
|
| 8 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> warmup steps: [1;36m1000[0m ]8;id=147732;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=919050;file:///data/jliu/psi/scripts/train.py#186\[2m186[0m]8;;\
|
| 9 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> validation steps: [1;36m500[0m ]8;id=25941;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=888976;file:///data/jliu/psi/scripts/train.py#187\[2m187[0m]8;;\
|
| 10 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> checkpoint steps: [1;36m5000[0m ]8;id=95089;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=418144;file:///data/jliu/psi/scripts/train.py#188\[2m188[0m]8;;\
|
| 11 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> max gradient norm: [1;36m1.0[0m ]8;id=122104;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=446109;file:///data/jliu/psi/scripts/train.py#189\[2m189[0m]8;;\
|
| 12 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> PyTorch version [1;36m2.8[0m.[1;36m0[0m available. ]8;id=514883;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/datasets/config.py\[2mconfig.py[0m]8;;\[2m:[0m]8;id=417488;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/datasets/config.py#54\[2m54[0m]8;;\
|
| 13 |
+
Resolving data files: 100%|███████████████████████████████████████████████████████████████████████████████████| 100/100 [00:00<00:00, 229699.01it/s]
|
| 14 |
+
Resolving data files: 100%|███████████████████████████████████████████████████████████████████████████████████| 100/100 [00:00<00:00, 195174.69it/s]
|
| 15 |
+
[2;36m[02:05:04 04/10][0m[2;36m [0m[34mINFO [0m | >> |=> Num training samples: ]8;id=591396;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=971225;file:///data/jliu/psi/scripts/train.py#192\[2m192[0m]8;;\
|
| 16 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Training dataset size: [1;36m45[0m,[1;36m530[0m ]8;id=603472;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=235431;file:///data/jliu/psi/scripts/train.py#193\[2m193[0m]8;;\
|
| 17 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Val dataset size: [1;36m45[0m,[1;36m530[0m ]8;id=548632;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=929903;file:///data/jliu/psi/scripts/train.py#195\[2m195[0m]8;;\
|
| 18 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0m*[1m][0m Initialize optimizers and schedulers[33m...[0m ]8;id=492873;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=149330;file:///data/jliu/psi/scripts/train.py#198\[2m198[0m]8;;\
|
| 19 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0m*[1m][0m ***** Running training ***** ]8;id=127940;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=657111;file:///data/jliu/psi/scripts/train.py#202\[2m202[0m]8;;\
|
| 20 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Num training examples = [1;36m45530[0m ]8;id=974614;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=572136;file:///data/jliu/psi/scripts/train.py#203\[2m203[0m]8;;\
|
| 21 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Max training Epochs = [1;36m113[0m ]8;id=574775;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=838653;file:///data/jliu/psi/scripts/train.py#204\[2m204[0m]8;;\
|
| 22 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Total optimization steps = [1;36m40000[0m ]8;id=927413;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=421473;file:///data/jliu/psi/scripts/train.py#205\[2m205[0m]8;;\
|
| 23 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Num steps Per Epoch = [1;36m356[0m ]8;id=9727;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=34717;file:///data/jliu/psi/scripts/train.py#206\[2m206[0m]8;;\
|
| 24 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Effective training epochs = [1;36m449.44[0m ]8;id=842580;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=548288;file:///data/jliu/psi/scripts/train.py#207\[2m207[0m]8;;\
|
| 25 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Global train batch size [1m([0mw. parallel, distributed & accumulation[1m)[0m = [1;36m128[0m ]8;id=272745;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=726293;file:///data/jliu/psi/scripts/train.py#208\[2m208[0m]8;;\
|
| 26 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Device train batch size = [1;36m32[0m ]8;id=521307;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=847206;file:///data/jliu/psi/scripts/train.py#209\[2m209[0m]8;;\
|
| 27 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Gradient Accumulation steps = [1;36m1[0m ]8;id=871789;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=304159;file:///data/jliu/psi/scripts/train.py#210\[2m210[0m]8;;\
|
| 28 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> |=> Num processes [1m([0mGPUs[1m)[0m = [1;36m4[0m ]8;id=459749;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=722818;file:///data/jliu/psi/scripts/train.py#211\[2m211[0m]8;;\
|
| 29 |
+
[2;36m[02:05:05 04/10][0m[2;36m [0m[34mINFO [0m | >> [1m[[0m*[1m][0m Accelerator runs in: ]8;id=966756;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=191940;file:///data/jliu/psi/scripts/train.py#218\[2m218[0m]8;;\
|
| 30 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m [2m [0m
|
| 31 |
+
Traing steps: 0%| | 0/40000 [00:00<?, ?it/s]/data/jliu/psi/.venv-dp/lib/python3.10/site-packages/transformers/deepspeed.py:24: FutureWarning: transformers.deepspeed module is deprecated and will be removed in a future version. Please import deepspeed modules directly from transformers.integrations
|
| 32 |
+
warnings.warn(
|
| 33 |
+
Traing steps: 25%|████████████████▏ | 9999/40000 [49:36<1:10:18, 7.11it/s, loss=0.0173, lr=8.7e-05]
|
| 34 |
+
|
| 35 |
+
[2;36m[02:30:02 04/10][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Saving current state to ]8;id=680052;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=279409;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/accelerator.py#3632\[2m3632[0m]8;;\
|
| 36 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints [2m [0m
|
| 37 |
+
[2;36m [0m /ckpt_5000 [2m [0m
|
| 38 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Model weights saved in ]8;id=898249;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=431290;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 39 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoint [2m [0m
|
| 40 |
+
[2;36m [0m s/ckpt_5000/model.safetensors [2m [0m
|
| 41 |
+
[2;36m[02:30:03 04/10][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Optimizer state saved in ]8;id=224;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=473149;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 42 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoint [2m [0m
|
| 43 |
+
[2;36m [0m s/ckpt_5000/optimizer.bin [2m [0m
|
| 44 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Scheduler state saved in ]8;id=639771;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=832328;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#129\[2m129[0m]8;;\
|
| 45 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoint [2m [0m
|
| 46 |
+
[2;36m [0m s/ckpt_5000/scheduler.bin [2m [0m
|
| 47 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Sampler state for dataloader [1;36m0[0m saved in ]8;id=788377;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=572992;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#146\[2m146[0m]8;;\
|
| 48 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoint [2m [0m
|
| 49 |
+
[2;36m [0m s/ckpt_5000/sampler.bin [2m [0m
|
| 50 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Sampler state for dataloader [1;36m1[0m saved in ]8;id=374604;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=162278;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#146\[2m146[0m]8;;\
|
| 51 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoint [2m [0m
|
| 52 |
+
[2;36m [0m s/ckpt_5000/sampler_1.bin [2m [0m
|
| 53 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Random states saved in ]8;id=806908;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=971194;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#179\[2m179[0m]8;;\
|
| 54 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoint [2m [0m
|
| 55 |
+
[2;36m [0m s/ckpt_5000/random_states_0.pkl [2m [0m
|
| 56 |
+
Saved state to .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/checkpoints/ckpt_5000
|
| 57 |
+
[2;36m[02:54:44 04/10][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Saving current state to ]8;id=755470;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=983054;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/accelerator.py#3632\[2m3632[0m]8;;\
|
| 58 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_10000 [2m [0m
|
| 59 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Model weights saved in ]8;id=616494;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=137590;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 60 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_10000/model [2m [0m
|
| 61 |
+
[2;36m [0m .safetensors [2m [0m
|
| 62 |
+
[2;36m[02:54:45 04/10][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Optimizer state saved in ]8;id=304800;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=88534;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 63 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_10000/optim [2m [0m
|
| 64 |
+
[2;36m [0m izer.bin [2m [0m
|
| 65 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Scheduler state saved in ]8;id=141920;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=821494;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#129\[2m129[0m]8;;\
|
| 66 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_10000/sched [2m [0m
|
| 67 |
+
[2;36m [0m uler.bin [2m [0m
|
| 68 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Sampler state for dataloader [1;36m0[0m saved in ]8;id=532364;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=432054;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#146\[2m146[0m]8;;\
|
| 69 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_10000/sampl [2m [0m
|
| 70 |
+
[2;36m [0m er.bin [2m [0m
|
| 71 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Sampler state for dataloader [1;36m1[0m saved in ]8;id=846104;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=206251;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#146\[2m146[0m]8;;\
|
| 72 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_10000/sampl [2m [0m
|
| 73 |
+
[2;36m [0m er_1.bin [2m [0m
|
| 74 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Random states saved in ]8;id=930895;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=77000;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#179\[2m179[0m]8;;\
|
| 75 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_10000/rando [2m [0m
|
| 76 |
+
[2;36m [0m m_states_0.pkl [2m [0m
|
| 77 |
+
Saved state to .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/checkpoints/ckpt_10000
|
| 78 |
+
[2;36m[03:19:31 04/10][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Saving current state to ]8;id=370569;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=979940;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/accelerator.py#3632\[2m3632[0m]8;;\
|
| 79 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_15000 [2m [0m
|
| 80 |
+
[2;36m[03:19:32 04/10][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Model weights saved in ]8;id=535180;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=499147;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 81 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_15000/model [2m [0m
|
| 82 |
+
[2;36m [0m .safetensors [2m [0m
|
| 83 |
+
[2;36m[03:19:33 04/10][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Optimizer state saved in ]8;id=341880;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=958353;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 84 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_15000/optim [2m [0m
|
| 85 |
+
[2;36m [0m izer.bin [2m [0m
|
| 86 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Scheduler state saved in ]8;id=599385;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=648655;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#129\[2m129[0m]8;;\
|
| 87 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_15000/sched [2m [0m
|
| 88 |
+
[2;36m [0m uler.bin [2m [0m
|
| 89 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Sampler state for dataloader [1;36m0[0m saved in ]8;id=544854;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=217040;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#146\[2m146[0m]8;;\
|
| 90 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_15000/sampl [2m [0m
|
| 91 |
+
[2;36m [0m er.bin [2m [0m
|
| 92 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Sampler state for dataloader [1;36m1[0m saved in ]8;id=627278;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=244660;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#146\[2m146[0m]8;;\
|
| 93 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_15000/sampl [2m [0m
|
| 94 |
+
[2;36m [0m er_1.bin [2m [0m
|
| 95 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Random states saved in ]8;id=418546;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=35811;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#179\[2m179[0m]8;;\
|
| 96 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_15000/rando [2m [0m
|
| 97 |
+
[2;36m [0m m_states_0.pkl [2m [0m
|
| 98 |
+
Saved state to .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/checkpoints/ckpt_15000
|
| 99 |
+
[2;36m[03:44:13 04/10][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Saving current state to ]8;id=47434;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=679719;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/accelerator.py#3632\[2m3632[0m]8;;\
|
| 100 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_20000 [2m [0m
|
| 101 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Model weights saved in ]8;id=473768;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=910263;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 102 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_20000/model [2m [0m
|
| 103 |
+
[2;36m [0m .safetensors [2m [0m
|
| 104 |
+
[2;36m[03:44:14 04/10][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Optimizer state saved in ]8;id=745480;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=711599;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 105 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_20000/optim [2m [0m
|
| 106 |
+
[2;36m [0m izer.bin [2m [0m
|
| 107 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Scheduler state saved in ]8;id=555678;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=229348;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#129\[2m129[0m]8;;\
|
| 108 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_20000/sched [2m [0m
|
| 109 |
+
[2;36m [0m uler.bin [2m [0m
|
| 110 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Sampler state for dataloader [1;36m0[0m saved in ]8;id=144411;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=168181;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#146\[2m146[0m]8;;\
|
| 111 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_20000/sampl [2m [0m
|
| 112 |
+
[2;36m [0m er.bin [2m [0m
|
| 113 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Sampler state for dataloader [1;36m1[0m saved in ]8;id=286058;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=446016;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#146\[2m146[0m]8;;\
|
| 114 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_20000/sampl [2m [0m
|
| 115 |
+
[2;36m [0m er_1.bin [2m [0m
|
| 116 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Random states saved in ]8;id=401798;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=319720;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#179\[2m179[0m]8;;\
|
| 117 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_20000/rando [2m [0m
|
| 118 |
+
[2;36m [0m m_states_0.pkl [2m [0m
|
| 119 |
+
Saved state to .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/checkpoints/ckpt_20000
|
| 120 |
+
[2;36m[04:08:59 04/10][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Saving current state to ]8;id=486450;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=521773;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/accelerator.py#3632\[2m3632[0m]8;;\
|
| 121 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_25000 [2m [0m
|
| 122 |
+
[2;36m[04:09:00 04/10][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Model weights saved in ]8;id=490419;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=185558;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 123 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_25000/model [2m [0m
|
| 124 |
+
[2;36m [0m .safetensors [2m [0m
|
| 125 |
+
[2;36m[04:09:01 04/10][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Optimizer state saved in ]8;id=125939;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=552863;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 126 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_25000/optim [2m [0m
|
| 127 |
+
[2;36m [0m izer.bin [2m [0m
|
| 128 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Scheduler state saved in ]8;id=855797;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=99159;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#129\[2m129[0m]8;;\
|
| 129 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_25000/sched [2m [0m
|
| 130 |
+
[2;36m [0m uler.bin [2m [0m
|
| 131 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Sampler state for dataloader [1;36m0[0m saved in ]8;id=999999;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=97642;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#146\[2m146[0m]8;;\
|
| 132 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_25000/sampl [2m [0m
|
| 133 |
+
[2;36m [0m er.bin [2m [0m
|
| 134 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Sampler state for dataloader [1;36m1[0m saved in ]8;id=427821;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=571536;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#146\[2m146[0m]8;;\
|
| 135 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_25000/sampl [2m [0m
|
| 136 |
+
[2;36m [0m er_1.bin [2m [0m
|
| 137 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Random states saved in ]8;id=702964;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=191314;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#179\[2m179[0m]8;;\
|
| 138 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_25000/rando [2m [0m
|
| 139 |
+
[2;36m [0m m_states_0.pkl [2m [0m
|
| 140 |
+
Saved state to .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/checkpoints/ckpt_25000
|
| 141 |
+
[2;36m[04:34:04 04/10][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Saving current state to ]8;id=368785;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=589302;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/accelerator.py#3632\[2m3632[0m]8;;\
|
| 142 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_30000 [2m [0m
|
| 143 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Model weights saved in ]8;id=227061;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=115021;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 144 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_30000/m [2m [0m
|
| 145 |
+
[2;36m [0m odel.safetensors [2m [0m
|
| 146 |
+
[2;36m[04:34:05 04/10][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Optimizer state saved in ]8;id=787738;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=595634;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 147 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_30000/o [2m [0m
|
| 148 |
+
[2;36m [0m ptimizer.bin [2m [0m
|
| 149 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Scheduler state saved in ]8;id=590171;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=281123;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#129\[2m129[0m]8;;\
|
| 150 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_30000/s [2m [0m
|
| 151 |
+
[2;36m [0m cheduler.bin [2m [0m
|
| 152 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Sampler state for dataloader [1;36m0[0m saved in ]8;id=819504;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=276522;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#146\[2m146[0m]8;;\
|
| 153 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_30000/s [2m [0m
|
| 154 |
+
[2;36m [0m ampler.bin [2m [0m
|
| 155 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Sampler state for dataloader [1;36m1[0m saved in ]8;id=717799;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=63361;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#146\[2m146[0m]8;;\
|
| 156 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_30000/s [2m [0m
|
| 157 |
+
[2;36m [0m ampler_1.bin [2m [0m
|
| 158 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Random states saved in ]8;id=194505;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=677321;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#179\[2m179[0m]8;;\
|
| 159 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_30000/r [2m [0m
|
| 160 |
+
[2;36m [0m andom_states_0.pkl [2m [0m
|
| 161 |
+
Saved state to .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/checkpoints/ckpt_30000
|
| 162 |
+
[2;36m[04:59:40 04/10][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Saving current state to ]8;id=290231;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=616676;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/accelerator.py#3632\[2m3632[0m]8;;\
|
| 163 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_35000 [2m [0m
|
| 164 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Model weights saved in ]8;id=329883;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=169230;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 165 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_35000/m [2m [0m
|
| 166 |
+
[2;36m [0m odel.safetensors [2m [0m
|
| 167 |
+
[2;36m[04:59:41 04/10][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Optimizer state saved in ]8;id=431638;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=908584;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 168 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_35000/o [2m [0m
|
| 169 |
+
[2;36m [0m ptimizer.bin [2m [0m
|
| 170 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Scheduler state saved in ]8;id=434664;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=16179;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#129\[2m129[0m]8;;\
|
| 171 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_35000/s [2m [0m
|
| 172 |
+
[2;36m [0m cheduler.bin [2m [0m
|
| 173 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Sampler state for dataloader [1;36m0[0m saved in ]8;id=324973;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=128996;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#146\[2m146[0m]8;;\
|
| 174 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_35000/s [2m [0m
|
| 175 |
+
[2;36m [0m ampler.bin [2m [0m
|
| 176 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Sampler state for dataloader [1;36m1[0m saved in ]8;id=981583;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=899091;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#146\[2m146[0m]8;;\
|
| 177 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_35000/s [2m [0m
|
| 178 |
+
[2;36m [0m ampler_1.bin [2m [0m
|
| 179 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Random states saved in ]8;id=636753;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=656123;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#179\[2m179[0m]8;;\
|
| 180 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m2604100204[0m/checkpoints/ckpt_35000/r [2m [0m
|
| 181 |
+
[2;36m [0m andom_states_0.pkl [2m [0m
|
| 182 |
+
Saved state to .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/checkpoints/ckpt_35000
|
| 183 |
+
[2;36m[05:25:16 04/10][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Saving current state to ]8;id=969664;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=423824;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/accelerator.py#3632\[2m3632[0m]8;;\
|
| 184 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m260[0m [2m [0m
|
| 185 |
+
[2;36m [0m [1;36m4100204[0m/checkpoints/ckpt_40000 [2m [0m
|
| 186 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Model weights saved in ]8;id=485144;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=446823;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 187 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m26[0m [2m [0m
|
| 188 |
+
[2;36m [0m [1;36m04100204[0m/checkpoints/ckpt_40000/model.safetensors [2m [0m
|
| 189 |
+
[2;36m[05:25:17 04/10][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Optimizer state saved in ]8;id=392321;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=903139;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 190 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m26[0m [2m [0m
|
| 191 |
+
[2;36m [0m [1;36m04100204[0m/checkpoints/ckpt_40000/optimizer.bin [2m [0m
|
| 192 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Scheduler state saved in ]8;id=407945;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=306059;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#129\[2m129[0m]8;;\
|
| 193 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m26[0m [2m [0m
|
| 194 |
+
[2;36m [0m [1;36m04100204[0m/checkpoints/ckpt_40000/scheduler.bin [2m [0m
|
| 195 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Sampler state for dataloader [1;36m0[0m saved in ]8;id=771109;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=749912;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#146\[2m146[0m]8;;\
|
| 196 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m26[0m [2m [0m
|
| 197 |
+
[2;36m [0m [1;36m04100204[0m/checkpoints/ckpt_40000/sampler.bin [2m [0m
|
| 198 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Sampler state for dataloader [1;36m1[0m saved in ]8;id=729075;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=346604;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#146\[2m146[0m]8;;\
|
| 199 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m26[0m [2m [0m
|
| 200 |
+
[2;36m [0m [1;36m04100204[0m/checkpoints/ckpt_40000/sampler_1.bin [2m [0m
|
| 201 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Random states saved in ]8;id=960350;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=128256;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#179\[2m179[0m]8;;\
|
| 202 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m26[0m [2m [0m
|
| 203 |
+
[2;36m [0m [1;36m04100204[0m/checkpoints/ckpt_40000/random_states_0.pkl [2m [0m
|
| 204 |
+
Saved state to .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/checkpoints/ckpt_40000
|
| 205 |
+
Training has reached maximum steps.
|
| 206 |
+
[2;36m[05:25:45 04/10][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Saving current state to ]8;id=614052;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/accelerator.py\[2maccelerator.py[0m]8;;\[2m:[0m]8;id=946822;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/accelerator.py#3632\[2m3632[0m]8;;\
|
| 207 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m260[0m [2m [0m
|
| 208 |
+
[2;36m [0m [1;36m4100204[0m/checkpoints/ckpt_39999 [2m [0m
|
| 209 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Model weights saved in ]8;id=588107;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=58872;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#115\[2m115[0m]8;;\
|
| 210 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m26[0m [2m [0m
|
| 211 |
+
[2;36m [0m [1;36m04100204[0m/checkpoints/ckpt_39999/model.safetensors [2m [0m
|
| 212 |
+
[2;36m[05:25:46 04/10][0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Optimizer state saved in ]8;id=128644;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=124028;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#122\[2m122[0m]8;;\
|
| 213 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m26[0m [2m [0m
|
| 214 |
+
[2;36m [0m [1;36m04100204[0m/checkpoints/ckpt_39999/optimizer.bin [2m [0m
|
| 215 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Scheduler state saved in ]8;id=625950;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=679365;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#129\[2m129[0m]8;;\
|
| 216 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m26[0m [2m [0m
|
| 217 |
+
[2;36m [0m [1;36m04100204[0m/checkpoints/ckpt_39999/scheduler.bin [2m [0m
|
| 218 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Sampler state for dataloader [1;36m0[0m saved in ]8;id=606210;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=132432;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#146\[2m146[0m]8;;\
|
| 219 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m26[0m [2m [0m
|
| 220 |
+
[2;36m [0m [1;36m04100204[0m/checkpoints/ckpt_39999/sampler.bin [2m [0m
|
| 221 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Sampler state for dataloader [1;36m1[0m saved in ]8;id=250930;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=814323;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#146\[2m146[0m]8;;\
|
| 222 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m26[0m [2m [0m
|
| 223 |
+
[2;36m [0m [1;36m04100204[0m/checkpoints/ckpt_39999/sampler_1.bin [2m [0m
|
| 224 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0mRANK [1;36m0[0m[1m][0m Random states saved in ]8;id=131290;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py\[2mcheckpointing.py[0m]8;;\[2m:[0m]8;id=192379;file:///data/jliu/psi/.venv-dp/lib/python3.10/site-packages/accelerate/checkpointing.py#179\[2m179[0m]8;;\
|
| 225 |
+
[2;36m [0m .runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.[1;36m0e-04[0m.b128.gpus4.[1;36m26[0m [2m [0m
|
| 226 |
+
[2;36m [0m [1;36m04100204[0m/checkpoints/ckpt_39999/random_states_0.pkl [2m [0m
|
| 227 |
+
[2;36m[05:25:47 04/10][0m[2;36m [0m[34mINFO [0m | >> [1m[[0m*[1m][0m Finalized DP Trainer. Epoch losses: [1m[[0m[1;36m0.0[0m, [1;36m0.6723278164863586[0m, ]8;id=567626;file:///data/jliu/psi/src/psi/trainers/diffusion_policy_g1.py\[2mdiffusion_policy_g1.py[0m]8;;\[2m:[0m]8;id=716966;file:///data/jliu/psi/src/psi/trainers/diffusion_policy_g1.py#360\[2m360[0m]8;;\
|
| 228 |
+
[2;36m [0m [1;36m0.2651745080947876[0m, [1;36m0.13162203133106232[0m, [1;36m0.11490532010793686[0m, [2m [0m
|
| 229 |
+
[2;36m [0m [1;36m0.06330759078264236[0m[1m][0m [2m [0m
|
| 230 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> [1m[[0m*[1m][0m Happy Ending! ]8;id=154857;file:///data/jliu/psi/scripts/train.py\[2mtrain.py[0m]8;;\[2m:[0m]8;id=239343;file:///data/jliu/psi/scripts/train.py#310\[2m310[0m]8;;\
|
dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/wandb/run-20260410_020501-sjem5sh4/files/requirements.txt
ADDED
|
@@ -0,0 +1,205 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
tifffile==2025.5.10
|
| 2 |
+
mpmath==1.3.0
|
| 3 |
+
pynput==1.8.1
|
| 4 |
+
llvmlite==0.46.0
|
| 5 |
+
exceptiongroup==1.3.0
|
| 6 |
+
sympy==1.14.0
|
| 7 |
+
aiohttp-cors==0.8.1
|
| 8 |
+
soupsieve==2.8
|
| 9 |
+
Farama-Notifications==0.0.4
|
| 10 |
+
numpydantic==1.6.7
|
| 11 |
+
uvicorn==0.38.0
|
| 12 |
+
waterbear==2.6.8
|
| 13 |
+
nvidia-cudnn-cu12==9.10.2.21
|
| 14 |
+
transformers==4.44.2
|
| 15 |
+
cmake==4.2.3
|
| 16 |
+
xxhash==3.5.0
|
| 17 |
+
annotated-doc==0.0.4
|
| 18 |
+
jsonlines==4.0.0
|
| 19 |
+
nvidia-cuda-nvrtc-cu12==12.8.93
|
| 20 |
+
pydantic==2.10.6
|
| 21 |
+
triton==3.4.0
|
| 22 |
+
nvidia-cufft-cu12==11.3.3.83
|
| 23 |
+
multidict==6.6.4
|
| 24 |
+
setuptools==82.0.1
|
| 25 |
+
shtab==1.7.2
|
| 26 |
+
pycollada==0.9.2
|
| 27 |
+
beautifulsoup4==4.14.2
|
| 28 |
+
msgspec==0.19.0
|
| 29 |
+
huggingface-hub==0.35.3
|
| 30 |
+
pydantic_core==2.27.2
|
| 31 |
+
iniconfig==2.3.0
|
| 32 |
+
zarr==2.18.3
|
| 33 |
+
pytz==2025.2
|
| 34 |
+
attrs==25.3.0
|
| 35 |
+
requests==2.32.5
|
| 36 |
+
hf_transfer==0.1.9
|
| 37 |
+
numba==0.64.0
|
| 38 |
+
einx==0.3.0
|
| 39 |
+
scipy==1.15.3
|
| 40 |
+
transforms3d==0.4.2
|
| 41 |
+
pyzmq==27.1.0
|
| 42 |
+
aiohappyeyeballs==2.6.1
|
| 43 |
+
pymunk==7.2.0
|
| 44 |
+
six==1.17.0
|
| 45 |
+
expandvars==1.1.2
|
| 46 |
+
prompt_toolkit==3.0.52
|
| 47 |
+
wcwidth==0.6.0
|
| 48 |
+
tomli==2.4.0
|
| 49 |
+
ruamel.yaml.clib==0.2.14
|
| 50 |
+
multiprocess==0.70.16
|
| 51 |
+
vhacdx==0.0.9
|
| 52 |
+
nvidia-cusparse-cu12==12.5.8.93
|
| 53 |
+
aiosignal==1.4.0
|
| 54 |
+
itsdangerous==2.2.0
|
| 55 |
+
av==17.0.0
|
| 56 |
+
nvidia-nccl-cu12==2.27.3
|
| 57 |
+
httpx==0.28.1
|
| 58 |
+
asciitree==0.3.3
|
| 59 |
+
platformdirs==4.4.0
|
| 60 |
+
sentry-sdk==2.39.0
|
| 61 |
+
draccus==0.11.5
|
| 62 |
+
nvidia-curand-cu12==10.3.9.90
|
| 63 |
+
plotly==6.2.0
|
| 64 |
+
jsonschema==4.25.1
|
| 65 |
+
fastapi==0.119.1
|
| 66 |
+
nvidia-nvjitlink-cu12==12.8.93
|
| 67 |
+
fsspec==2025.3.0
|
| 68 |
+
MarkupSafe==3.0.3
|
| 69 |
+
pydantic-yaml==1.6.0
|
| 70 |
+
h11==0.16.0
|
| 71 |
+
typer==0.24.1
|
| 72 |
+
mypy_extensions==1.1.0
|
| 73 |
+
pytest==9.0.2
|
| 74 |
+
h5py==3.14.0
|
| 75 |
+
python-xlib==0.33
|
| 76 |
+
lazy_loader==0.4
|
| 77 |
+
einops==0.8.1
|
| 78 |
+
albumentations==1.4.18
|
| 79 |
+
params_proto==2.13.2
|
| 80 |
+
fasteners==0.20
|
| 81 |
+
psutil==7.1.0
|
| 82 |
+
starlette==0.48.0
|
| 83 |
+
anyio==4.11.0
|
| 84 |
+
gdown==5.2.0
|
| 85 |
+
charset-normalizer==3.4.3
|
| 86 |
+
tyro==0.9.32
|
| 87 |
+
filelock==3.19.1
|
| 88 |
+
websockets==15.0.1
|
| 89 |
+
accelerate==1.13.0
|
| 90 |
+
python-dotenv==1.2.1
|
| 91 |
+
orderly-set==5.5.0
|
| 92 |
+
inquirerpy==0.3.4
|
| 93 |
+
urllib3==2.5.0
|
| 94 |
+
PyYAML==6.0.3
|
| 95 |
+
rerun-sdk==0.23.1
|
| 96 |
+
antlr4-python3-runtime==4.9.3
|
| 97 |
+
mdurl==0.1.2
|
| 98 |
+
omegaconf==2.3.0
|
| 99 |
+
referencing==0.37.0
|
| 100 |
+
pycparser==3.0
|
| 101 |
+
pluggy==1.6.0
|
| 102 |
+
torchvision==0.23.0
|
| 103 |
+
docstring_parser==0.17.0
|
| 104 |
+
wandb==0.25.1
|
| 105 |
+
numpy==1.26.4
|
| 106 |
+
GitPython==3.1.45
|
| 107 |
+
opencv-python-headless==4.11.0.86
|
| 108 |
+
cffi==2.0.0
|
| 109 |
+
yourdfpy==0.0.58
|
| 110 |
+
async-timeout==4.0.3
|
| 111 |
+
shapely==2.1.2
|
| 112 |
+
frozenlist==1.7.0
|
| 113 |
+
simplejpeg==1.9.0
|
| 114 |
+
nvidia-cuda-cupti-cu12==12.8.90
|
| 115 |
+
Pygments==2.19.2
|
| 116 |
+
rtree==1.4.1
|
| 117 |
+
gitdb==4.0.12
|
| 118 |
+
colorlog==6.10.1
|
| 119 |
+
Werkzeug==3.1.6
|
| 120 |
+
zipp==3.23.0
|
| 121 |
+
albucore==0.0.17
|
| 122 |
+
imageio==2.34.2
|
| 123 |
+
frozendict==2.4.6
|
| 124 |
+
jsonschema-specifications==2025.9.1
|
| 125 |
+
rpds-py==0.28.0
|
| 126 |
+
tokenizers==0.19.1
|
| 127 |
+
blinker==1.9.0
|
| 128 |
+
argparse==1.4.0
|
| 129 |
+
nvidia-nvtx-cu12==12.8.90
|
| 130 |
+
msgpack==1.1.1
|
| 131 |
+
smmap==5.0.2
|
| 132 |
+
packaging==25.0
|
| 133 |
+
embreex==2.17.7.post7
|
| 134 |
+
cloudpickle==3.1.2
|
| 135 |
+
vuer==0.0.68
|
| 136 |
+
propcache==0.3.2
|
| 137 |
+
torchaudio==2.8.0
|
| 138 |
+
pfzy==0.3.4
|
| 139 |
+
dm-tree==0.1.8
|
| 140 |
+
nvidia-cuda-runtime-cu12==12.8.90
|
| 141 |
+
datasets==3.6.0
|
| 142 |
+
numcodecs==0.13.1
|
| 143 |
+
sniffio==1.3.1
|
| 144 |
+
Jinja2==3.1.6
|
| 145 |
+
narwhals==2.6.0
|
| 146 |
+
gymnasium==0.29.1
|
| 147 |
+
yarl==1.20.1
|
| 148 |
+
typeguard==4.4.4
|
| 149 |
+
termcolor==3.1.0
|
| 150 |
+
importlib_metadata==8.7.1
|
| 151 |
+
eval_type_backport==0.2.2
|
| 152 |
+
certifi==2025.8.3
|
| 153 |
+
tzdata==2025.2
|
| 154 |
+
typing_extensions==4.15.0
|
| 155 |
+
viser==1.0.15
|
| 156 |
+
typing-inspect==0.9.0
|
| 157 |
+
torch==2.8.0
|
| 158 |
+
annotated-types==0.7.0
|
| 159 |
+
svg.path==7.0
|
| 160 |
+
mergedeep==1.3.4
|
| 161 |
+
psi==0.0.0
|
| 162 |
+
deepdiff==8.6.1
|
| 163 |
+
nvidia-cufile-cu12==1.13.1.3
|
| 164 |
+
toml==0.10.2
|
| 165 |
+
click==8.3.0
|
| 166 |
+
diffusers==0.30.2
|
| 167 |
+
dotenv==0.9.9
|
| 168 |
+
trimesh==4.8.3
|
| 169 |
+
nvidia-cusparselt-cu12==0.7.1
|
| 170 |
+
pyyaml-include==1.4.1
|
| 171 |
+
scikit-image==0.25.2
|
| 172 |
+
PySocks==1.7.1
|
| 173 |
+
aiohttp==3.10.5
|
| 174 |
+
regex==2026.2.28
|
| 175 |
+
opencv-python==4.11.0.86
|
| 176 |
+
nodeenv==1.9.1
|
| 177 |
+
torchcodec==0.6.0
|
| 178 |
+
protobuf==6.33.2
|
| 179 |
+
pillow==11.3.0
|
| 180 |
+
nvidia-cusolver-cu12==11.7.3.90
|
| 181 |
+
idna==3.10
|
| 182 |
+
Flask==3.1.3
|
| 183 |
+
lerobot==0.1.0
|
| 184 |
+
safetensors==0.7.0
|
| 185 |
+
killport==1.2.0
|
| 186 |
+
hf-xet==1.1.10
|
| 187 |
+
nvidia-cublas-cu12==12.8.4.1
|
| 188 |
+
imageio-ffmpeg==0.6.0
|
| 189 |
+
httpcore==1.0.9
|
| 190 |
+
networkx==3.4.2
|
| 191 |
+
ruamel.yaml==0.18.15
|
| 192 |
+
manifold3d==3.2.1
|
| 193 |
+
shellingham==1.5.4
|
| 194 |
+
mapbox_earcut==1.0.3
|
| 195 |
+
pyarrow==21.0.0
|
| 196 |
+
dill==0.3.8
|
| 197 |
+
markdown-it-py==4.0.0
|
| 198 |
+
tqdm==4.67.1
|
| 199 |
+
argcomplete==3.6.2
|
| 200 |
+
pandas==2.3.3
|
| 201 |
+
lxml==6.0.2
|
| 202 |
+
python-dateutil==2.9.0.post0
|
| 203 |
+
evdev==1.9.3
|
| 204 |
+
rich==14.1.0
|
| 205 |
+
psi==0.0.0
|
dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/wandb/run-20260410_020501-sjem5sh4/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-6.8.0-106-generic-x86_64-with-glibc2.39",
|
| 3 |
+
"python": "CPython 3.10.20",
|
| 4 |
+
"startedAt": "2026-04-10T02:05:01.005456Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"simple_dp_config",
|
| 7 |
+
"--seed=2026",
|
| 8 |
+
"--exp=g1wholebodyhandover-v0",
|
| 9 |
+
"--train.name=diffusion-policy-g1",
|
| 10 |
+
"--log.report-to=wandb",
|
| 11 |
+
"--train.data_parallel=ddp",
|
| 12 |
+
"--train.mixed_precision=bf16",
|
| 13 |
+
"--train.train-batch-size=32",
|
| 14 |
+
"--train.warmup-steps=1000",
|
| 15 |
+
"--train.warmup-ratio=None",
|
| 16 |
+
"--train.checkpointing-steps=5000",
|
| 17 |
+
"--train.validation_steps=500",
|
| 18 |
+
"--train.val_num_batches=20",
|
| 19 |
+
"--train.gradient_accumulation_steps=1",
|
| 20 |
+
"--train.max-training-steps=40000",
|
| 21 |
+
"--train.learning-rate=1e-4",
|
| 22 |
+
"--train.max-grad-norm=1.0",
|
| 23 |
+
"--train.lr_scheduler_kwargs.weight_decay=1e-6",
|
| 24 |
+
"--train.lr_scheduler_kwargs.betas",
|
| 25 |
+
"0.95",
|
| 26 |
+
"0.999",
|
| 27 |
+
"--train.lr_scheduler_type=cosine",
|
| 28 |
+
"--data.root_dir=/data/jliu/data",
|
| 29 |
+
"--data.train-repo-ids=G1WholebodyHandover-v0",
|
| 30 |
+
"--data.transform.repack.action_chunk_size=16",
|
| 31 |
+
"--data.transform.repack.pad-action-dim=36",
|
| 32 |
+
"--data.transform.repack.pad-state-dim=36",
|
| 33 |
+
"--data.transform.field.stat-path=meta/stats_psi0.json",
|
| 34 |
+
"--data.transform.field.stat-action-key=action",
|
| 35 |
+
"--data.transform.field.stat-state-key=states",
|
| 36 |
+
"--data.transform.field.normalize-state",
|
| 37 |
+
"--data.transform.field.action-norm-type=bounds",
|
| 38 |
+
"--data.transform.field.pad-action-dim=36",
|
| 39 |
+
"--data.transform.field.pad-state-dim=36",
|
| 40 |
+
"--data.transform.model.img-aug",
|
| 41 |
+
"--model.action-chunk-size=16",
|
| 42 |
+
"--model.action-dim=36",
|
| 43 |
+
"--model.obs-dim=36"
|
| 44 |
+
],
|
| 45 |
+
"program": "/data/jliu/psi/scripts/train.py",
|
| 46 |
+
"codePath": "scripts/train.py",
|
| 47 |
+
"codePathLocal": "scripts/train.py",
|
| 48 |
+
"git": {
|
| 49 |
+
"remote": "https://github.com/songlin/psi.git",
|
| 50 |
+
"commit": "04614628ecb677f5f278e2d31c1103aed8127f26"
|
| 51 |
+
},
|
| 52 |
+
"email": "jliu530@163.com",
|
| 53 |
+
"root": "/data/jliu/psi/.runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204",
|
| 54 |
+
"host": "nebula99",
|
| 55 |
+
"executable": "/data/jliu/psi/.venv-dp/bin/python3",
|
| 56 |
+
"cpu_count": 128,
|
| 57 |
+
"cpu_count_logical": 128,
|
| 58 |
+
"gpu": "NVIDIA A100-SXM4-80GB",
|
| 59 |
+
"gpu_count": 8,
|
| 60 |
+
"disk": {
|
| 61 |
+
"/": {
|
| 62 |
+
"total": "105089261568",
|
| 63 |
+
"used": "82290688000"
|
| 64 |
+
}
|
| 65 |
+
},
|
| 66 |
+
"memory": {
|
| 67 |
+
"total": "1623177445376"
|
| 68 |
+
},
|
| 69 |
+
"gpu_nvidia": [
|
| 70 |
+
{
|
| 71 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 72 |
+
"memoryTotal": "85899345920",
|
| 73 |
+
"cudaCores": 6912,
|
| 74 |
+
"architecture": "Ampere",
|
| 75 |
+
"uuid": "GPU-02b51758-2aec-8c71-c751-92567fdb15ad"
|
| 76 |
+
},
|
| 77 |
+
{
|
| 78 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 79 |
+
"memoryTotal": "85899345920",
|
| 80 |
+
"cudaCores": 6912,
|
| 81 |
+
"architecture": "Ampere",
|
| 82 |
+
"uuid": "GPU-be645bb4-1ef0-15da-fc6b-53c5378650df"
|
| 83 |
+
},
|
| 84 |
+
{
|
| 85 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 86 |
+
"memoryTotal": "85899345920",
|
| 87 |
+
"cudaCores": 6912,
|
| 88 |
+
"architecture": "Ampere",
|
| 89 |
+
"uuid": "GPU-44868ccd-5809-61e2-f8cc-1dff8efc7eb1"
|
| 90 |
+
},
|
| 91 |
+
{
|
| 92 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 93 |
+
"memoryTotal": "85899345920",
|
| 94 |
+
"cudaCores": 6912,
|
| 95 |
+
"architecture": "Ampere",
|
| 96 |
+
"uuid": "GPU-a12b8de3-fb12-2901-c3a2-1e0e2b050b7b"
|
| 97 |
+
},
|
| 98 |
+
{
|
| 99 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 100 |
+
"memoryTotal": "85899345920",
|
| 101 |
+
"cudaCores": 6912,
|
| 102 |
+
"architecture": "Ampere",
|
| 103 |
+
"uuid": "GPU-57787183-4904-b2c9-cf52-1c1af976b365"
|
| 104 |
+
},
|
| 105 |
+
{
|
| 106 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 107 |
+
"memoryTotal": "85899345920",
|
| 108 |
+
"cudaCores": 6912,
|
| 109 |
+
"architecture": "Ampere",
|
| 110 |
+
"uuid": "GPU-e82d0fd0-9dcb-3d7c-9451-c4770ad6ddc3"
|
| 111 |
+
},
|
| 112 |
+
{
|
| 113 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 114 |
+
"memoryTotal": "85899345920",
|
| 115 |
+
"cudaCores": 6912,
|
| 116 |
+
"architecture": "Ampere",
|
| 117 |
+
"uuid": "GPU-9a825fa3-60d0-3acf-6c79-67e9230ae24c"
|
| 118 |
+
},
|
| 119 |
+
{
|
| 120 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 121 |
+
"memoryTotal": "85899345920",
|
| 122 |
+
"cudaCores": 6912,
|
| 123 |
+
"architecture": "Ampere",
|
| 124 |
+
"uuid": "GPU-6fe02e5e-b886-9c02-e433-6237f3897dd8"
|
| 125 |
+
}
|
| 126 |
+
],
|
| 127 |
+
"cudaVersion": "12.9",
|
| 128 |
+
"writerId": "6ptlrch3t1wewxljiv5hkmtr0grgv5w6"
|
| 129 |
+
}
|
dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/wandb/run-20260410_020501-sjem5sh4/files/wandb-summary.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"val/denorm_err_l1_rpy":0.004311066120862961,"val/denorm_err_l1_hand_joints":0.2085319459438324,"val/denorm_err_l1_torso_vx":0.0048075346276164055,"_timestamp":1.7757987454488413e+09,"_runtime":12045.173678753,"train/epoch":112,"val/denorm_err_l1_torso_vy":0.003890593070536852,"train/loss":0.0026351141277700663,"val/denorm_err_l1_height":0,"_step":40000,"val/bc_loss":0.0043721916154026985,"val/denorm_err_l1_arm_joints":0.027254171669483185,"_wandb":{"runtime":12045},"val/denorm_err_l1_torso_target_yaw":0.0004649795009754598,"val/denorm_err_l1_torso_vyaw":0.004365503787994385}
|
dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/wandb/run-20260410_020501-sjem5sh4/logs/debug-core.log
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-04-10T02:05:01.084000894Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmprpqrafn8/port-1404689.txt","pid":1404689,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
|
| 2 |
+
{"time":"2026-04-10T02:05:01.08475924Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":1404689}
|
| 3 |
+
{"time":"2026-04-10T02:05:01.08470711Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1404689-1607107-1367746566/socket","Net":"unix"}}
|
| 4 |
+
{"time":"2026-04-10T02:05:01.256956327Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
|
| 5 |
+
{"time":"2026-04-10T02:05:01.267618045Z","level":"INFO","msg":"handleInformInit: received","streamId":"sjem5sh4","id":"1(@)"}
|
| 6 |
+
{"time":"2026-04-10T02:05:01.59035368Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"sjem5sh4","id":"1(@)"}
|
| 7 |
+
{"time":"2026-04-10T02:05:07.330804152Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"1v3sozvmfc6a"}
|
| 8 |
+
{"time":"2026-04-10T05:25:47.35681973Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"1v3sozvmfc6a"}
|
| 9 |
+
{"time":"2026-04-10T05:25:48.207051511Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"1v3sozvmfc6a"}
|
| 10 |
+
{"time":"2026-04-10T05:25:48.207779707Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"sjem5sh4","id":"1(@)"}
|
| 11 |
+
{"time":"2026-04-10T05:25:48.208265471Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"sjem5sh4","id":"1(@)"}
|
| 12 |
+
{"time":"2026-04-10T05:26:48.572037497Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
|
| 13 |
+
{"time":"2026-04-10T05:26:48.572102657Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
|
| 14 |
+
{"time":"2026-04-10T05:26:48.572112678Z","level":"INFO","msg":"server is shutting down"}
|
| 15 |
+
{"time":"2026-04-10T05:26:48.572164568Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
|
| 16 |
+
{"time":"2026-04-10T05:26:48.572199678Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1404689-1607107-1367746566/socket","Net":"unix"}}
|
| 17 |
+
{"time":"2026-04-10T05:26:48.572300229Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
|
| 18 |
+
{"time":"2026-04-10T05:26:48.57232799Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
|
| 19 |
+
{"time":"2026-04-10T05:26:48.57234687Z","level":"INFO","msg":"server is closed"}
|
dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/wandb/run-20260410_020501-sjem5sh4/logs/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/wandb/run-20260410_020501-sjem5sh4/logs/debug.log
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-04-10 02:05:01,007 INFO MainThread:1404689 [wandb_setup.py:_flush():81] Current SDK version is 0.25.1
|
| 2 |
+
2026-04-10 02:05:01,007 INFO MainThread:1404689 [wandb_setup.py:_flush():81] Configure stats pid to 1404689
|
| 3 |
+
2026-04-10 02:05:01,007 INFO MainThread:1404689 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-04-10 02:05:01,007 INFO MainThread:1404689 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data/jliu/psi/.runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/wandb/run-20260410_020501-sjem5sh4/logs/debug.log
|
| 5 |
+
2026-04-10 02:05:01,007 INFO MainThread:1404689 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data/jliu/psi/.runs/diffusion-policy-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/wandb/run-20260410_020501-sjem5sh4/logs/debug-internal.log
|
| 6 |
+
2026-04-10 02:05:01,007 INFO MainThread:1404689 [wandb_init.py:init():844] calling init triggers
|
| 7 |
+
2026-04-10 02:05:01,007 INFO MainThread:1404689 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'_wandb': {}}
|
| 9 |
+
2026-04-10 02:05:01,007 INFO MainThread:1404689 [wandb_init.py:init():892] starting backend
|
| 10 |
+
2026-04-10 02:05:01,257 INFO MainThread:1404689 [wandb_init.py:init():895] sending inform_init request
|
| 11 |
+
2026-04-10 02:05:01,265 INFO MainThread:1404689 [wandb_init.py:init():903] backend started and connected
|
| 12 |
+
2026-04-10 02:05:01,266 INFO MainThread:1404689 [wandb_init.py:init():973] updated telemetry
|
| 13 |
+
2026-04-10 02:05:01,272 INFO MainThread:1404689 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-04-10 02:05:02,182 INFO MainThread:1404689 [wandb_init.py:init():1042] starting run threads in backend
|
| 15 |
+
2026-04-10 02:05:02,323 INFO MainThread:1404689 [wandb_run.py:_console_start():2524] atexit reg
|
| 16 |
+
2026-04-10 02:05:02,323 INFO MainThread:1404689 [wandb_run.py:_redirect():2373] redirect: wrap_raw
|
| 17 |
+
2026-04-10 02:05:02,323 INFO MainThread:1404689 [wandb_run.py:_redirect():2442] Wrapping output streams.
|
| 18 |
+
2026-04-10 02:05:02,323 INFO MainThread:1404689 [wandb_run.py:_redirect():2465] Redirects installed.
|
| 19 |
+
2026-04-10 02:05:02,328 INFO MainThread:1404689 [wandb_init.py:init():1082] run started, returning control to user process
|
| 20 |
+
2026-04-10 02:05:02,329 INFO MainThread:1404689 [wandb_run.py:_config_callback():1403] config_cb None None {'exp': 'g1wholebodyhandover-v0', 'seed': 2026, 'auto_tag_run': False, 'eval': False, 'debug': False, 'timestamp': '2604100204', 'log': {'logging_dir': 'logs', 'report_to': 'wandb', 'log_freq': 100}, 'wandb': {'project': 'psi', 'entity': 'jliu530-soochow-university', 'group': None, 'id': None, 'name': None, 'resume': 'allow'}, 'train': {'num_workers': 8, 'overfit_single_batch': False, 'name': 'diffusion-policy-g1', 'resume_from_checkpoint': None, 'skip_resumed_steps': False, 'hf_token': '.hf_token', 'lora': False, 'output_dir': '.runs', 'gradient_accumulation_steps': 1, 'mixed_precision': 'bf16', 'max_grad_norm': 1.0, 'optimizer_foreach': None, 'train_batch_size': 32, 'val_batch_size': 16, 'val_num_batches': 20, 'checkpointing_steps': 5000, 'max_checkpoints_to_keep': None, 'validation_steps': 500, 'learning_rate': 0.0001, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {'betas': [0.95, 0.999], 'weight_decay': 1e-06, 'eps': 1e-08}, 'scheduler_specific_kwargs': {}, 'data_parallel': 'ddp', 'sharding_strategy': 'full-shard', 'deepspeed_config': '/data/jliu/psi/src/InternVLA/config/deepseeds/zero3.json', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': True, 'max_training_steps': 40000, 'num_train_epochs': None, 'warmup_steps': 1000, 'warmup_ratio': None}, 'data': {'transform': {'repack': {'dataset_name': 'simple', 'num_past_frames': 0, 'action_chunk_size': 16, 'pad_action_dim': 36, 'pad_state_dim': 36}, 'model': {'resize': {'size': [256, 480]}, 'color_jitter': {'brightness': 0.2, 'contrast': [0.8, 1.2], 'saturation': [0.8, 1.2], 'hue': 0.05}, 'center_crop': {'size': [224, 224]}, 'normalize': {'mean': [0.485, 0.456, 0.406], 'std': [0.229, 0.224, 0.225]}, 'img_aug': True}, 'field': {'stat_path': 'meta/stats_psi0.json', 'action_norm_type': 'bounds', 'stat_action_key': 'action', 'stat_state_key': 'states', 'use_norm_mask': False, 'action_norm_masks': [True, True, True, True, True, True, False], 'action_min': [-0.5, -1.1095792134107943e-16, -1.1095792134107943e-16, -1.5, -1.5, -1.5, -1.5, -0.5, -0.699999988079071, -0.699999988079071, -2.2166350627321588e-16, -2.2166350627321588e-16, 0.0, -2.2166350627321588e-16, -0.47567468881607056, 0.1900009959936142, -0.512170135974884, -0.6265152096748352, -0.5008617043495178, -0.8220608830451965, -0.9223371148109436, -0.49507391452789307, -0.3437551259994507, -0.6871383190155029, -0.7637607455253601, -0.7568023204803467, -0.576077401638031, -0.4588268995285034, -0.13876836001873016, -0.10360867530107498, -0.47856518626213074, 0.7400000095367432, -0.5, -0.5, -0.26161932945251465, -0.06718750298023224], 'action_max': [0.5, 0.699999988079071, 0.699999988079071, 0.0, 2.2146225653890418e-16, 2.2146225653890418e-16, 2.2146225653890418e-16, 1.2266071310501902e-19, 1.1078670818917075e-16, 1.1078670818917075e-16, 1.5, 1.5, 0.6000000238418579, 1.5, 0.2472410947084427, 0.7092280983924866, 1.2571598291397095, 0.42311304807662964, 0.8564174771308899, 0.5002086162567139, 0.5172277092933655, 0.16140148043632507, -0.1900009959936142, 0.5362864136695862, 0.5715147256851196, 0.5002322196960449, 0.566592276096344, 0.6392397880554199, 0.1580466777086258, 0.2233395129442215, 0.2582152187824249, 0.7400000095367432, 0.5, 0.5, 0.3454970121383667, 0.2899305522441864], 'state_min': [-0.5564982891082764, -0.48307520151138306, -0.0005447770818136632, -0.8388738632202148, -1.3970016241073608, -0.8296014666557312, -1.4599460363388062, -0.5806806683540344, -0.5149835348129272, -0.6775947213172913, -0.001480442238971591, -0.0002713006397243589, -0.000914653530344367, -0.00019419840828049928, -0.4206617772579193, 0.13972464203834534, -0.546251654624939, -0.5596316456794739, -0.4764360189437866, -0.7253566384315491, -0.9443663954734802, -0.4381798803806305, -0.3338131606578827, -0.667724072933197, -0.6881827116012573, -0.7544379830360413, -0.5189417600631714, -0.4484957158565521, -0.13709338009357452, -0.07360810041427612, -0.4748336970806122, 0.7400000095367432, 0.0, 0.0, 0.0, 0.0], 'state_max': [0.43566983938217163, 0.3739710748195648, 0.6575677990913391, 0.004060761071741581, 0.0005700877518393099, 0.0004725759499706328, 0.00010080631182063371, 1.310737025050912e-05, 0.21882089972496033, 0.0005271440604701638, 0.530737042427063, 1.4406861066818237, 1.4605127573013306, 1.4595911502838135, 0.2663630545139313, 0.657910943031311, 1.2515853643417358, 0.502498209476471, 0.8292973637580872, 0.5248894095420837, 0.4653257131576538, 0.18638382852077484, -0.16696421802043915, 0.49318820238113403, 0.6363148093223572, 0.45773962140083313, 0.6238265037536621, 0.653800904750824, 0.1436084657907486, 0.25937986373901367, 0.26422709226608276, 0.7400000095367432, 0.0, 0.0, 0.0, 0.0], 'normalize_state': True, 'pad_action_dim': 36, 'pad_state_dim': 36}}, 'root_dir': '/data/jliu/data', 'train_repo_ids': ['G1WholebodyHandover-v0'], 'val_repo_ids': ['G1WholebodyHandover-v0']}, 'model': {'num_diffusion_iters': 100, 'action_chunk_size': 16, 'num_cameras': 1, 'share_vision_encoder': False, 'obs_horizon': 1, 'action_exec_horizon': 6, 'action_dim': 36, 'obs_dim': 36}, 'environment_variables': {'OMP_NUM_THREADS': '32', 'HF_HOME': '/data/cache', 'TORCH_HOME': '/data/cache', 'HF_TOKEN': 'hf_...TiKa', 'HF_LEROBOT_HOME': '/data/data/lerobot', 'WE_HOME': 'Not Set', 'DATA_HOME': '/data/data', 'UV_CACHE_DIR': '/data/cache', 'WANDB_API_KEY': '90e...5c06', 'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION': 'python', 'CUDA_VISIBLE_DEVICES': '4,5,6,7', 'WORLD_SIZE': '4', 'LOCAL_WORLD_SIZE': '4', 'RANK': '0', 'LOCAL_RANK': '0', 'MASTER_ADDR': 'nebula99', 'MASTER_PORT': '38477'}}
|
| 21 |
+
2026-04-10 05:25:47,354 INFO MainThread:1404689 [wandb_run.py:_finish():2291] finishing run jliu530-soochow-university/psi/sjem5sh4
|
| 22 |
+
2026-04-10 05:25:47,355 INFO MainThread:1404689 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0
|
| 23 |
+
2026-04-10 05:25:47,356 INFO MainThread:1404689 [wandb_run.py:_restore():2472] restore
|
| 24 |
+
2026-04-10 05:25:47,356 INFO MainThread:1404689 [wandb_run.py:_restore():2478] restore done
|
| 25 |
+
2026-04-10 05:25:48,206 INFO MainThread:1404689 [wandb_run.py:_footer_sync_info():3868] logging synced files
|
dp/diffusion-policy-g1-sim/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604100204/wandb/run-20260410_020501-sjem5sh4/run-sjem5sh4.wandb
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:bda83724181dddc3ec942bb2e4ae2ea91b04643e977a2bb1e62357e2878d28f6
|
| 3 |
+
size 36695569
|
dp/diffusion-policy-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/argv.txt
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
scripts/train.py
|
| 2 |
+
simple_dp_config
|
| 3 |
+
--seed=2026
|
| 4 |
+
--exp=g1wholebodylocomotionpickbetweentablesteleop-v0
|
| 5 |
+
--train.name=diffusion-policy-g1
|
| 6 |
+
--log.report-to=wandb
|
| 7 |
+
--train.data_parallel=ddp
|
| 8 |
+
--train.mixed_precision=bf16
|
| 9 |
+
--train.train-batch-size=32
|
| 10 |
+
--train.warmup-steps=1000
|
| 11 |
+
--train.warmup-ratio=None
|
| 12 |
+
--train.checkpointing-steps=5000
|
| 13 |
+
--train.validation_steps=500
|
| 14 |
+
--train.val_num_batches=20
|
| 15 |
+
--train.gradient_accumulation_steps=1
|
| 16 |
+
--train.max-training-steps=40000
|
| 17 |
+
--train.learning-rate=1e-4
|
| 18 |
+
--train.max-grad-norm=1.0
|
| 19 |
+
--train.lr_scheduler_kwargs.weight_decay=1e-6
|
| 20 |
+
--train.lr_scheduler_kwargs.betas 0.95 0.999
|
| 21 |
+
--train.lr_scheduler_type=cosine
|
| 22 |
+
--data.root_dir=/data/jliu/data
|
| 23 |
+
--data.train-repo-ids=G1WholebodyLocomotionPickBetweenTablesTeleop-v0
|
| 24 |
+
--data.transform.repack.action_chunk_size=16
|
| 25 |
+
--data.transform.repack.pad-action-dim=36
|
| 26 |
+
--data.transform.repack.pad-state-dim=36
|
| 27 |
+
--data.transform.field.stat-path=meta/stats_psi0.json
|
| 28 |
+
--data.transform.field.stat-action-key=action
|
| 29 |
+
--data.transform.field.stat-state-key=states
|
| 30 |
+
--data.transform.field.normalize-state
|
| 31 |
+
--data.transform.field.action-norm-type=bounds
|
| 32 |
+
--data.transform.field.pad-action-dim=36
|
| 33 |
+
--data.transform.field.pad-state-dim=36
|
| 34 |
+
--data.transform.model.img-aug
|
| 35 |
+
--model.action-chunk-size=16
|
| 36 |
+
--model.action-dim=36
|
| 37 |
+
--model.obs-dim=36
|
dp/diffusion-policy-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_40000/ema_net.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ae7cb6d75cb6937356f77cab48db617379d3a6c8ef61300bd9d1b2ad7e8c4640
|
| 3 |
+
size 337302879
|
dp/diffusion-policy-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_40000/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7b66c50b14c3bc9035cc256ccce61b77be6386ed4c668b260b59139adfddb26d
|
| 3 |
+
size 337238328
|
dp/diffusion-policy-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_40000/optimizer.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:de5d8ac0c1349b326c1f3a01e11f8d1f6961def58962424b014b50e510554296
|
| 3 |
+
size 674602251
|
dp/diffusion-policy-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_40000/random_states_0.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6e4affa92a6082563f121a744b3e33daacbb1669423413eeaba7ad196031f397
|
| 3 |
+
size 15473
|
dp/diffusion-policy-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_40000/random_states_1.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:655a5decc24cfd238cd0339b5b106cef192a7e3e4988ec6be31ff16df8528fcc
|
| 3 |
+
size 15473
|