Hugging Face's logo

StrongRoboticsLab
/
pi05-so100-diverse

Robotics
LeRobot
Safetensors
pi05
Model card Files
xet
 Community
bot commited on
Commit
b8e642f
·
1 Parent(s): 58a716e

Import Path, gitignore checkpoints locally

Browse files
Files changed (5) hide show
  1. .gitignore +1 -0
  2. lerobot/src/lerobot/scripts/lerobot_train.py +1 -0
  3. logs/training_20260324_023417.log +0 -1187
  4. logs/training_20260324_024348.log +0 -245
  5. logs/training_20260324_025549.log +0 -1136
.gitignore CHANGED
@@ -1 +1,2 @@
1
  checkpoints/
 
 
1
  checkpoints/
2
+ logs/
lerobot/src/lerobot/scripts/lerobot_train.py CHANGED
@@ -18,6 +18,7 @@ import logging
18
  import shutil
19
  import time
20
  from contextlib import nullcontext
 
21
  from pprint import pformat
22
  from typing import Any
23
 
 
18
  import shutil
19
  import time
20
  from contextlib import nullcontext
21
+ from pathlib import Path
22
  from pprint import pformat
23
  from typing import Any
24
 
logs/training_20260324_023417.log DELETED
@@ -1,1187 +0,0 @@
1
- === Starting Training ===
2
- The following values were not passed to `accelerate launch` and had defaults used instead:
3
- `--num_processes` was set to a value of `1`
4
- `--num_machines` was set to a value of `1`
5
- `--mixed_precision` was set to a value of `'no'`
6
- `--dynamo_backend` was set to a value of `'no'`
7
- To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`.
8
- WARNING:lerobot.configs.policies:Device 'mps' is not available. Switching to 'cuda'.
9
- WARNING:lerobot.configs.policies:Device 'mps' is not available. Switching to 'cuda'.
10
- INFO 2026-03-24 02:34:25 ot_train.py:198 {'batch_size': 64,
11
- 'checkpoint_path': None,
12
- 'cudnn_deterministic': False,
13
- 'dataset': {'episodes': None,
14
- 'image_transforms': {'enable': False,
15
- 'max_num_transforms': 3,
16
- 'random_order': False,
17
- 'tfs': {'affine': {'kwargs': {'degrees': [-5.0,
18
- 5.0],
19
- 'translate': [0.05,
20
- 0.05]},
21
- 'type': 'RandomAffine',
22
- 'weight': 1.0},
23
- 'brightness': {'kwargs': {'brightness': [0.8,
24
- 1.2]},
25
- 'type': 'ColorJitter',
26
- 'weight': 1.0},
27
- 'contrast': {'kwargs': {'contrast': [0.8,
28
- 1.2]},
29
- 'type': 'ColorJitter',
30
- 'weight': 1.0},
31
- 'hue': {'kwargs': {'hue': [-0.05,
32
- 0.05]},
33
- 'type': 'ColorJitter',
34
- 'weight': 1.0},
35
- 'saturation': {'kwargs': {'saturation': [0.5,
36
- 1.5]},
37
- 'type': 'ColorJitter',
38
- 'weight': 1.0},
39
- 'sharpness': {'kwargs': {'sharpness': [0.5,
40
- 1.5]},
41
- 'type': 'SharpnessJitter',
42
- 'weight': 1.0}}},
43
- 'repo_id': 'so100:/ephemeral/community_dataset_v3:/workspace/pi05-so100-diverse/filtered_index.json:/workspace/pi05-so100-diverse/norm_stats.json',
44
- 'revision': None,
45
- 'root': None,
46
- 'streaming': False,
47
- 'use_imagenet_stats': True,
48
- 'video_backend': 'torchcodec'},
49
- 'env': None,
50
- 'eval': {'batch_size': 50, 'n_episodes': 50, 'use_async_envs': False},
51
- 'eval_freq': 20000,
52
- 'job_name': 'pi05',
53
- 'log_freq': 50,
54
- 'num_workers': 4,
55
- 'optimizer': {'betas': [0.9, 0.95],
56
- 'eps': 1e-08,
57
- 'grad_clip_norm': 1.0,
58
- 'lr': 2.5e-05,
59
- 'type': 'adamw',
60
- 'weight_decay': 0.01},
61
- 'output_dir': '/ephemeral/production_run',
62
- 'peft': None,
63
- 'policy': {'action_expert_variant': 'gemma_300m',
64
- 'chunk_size': 50,
65
- 'compile_mode': 'max-autotune',
66
- 'compile_model': False,
67
- 'device': 'cuda',
68
- 'dtype': 'bfloat16',
69
- 'empty_cameras': 0,
70
- 'freeze_vision_encoder': False,
71
- 'gradient_checkpointing': False,
72
- 'image_resolution': [224, 224],
73
- 'input_features': {'observation.images.base_0_rgb': {'shape': [3,
74
- 224,
75
- 224],
76
- 'type': <FeatureType.VISUAL: 'VISUAL'>},
77
- 'observation.images.left_wrist_0_rgb': {'shape': [3,
78
- 224,
79
- 224],
80
- 'type': <FeatureType.VISUAL: 'VISUAL'>},
81
- 'observation.images.right_wrist_0_rgb': {'shape': [3,
82
- 224,
83
- 224],
84
- 'type': <FeatureType.VISUAL: 'VISUAL'>},
85
- 'observation.state': {'shape': [32],
86
- 'type': <FeatureType.STATE: 'STATE'>}},
87
- 'license': None,
88
- 'max_action_dim': 32,
89
- 'max_period': 4.0,
90
- 'max_state_dim': 32,
91
- 'min_period': 0.004,
92
- 'n_action_steps': 50,
93
- 'n_obs_steps': 1,
94
- 'normalization_mapping': {'ACTION': <NormalizationMode.MEAN_STD: 'MEAN_STD'>,
95
- 'STATE': <NormalizationMode.MEAN_STD: 'MEAN_STD'>,
96
- 'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>},
97
- 'num_inference_steps': 10,
98
- 'optimizer_betas': [0.9, 0.95],
99
- 'optimizer_eps': 1e-08,
100
- 'optimizer_grad_clip_norm': 1.0,
101
- 'optimizer_lr': 2.5e-05,
102
- 'optimizer_weight_decay': 0.01,
103
- 'output_features': {'action': {'shape': [32],
104
- 'type': <FeatureType.ACTION: 'ACTION'>}},
105
- 'paligemma_variant': 'gemma_2b',
106
- 'pretrained_path': 'lerobot/pi05_base',
107
- 'private': None,
108
- 'push_to_hub': True,
109
- 'repo_id': 'StrongRoboticsLab/pi05-so100-diverse',
110
- 'rtc_config': None,
111
- 'scheduler_decay_lr': 2.5e-06,
112
- 'scheduler_decay_steps': 85000,
113
- 'scheduler_warmup_steps': 1000,
114
- 'tags': None,
115
- 'time_sampling_beta_alpha': 1.5,
116
- 'time_sampling_beta_beta': 1.0,
117
- 'time_sampling_offset': 0.001,
118
- 'time_sampling_scale': 0.999,
119
- 'tokenizer_max_length': 200,
120
- 'train_expert_only': True,
121
- 'type': 'pi05',
122
- 'use_amp': False,
123
- 'use_peft': False},
124
- 'rabc_epsilon': 1e-06,
125
- 'rabc_head_mode': 'sparse',
126
- 'rabc_kappa': 0.01,
127
- 'rabc_progress_path': None,
128
- 'rename_map': {'observation.images.image': 'observation.images.base_0_rgb',
129
- 'observation.images.image2': 'observation.images.left_wrist_0_rgb'},
130
- 'resume': False,
131
- 'save_checkpoint': True,
132
- 'save_freq': 500,
133
- 'scheduler': {'decay_lr': 2.5e-06,
134
- 'num_decay_steps': 85000,
135
- 'num_warmup_steps': 1000,
136
- 'peak_lr': 2.5e-05,
137
- 'type': 'cosine_decay_with_warmup'},
138
- 'seed': 1000,
139
- 'steps': 85000,
140
- 'tolerance_s': 0.0001,
141
- 'use_policy_training_preset': True,
142
- 'use_rabc': False,
143
- 'wandb': {'add_tags': True,
144
- 'disable_artifact': False,
145
- 'enable': True,
146
- 'entity': None,
147
- 'mode': None,
148
- 'notes': None,
149
- 'project': 'pi05-so100-diverse',
150
- 'run_id': None}}
151
- INFO 2026-03-24 02:34:27 db_utils.py:117 Logs will be synced with wandb.
152
- INFO 2026-03-24 02:34:27 db_utils.py:118 Track this run --> https://wandb.ai/gptjustin-strong-robotics-lab/pi05-so100-diverse/runs/p4iqhvmi
153
- INFO 2026-03-24 02:34:27 ot_train.py:222 Creating dataset
154
- INFO 2026-03-24 02:34:30 ot_train.py:240 Creating policy
155
- INFO 2026-03-24 02:36:25 _client.py:1025 HTTP Request: HEAD https://huggingface.co/lerobot/pi05_base/resolve/main/model.safetensors "HTTP/1.1 302 Found"
156
- INFO 2026-03-24 02:36:25 _client.py:1025 HTTP Request: GET https://huggingface.co/api/models/lerobot/pi05_base/xet-read-token/9e55186ad36e66b95cda57bc47818d9e6237ae30 "HTTP/1.1 200 OK"
157
- WARNING 2026-03-24 02:36:44 ng_pi05.py:1102 Vision embedding key might need handling: paligemma_with_expert.paligemma.model.vision_tower.vision_model.embeddings.patch_embedding.bias
158
- WARNING 2026-03-24 02:36:44 ng_pi05.py:1102 Vision embedding key might need handling: paligemma_with_expert.paligemma.model.vision_tower.vision_model.embeddings.patch_embedding.weight
159
- INFO 2026-03-24 02:36:49 _client.py:1025 HTTP Request: HEAD https://huggingface.co/lerobot/pi05_base/resolve/main/policy_preprocessor.json "HTTP/1.1 307 Temporary Redirect"
160
- INFO 2026-03-24 02:36:49 _client.py:1025 HTTP Request: HEAD https://huggingface.co/api/resolve-cache/models/lerobot/pi05_base/9e55186ad36e66b95cda57bc47818d9e6237ae30/policy_preprocessor.json "HTTP/1.1 200 OK"
161
- INFO 2026-03-24 02:36:49 _client.py:1025 HTTP Request: GET https://huggingface.co/api/resolve-cache/models/lerobot/pi05_base/9e55186ad36e66b95cda57bc47818d9e6237ae30/policy_preprocessor.json "HTTP/1.1 200 OK"
162
- INFO 2026-03-24 02:36:49 _client.py:1025 HTTP Request: HEAD https://huggingface.co/google/paligemma-3b-pt-224/resolve/main/config.json "HTTP/1.1 200 OK"
163
- INFO 2026-03-24 02:36:49 _client.py:1025 HTTP Request: GET https://huggingface.co/google/paligemma-3b-pt-224/resolve/main/config.json "HTTP/1.1 200 OK"
164
- INFO 2026-03-24 02:36:50 _client.py:1025 HTTP Request: HEAD https://huggingface.co/google/paligemma-3b-pt-224/resolve/main/tokenizer_config.json "HTTP/1.1 200 OK"
165
- INFO 2026-03-24 02:36:50 _client.py:1025 HTTP Request: GET https://huggingface.co/google/paligemma-3b-pt-224/resolve/main/tokenizer_config.json "HTTP/1.1 200 OK"
166
- INFO 2026-03-24 02:36:50 _client.py:1025 HTTP Request: GET https://huggingface.co/api/models/google/paligemma-3b-pt-224/tree/main/additional_chat_templates?recursive=false&expand=false "HTTP/1.1 404 Not Found"
167
- INFO 2026-03-24 02:36:50 _client.py:1025 HTTP Request: GET https://huggingface.co/api/models/google/paligemma-3b-pt-224/tree/main?recursive=true&expand=false "HTTP/1.1 200 OK"
168
- INFO 2026-03-24 02:36:50 _client.py:1025 HTTP Request: HEAD https://huggingface.co/google/paligemma-3b-pt-224/resolve/main/tokenizer.json "HTTP/1.1 302 Found"
169
- INFO 2026-03-24 02:36:50 _client.py:1025 HTTP Request: GET https://huggingface.co/api/models/google/paligemma-3b-pt-224/xet-read-token/35e4f46485b4d07967e7e9935bc3786aad50687c "HTTP/1.1 200 OK"
170
- INFO 2026-03-24 02:36:50 _client.py:1025 HTTP Request: HEAD https://huggingface.co/google/paligemma-3b-pt-224/resolve/main/added_tokens.json "HTTP/1.1 200 OK"
171
- INFO 2026-03-24 02:36:50 _client.py:1025 HTTP Request: GET https://huggingface.co/google/paligemma-3b-pt-224/resolve/main/added_tokens.json "HTTP/1.1 200 OK"
172
- INFO 2026-03-24 02:36:50 _client.py:1025 HTTP Request: HEAD https://huggingface.co/google/paligemma-3b-pt-224/resolve/main/special_tokens_map.json "HTTP/1.1 200 OK"
173
- INFO 2026-03-24 02:36:50 _client.py:1025 HTTP Request: GET https://huggingface.co/google/paligemma-3b-pt-224/resolve/main/special_tokens_map.json "HTTP/1.1 200 OK"
174
- INFO 2026-03-24 02:36:51 _client.py:1025 HTTP Request: HEAD https://huggingface.co/google/paligemma-3b-pt-224/resolve/main/chat_template.jinja "HTTP/1.1 404 Not Found"
175
- INFO 2026-03-24 02:36:52 _client.py:1025 HTTP Request: GET https://huggingface.co/api/models/google/paligemma-3b-pt-224 "HTTP/1.1 200 OK"
176
- INFO 2026-03-24 02:36:52 _client.py:1025 HTTP Request: HEAD https://huggingface.co/lerobot/pi05_base/resolve/main/policy_postprocessor.json "HTTP/1.1 307 Temporary Redirect"
177
- INFO 2026-03-24 02:36:52 _client.py:1025 HTTP Request: HEAD https://huggingface.co/api/resolve-cache/models/lerobot/pi05_base/9e55186ad36e66b95cda57bc47818d9e6237ae30/policy_postprocessor.json "HTTP/1.1 200 OK"
178
- INFO 2026-03-24 02:36:52 _client.py:1025 HTTP Request: GET https://huggingface.co/api/resolve-cache/models/lerobot/pi05_base/9e55186ad36e66b95cda57bc47818d9e6237ae30/policy_postprocessor.json "HTTP/1.1 200 OK"
179
- INFO 2026-03-24 02:36:52 ot_train.py:295 Creating optimizer and scheduler
180
- INFO 2026-03-24 02:36:52 ot_train.py:330 Output dir: /ephemeral/production_run
181
- INFO 2026-03-24 02:36:52 ot_train.py:337 cfg.steps=85000 (85K)
182
- INFO 2026-03-24 02:36:52 ot_train.py:338 dataset.num_frames=4923840 (5M)
183
- INFO 2026-03-24 02:36:52 ot_train.py:339 dataset.num_episodes=10155
184
- INFO 2026-03-24 02:36:52 ot_train.py:342 Effective batch size: 64 x 1 = 64
185
- INFO 2026-03-24 02:36:52 ot_train.py:343 num_learnable_params=693422112 (693M)
186
- INFO 2026-03-24 02:36:52 ot_train.py:344 num_total_params=4143404816 (4B)
187
- The PI05 model is a direct port of the OpenPI implementation.
188
- This implementation follows the original OpenPI structure for compatibility.
189
- Original implementation: https://github.com/Physical-Intelligence/openpi
190
- Loading model from: lerobot/pi05_base
191
- ✓ Loaded state dict from model.safetensors
192
- Remapped 812 state dict keys
193
- All keys loaded successfully!
194
-
195
- Traceback (most recent call last):
196
- File "<frozen runpy>", line 198, in _run_module_as_main
197
- File "<frozen runpy>", line 88, in _run_code
198
- File "/workspace/pi05-so100-diverse/lerobot/src/lerobot/scripts/lerobot_train.py", line 575, in <module>
199
- main()
200
- File "/workspace/pi05-so100-diverse/lerobot/src/lerobot/scripts/lerobot_train.py", line 571, in main
201
- train()
202
- File "/workspace/pi05-so100-diverse/lerobot/src/lerobot/configs/parser.py", line 233, in wrapper_inner
203
- response = fn(cfg, *args, **kwargs)
204
- ^^^^^^^^^^^^^^^^^^^^^^^^
205
- File "/workspace/pi05-so100-diverse/lerobot/src/lerobot/scripts/lerobot_train.py", line 419, in train
206
- train_tracker, output_dict = update_policy(
207
- ^^^^^^^^^^^^^^
208
- File "/workspace/pi05-so100-diverse/lerobot/src/lerobot/scripts/lerobot_train.py", line 118, in update_policy
209
- loss, output_dict = policy.forward(batch)
210
- ^^^^^^^^^^^^^^^^^^^^^
211
- File "/workspace/pi05-so100-diverse/lerobot/src/lerobot/policies/pi05/modeling_pi05.py", line 1264, in forward
212
- losses = self.model.forward(images, img_masks, tokens, masks, actions)
213
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
214
- File "/workspace/pi05-so100-diverse/lerobot/src/lerobot/policies/pi05/modeling_pi05.py", line 771, in forward
215
- suffix_out = self._apply_checkpoint(
216
- ^^^^^^^^^^^^^^^^^^^^^^^
217
- File "/workspace/pi05-so100-diverse/lerobot/src/lerobot/policies/pi05/modeling_pi05.py", line 617, in _apply_checkpoint
218
- return func(*args, **kwargs)
219
- ^^^^^^^^^^^^^^^^^^^^^
220
- File "/workspace/pi05-so100-diverse/lerobot/src/lerobot/policies/pi05/modeling_pi05.py", line 761, in forward_func
221
- (_, suffix_out), _ = self.paligemma_with_expert.forward(
222
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
223
- File "/workspace/pi05-so100-diverse/lerobot/src/lerobot/policies/pi05/modeling_pi05.py", line 513, in forward
224
- inputs_embeds = compute_layer_complete(
225
- ^^^^^^^^^^^^^^^^^^^^^^^
226
- File "/workspace/pi05-so100-diverse/lerobot/src/lerobot/policies/pi05/modeling_pi05.py", line 291, in compute_layer_complete
227
- out_emb = layer.mlp(out_emb)
228
- ^^^^^^^^^^^^^^^^^^
229
- File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
230
- return self._call_impl(*args, **kwargs)
231
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
232
- File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1750, in _call_impl
233
- return forward_call(*args, **kwargs)
234
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
235
- File "/usr/local/lib/python3.12/dist-packages/transformers/models/gemma/modeling_gemma.py", line 81, in forward
236
- down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
237
- ^^^^^^^^^^^^^^^
238
- File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
239
- return self._call_impl(*args, **kwargs)
240
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
241
- File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1750, in _call_impl
242
- return forward_call(*args, **kwargs)
243
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
244
- File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/linear.py", line 125, in forward
245
- return F.linear(input, self.weight, self.bias)
246
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
247
- torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 1.89 GiB. GPU 0 has a total capacity of 79.19 GiB of which 1.52 GiB is free. Including non-PyTorch memory, this process has 77.65 GiB memory in use. Of the allocated memory 74.27 GiB is allocated by PyTorch, and 2.79 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
248
- Exception in thread Thread-3 (_pin_memory_loop):
249
- Traceback (most recent call last):
250
- File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
251
- self.run()
252
- File "/usr/lib/python3.12/threading.py", line 1012, in run
253
- self._target(*self._args, **self._kwargs)
254
- File "/usr/local/lib/python3.12/dist-packages/torch/utils/data/_utils/pin_memory.py", line 59, in _pin_memory_loop
255
- do_one_step()
256
- File "/usr/local/lib/python3.12/dist-packages/torch/utils/data/_utils/pin_memory.py", line 35, in do_one_step
257
- r = in_queue.get(timeout=MP_STATUS_CHECK_INTERVAL)
258
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
259
- File "/usr/lib/python3.12/multiprocessing/queues.py", line 122, in get
260
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
261
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
262
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
263
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
264
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
265
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
266
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
267
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
268
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
269
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
270
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
271
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
272
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
273
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
274
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
275
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
276
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
277
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
278
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
279
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
280
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
281
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
282
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
283
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
284
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
285
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
286
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
287
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
288
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
289
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
290
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
291
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
292
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
293
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
294
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
295
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
296
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
297
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
298
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
299
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
300
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
301
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
302
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
303
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
304
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
305
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
306
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
307
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
308
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
309
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
310
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
311
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
312
- return _ForkingPickler.loads(res)
313
- ^^^^^^^^^^^^^^^^^^^^^^^^^^
314
- File "/usr/local/lib/python3.12/dist-packages/torch/multiprocessing/reductions.py", line 541, in rebuild_storage_fd
315
- fd = df.detach()
316
- ^^^^^^^^^^^
317
- File "/usr/lib/python3.12/multiprocessing/resource_sharer.py", line 57, in detach
318
- with _resource_sharer.get_connection(self._id) as conn:
319
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
320
- File "/usr/lib/python3.12/multiprocessing/resource_sharer.py", line 86, in get_connection
321
- c = Client(address, authkey=process.current_process().authkey)
322
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
323
- File "/usr/lib/python3.12/multiprocessing/connection.py", line 525, in Client
324
- answer_challenge(c, authkey)
325
- File "/usr/lib/python3.12/multiprocessing/connection.py", line 953, in answer_challenge
326
- message = connection.recv_bytes(256) # reject large message
327
- ^^^^^^^^^^^^^^^^^^^^^^^^^^
328
- File "/usr/lib/python3.12/multiprocessing/connection.py", line 216, in recv_bytes
329
- buf = self._recv_bytes(maxlength)
330
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^
331
- File "/usr/lib/python3.12/multiprocessing/connection.py", line 430, in _recv_bytes
332
- buf = self._recv(4)
333
- ^^^^^^^^^^^^^
334
- File "/usr/lib/python3.12/multiprocessing/connection.py", line 395, in _recv
335
- chunk = read(handle, remaining)
336
- ^^^^^^^^^^^^^^^^^^^^^^^
337
- ConnectionResetError: [Errno 104] Connection reset by peer
338
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
339
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
340
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
341
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
342
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
343
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
344
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
345
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
346
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
347
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
348
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
349
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
350
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
351
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
352
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
353
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
354
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
355
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
356
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
357
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
358
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
359
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
360
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
361
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
362
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
363
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
364
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
365
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
366
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
367
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
368
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
369
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
370
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
371
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
372
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
373
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
374
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
375
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
376
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
377
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
378
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
379
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
380
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
381
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
382
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
383
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
384
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
385
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
386
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
387
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
388
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
389
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
390
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
391
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
392
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
393
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
394
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
395
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
396
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
397
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
398
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
399
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
400
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
401
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
402
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
403
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
404
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
405
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
406
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
407
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
408
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
409
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
410
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
411
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
412
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
413
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
414
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
415
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
416
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
417
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
418
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
419
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
420
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
421
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
422
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
423
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
424
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
425
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
426
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
427
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
428
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
429
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
430
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
431
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
432
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
433
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
434
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
435
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
436
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
437
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
438
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
439
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
440
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
441
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
442
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
443
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
444
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
445
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
446
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
447
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
448
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
449
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
450
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
451
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
452
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
453
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
454
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
455
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
456
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
457
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
458
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
459
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
460
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
461
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
462
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
463
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
464
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
465
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
466
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
467
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
468
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
469
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
470
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
471
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
472
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
473
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
474
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
475
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
476
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
477
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
478
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
479
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
480
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
481
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
482
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
483
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
484
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
485
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
486
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
487
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
488
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
489
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
490
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
491
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
492
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
493
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
494
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
495
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
496
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
497
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
498
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
499
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
500
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
501
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
502
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
503
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
504
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
505
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
506
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
507
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
508
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
509
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
510
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
511
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
512
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
513
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
514
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
515
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
516
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
517
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
518
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
519
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
520
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
521
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
522
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
523
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
524
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
525
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
526
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
527
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
528
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
529
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
530
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
531
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
532
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
533
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
534
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
535
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
536
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
537
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
538
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
539
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
540
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
541
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
542
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
543
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
544
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
545
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
546
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
547
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
548
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
549
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
550
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
551
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
552
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
553
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
554
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
555
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
556
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
557
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
558
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
559
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
560
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
561
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
562
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
563
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
564
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
565
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
566
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
567
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
568
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
569
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
570
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
571
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
572
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
573
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
574
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
575
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
576
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
577
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
578
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
579
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
580
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
581
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
582
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
583
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
584
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
585
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
586
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
587
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
588
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
589
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
590
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
591
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
592
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
593
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
594
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
595
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
596
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
597
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
598
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
599
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
600
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
601
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
602
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
603
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
604
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
605
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
606
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
607
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
608
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
609
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
610
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
611
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
612
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
613
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
614
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
615
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
616
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
617
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
618
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
619
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
620
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
621
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
622
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
623
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
624
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
625
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
626
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
627
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
628
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
629
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
630
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
631
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
632
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
633
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
634
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
635
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
636
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
637
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
638
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
639
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
640
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
641
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
642
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
643
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
644
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
645
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
646
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
647
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
648
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
649
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
650
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
651
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
652
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
653
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
654
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
655
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
656
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
657
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
658
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
659
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
660
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
661
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
662
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
663
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
664
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
665
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
666
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
667
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
668
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
669
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
670
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
671
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
672
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
673
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
674
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
675
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
676
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
677
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
678
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
679
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
680
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
681
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
682
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
683
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
684
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
685
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
686
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
687
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
688
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
689
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
690
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
691
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
692
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
693
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
694
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
695
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
696
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
697
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
698
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
699
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
700
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
701
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
702
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
703
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
704
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
705
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
706
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
707
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
708
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
709
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
710
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
711
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
712
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
713
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
714
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
715
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
716
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
717
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
718
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
719
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
720
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
721
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
722
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
723
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
724
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
725
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
726
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
727
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
728
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
729
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
730
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
731
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
732
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
733
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
734
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
735
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
736
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
737
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
738
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
739
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
740
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
741
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
742
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
743
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
744
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
745
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
746
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
747
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
748
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
749
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
750
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
751
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
752
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
753
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
754
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
755
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
756
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
757
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
758
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
759
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
760
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
761
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
762
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
763
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
764
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
765
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
766
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
767
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
768
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
769
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
770
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
771
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
772
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
773
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
774
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
775
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
776
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
777
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
778
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
779
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
780
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
781
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
782
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
783
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
784
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
785
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
786
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
787
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
788
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
789
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
790
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
791
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
792
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
793
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
794
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
795
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
796
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
797
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
798
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
799
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
800
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
801
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
802
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
803
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
804
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
805
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
806
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
807
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
808
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
809
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
810
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
811
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
812
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
813
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
814
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
815
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
816
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
817
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
818
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
819
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
820
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
821
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
822
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
823
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
824
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
825
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
826
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
827
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
828
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
829
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
830
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
831
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
832
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
833
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
834
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
835
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
836
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
837
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
838
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
839
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
840
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
841
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
842
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
843
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
844
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
845
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
846
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
847
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
848
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
849
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
850
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
851
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
852
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
853
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
854
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
855
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
856
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
857
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
858
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
859
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
860
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
861
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
862
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
863
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
864
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
865
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
866
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
867
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
868
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
869
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
870
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
871
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
872
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
873
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
874
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
875
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
876
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
877
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
878
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
879
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
880
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
881
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
882
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
883
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
884
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
885
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
886
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
887
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
888
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
889
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
890
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
891
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
892
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
893
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
894
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
895
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
896
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
897
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
898
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
899
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
900
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
901
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
902
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
903
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
904
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
905
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
906
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
907
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
908
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
909
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
910
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
911
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
912
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
913
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
914
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
915
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
916
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
917
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
918
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
919
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
920
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
921
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
922
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
923
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
924
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
925
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
926
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
927
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
928
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
929
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
930
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
931
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
932
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
933
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
934
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
935
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
936
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
937
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
938
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
939
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
940
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
941
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
942
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
943
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
944
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
945
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
946
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
947
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
948
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
949
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
950
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
951
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
952
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
953
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
954
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
955
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
956
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
957
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
958
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
959
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
960
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
961
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
962
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
963
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
964
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
965
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
966
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
967
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
968
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
969
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
970
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
971
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
972
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
973
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
974
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
975
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
976
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
977
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
978
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
979
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
980
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
981
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
982
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
983
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
984
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
985
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
986
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
987
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
988
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
989
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
990
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
991
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
992
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
993
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
994
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
995
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
996
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
997
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
998
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
999
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1000
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1001
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1002
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1003
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1004
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1005
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1006
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1007
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1008
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1009
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1010
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1011
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1012
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1013
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1014
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1015
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1016
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1017
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1018
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1019
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1020
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1021
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1022
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1023
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1024
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1025
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1026
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1027
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1028
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1029
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1030
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1031
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1032
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1033
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1034
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1035
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1036
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1037
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1038
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1039
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1040
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1041
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1042
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1043
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1044
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1045
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1046
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1047
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1048
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1049
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1050
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1051
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1052
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1053
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1054
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1055
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1056
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1057
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1058
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1059
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1060
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1061
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1062
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1063
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1064
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1065
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1066
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1067
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1068
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1069
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1070
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1071
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1072
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1073
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1074
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1075
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1076
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1077
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1078
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1079
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1080
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1081
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1082
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1083
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1084
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1085
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1086
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1087
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1088
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1089
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1090
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1091
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1092
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1093
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1094
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1095
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1096
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1097
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1098
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1099
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1100
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1101
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1102
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1103
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1104
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1105
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1106
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1107
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1108
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1109
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1110
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1111
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1112
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1113
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1114
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1115
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1116
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1117
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1118
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1119
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1120
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1121
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1122
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1123
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1124
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1125
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1126
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1127
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1128
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1129
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1130
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1131
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1132
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1133
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1134
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1135
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1136
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1137
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1138
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1139
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1140
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1141
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1142
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1143
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1144
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1145
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1146
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1147
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1148
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1149
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1150
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1151
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1152
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1153
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1154
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1155
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1156
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1157
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1158
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1159
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1160
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1161
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1162
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1163
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1164
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1165
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1166
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1167
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1168
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1169
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1170
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1171
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1172
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1173
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1174
- WARNING 2026-03-24 02:36:59 _events.py:1068 socket.send() raised exception.
1175
- Traceback (most recent call last):
1176
- File "<frozen runpy>", line 198, in _run_module_as_main
1177
- File "<frozen runpy>", line 88, in _run_code
1178
- File "/usr/local/lib/python3.12/dist-packages/accelerate/commands/launch.py", line 1415, in <module>
1179
- main()
1180
- File "/usr/local/lib/python3.12/dist-packages/accelerate/commands/launch.py", line 1411, in main
1181
- launch_command(args)
1182
- File "/usr/local/lib/python3.12/dist-packages/accelerate/commands/launch.py", line 1405, in launch_command
1183
- simple_launcher(args)
1184
- File "/usr/local/lib/python3.12/dist-packages/accelerate/commands/launch.py", line 993, in simple_launcher
1185
- raise subprocess.CalledProcessError(returncode=process.returncode, cmd=cmd)
1186
- subprocess.CalledProcessError: Command '['/usr/bin/python3.12', '-m', 'lerobot.scripts.lerobot_train', '--dataset.repo_id=so100:/ephemeral/community_dataset_v3:/workspace/pi05-so100-diverse/filtered_index.json:/workspace/pi05-so100-diverse/norm_stats.json', '--policy.path=lerobot/pi05_base', '--policy.train_expert_only=true', '--policy.dtype=bfloat16', '--policy.gradient_checkpointing=false', '--policy.push_to_hub=true', '--policy.repo_id=StrongRoboticsLab/pi05-so100-diverse', '--policy.normalization_mapping={"VISUAL": "IDENTITY", "STATE": "MEAN_STD", "ACTION": "MEAN_STD"}', '--policy.scheduler_warmup_steps=1000', '--policy.scheduler_decay_steps=85000', '--rename_map={"observation.images.image": "observation.images.base_0_rgb", "observation.images.image2": "observation.images.left_wrist_0_rgb"}', '--batch_size=64', '--steps=85000', '--save_freq=500', '--log_freq=50', '--num_workers=4', '--wandb.enable=true', '--wandb.project=pi05-so100-diverse', '--output_dir=/ephemeral/production_run']' returned non-zero exit status 1.
1187
- === Training Complete (exit: 1) ===
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
logs/training_20260324_024348.log DELETED
@@ -1,245 +0,0 @@
1
- === Starting Training ===
2
- The following values were not passed to `accelerate launch` and had defaults used instead:
3
- `--num_processes` was set to a value of `1`
4
- `--num_machines` was set to a value of `1`
5
- `--mixed_precision` was set to a value of `'no'`
6
- `--dynamo_backend` was set to a value of `'no'`
7
- To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`.
8
- WARNING:lerobot.configs.policies:Device 'mps' is not available. Switching to 'cuda'.
9
- WARNING:lerobot.configs.policies:Device 'mps' is not available. Switching to 'cuda'.
10
- INFO 2026-03-24 02:43:56 ot_train.py:198 {'batch_size': 32,
11
- 'checkpoint_path': None,
12
- 'cudnn_deterministic': False,
13
- 'dataset': {'episodes': None,
14
- 'image_transforms': {'enable': False,
15
- 'max_num_transforms': 3,
16
- 'random_order': False,
17
- 'tfs': {'affine': {'kwargs': {'degrees': [-5.0,
18
- 5.0],
19
- 'translate': [0.05,
20
- 0.05]},
21
- 'type': 'RandomAffine',
22
- 'weight': 1.0},
23
- 'brightness': {'kwargs': {'brightness': [0.8,
24
- 1.2]},
25
- 'type': 'ColorJitter',
26
- 'weight': 1.0},
27
- 'contrast': {'kwargs': {'contrast': [0.8,
28
- 1.2]},
29
- 'type': 'ColorJitter',
30
- 'weight': 1.0},
31
- 'hue': {'kwargs': {'hue': [-0.05,
32
- 0.05]},
33
- 'type': 'ColorJitter',
34
- 'weight': 1.0},
35
- 'saturation': {'kwargs': {'saturation': [0.5,
36
- 1.5]},
37
- 'type': 'ColorJitter',
38
- 'weight': 1.0},
39
- 'sharpness': {'kwargs': {'sharpness': [0.5,
40
- 1.5]},
41
- 'type': 'SharpnessJitter',
42
- 'weight': 1.0}}},
43
- 'repo_id': 'so100:/ephemeral/community_dataset_v3:/workspace/pi05-so100-diverse/filtered_index.json:/workspace/pi05-so100-diverse/norm_stats.json',
44
- 'revision': None,
45
- 'root': None,
46
- 'streaming': False,
47
- 'use_imagenet_stats': True,
48
- 'video_backend': 'torchcodec'},
49
- 'env': None,
50
- 'eval': {'batch_size': 50, 'n_episodes': 50, 'use_async_envs': False},
51
- 'eval_freq': 20000,
52
- 'job_name': 'pi05',
53
- 'log_freq': 50,
54
- 'num_workers': 4,
55
- 'optimizer': {'betas': [0.9, 0.95],
56
- 'eps': 1e-08,
57
- 'grad_clip_norm': 1.0,
58
- 'lr': 2.5e-05,
59
- 'type': 'adamw',
60
- 'weight_decay': 0.01},
61
- 'output_dir': '/ephemeral/production_run',
62
- 'peft': None,
63
- 'policy': {'action_expert_variant': 'gemma_300m',
64
- 'chunk_size': 50,
65
- 'compile_mode': 'max-autotune',
66
- 'compile_model': False,
67
- 'device': 'cuda',
68
- 'dtype': 'bfloat16',
69
- 'empty_cameras': 0,
70
- 'freeze_vision_encoder': False,
71
- 'gradient_checkpointing': False,
72
- 'image_resolution': [224, 224],
73
- 'input_features': {'observation.images.base_0_rgb': {'shape': [3,
74
- 224,
75
- 224],
76
- 'type': <FeatureType.VISUAL: 'VISUAL'>},
77
- 'observation.images.left_wrist_0_rgb': {'shape': [3,
78
- 224,
79
- 224],
80
- 'type': <FeatureType.VISUAL: 'VISUAL'>},
81
- 'observation.images.right_wrist_0_rgb': {'shape': [3,
82
- 224,
83
- 224],
84
- 'type': <FeatureType.VISUAL: 'VISUAL'>},
85
- 'observation.state': {'shape': [32],
86
- 'type': <FeatureType.STATE: 'STATE'>}},
87
- 'license': None,
88
- 'max_action_dim': 32,
89
- 'max_period': 4.0,
90
- 'max_state_dim': 32,
91
- 'min_period': 0.004,
92
- 'n_action_steps': 50,
93
- 'n_obs_steps': 1,
94
- 'normalization_mapping': {'ACTION': <NormalizationMode.MEAN_STD: 'MEAN_STD'>,
95
- 'STATE': <NormalizationMode.MEAN_STD: 'MEAN_STD'>,
96
- 'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>},
97
- 'num_inference_steps': 10,
98
- 'optimizer_betas': [0.9, 0.95],
99
- 'optimizer_eps': 1e-08,
100
- 'optimizer_grad_clip_norm': 1.0,
101
- 'optimizer_lr': 2.5e-05,
102
- 'optimizer_weight_decay': 0.01,
103
- 'output_features': {'action': {'shape': [32],
104
- 'type': <FeatureType.ACTION: 'ACTION'>}},
105
- 'paligemma_variant': 'gemma_2b',
106
- 'pretrained_path': 'lerobot/pi05_base',
107
- 'private': None,
108
- 'push_to_hub': True,
109
- 'repo_id': 'StrongRoboticsLab/pi05-so100-diverse',
110
- 'rtc_config': None,
111
- 'scheduler_decay_lr': 2.5e-06,
112
- 'scheduler_decay_steps': 170000,
113
- 'scheduler_warmup_steps': 1000,
114
- 'tags': None,
115
- 'time_sampling_beta_alpha': 1.5,
116
- 'time_sampling_beta_beta': 1.0,
117
- 'time_sampling_offset': 0.001,
118
- 'time_sampling_scale': 0.999,
119
- 'tokenizer_max_length': 200,
120
- 'train_expert_only': True,
121
- 'type': 'pi05',
122
- 'use_amp': False,
123
- 'use_peft': False},
124
- 'rabc_epsilon': 1e-06,
125
- 'rabc_head_mode': 'sparse',
126
- 'rabc_kappa': 0.01,
127
- 'rabc_progress_path': None,
128
- 'rename_map': {'observation.images.image': 'observation.images.base_0_rgb',
129
- 'observation.images.image2': 'observation.images.left_wrist_0_rgb'},
130
- 'resume': False,
131
- 'save_checkpoint': True,
132
- 'save_freq': 500,
133
- 'scheduler': {'decay_lr': 2.5e-06,
134
- 'num_decay_steps': 170000,
135
- 'num_warmup_steps': 1000,
136
- 'peak_lr': 2.5e-05,
137
- 'type': 'cosine_decay_with_warmup'},
138
- 'seed': 1000,
139
- 'steps': 170000,
140
- 'tolerance_s': 0.0001,
141
- 'use_policy_training_preset': True,
142
- 'use_rabc': False,
143
- 'wandb': {'add_tags': True,
144
- 'disable_artifact': False,
145
- 'enable': True,
146
- 'entity': None,
147
- 'mode': None,
148
- 'notes': None,
149
- 'project': 'pi05-so100-diverse',
150
- 'run_id': None}}
151
- INFO 2026-03-24 02:43:57 db_utils.py:117 Logs will be synced with wandb.
152
- INFO 2026-03-24 02:43:57 db_utils.py:118 Track this run --> https://wandb.ai/gptjustin-strong-robotics-lab/pi05-so100-diverse/runs/pajke257
153
- INFO 2026-03-24 02:43:57 ot_train.py:222 Creating dataset
154
- INFO 2026-03-24 02:44:00 ot_train.py:240 Creating policy
155
- INFO 2026-03-24 02:46:22 _client.py:1025 HTTP Request: HEAD https://huggingface.co/lerobot/pi05_base/resolve/main/model.safetensors "HTTP/1.1 302 Found"
156
- INFO 2026-03-24 02:46:22 _client.py:1025 HTTP Request: GET https://huggingface.co/api/models/lerobot/pi05_base/xet-read-token/9e55186ad36e66b95cda57bc47818d9e6237ae30 "HTTP/1.1 200 OK"
157
- WARNING 2026-03-24 02:46:40 ng_pi05.py:1102 Vision embedding key might need handling: paligemma_with_expert.paligemma.model.vision_tower.vision_model.embeddings.patch_embedding.bias
158
- WARNING 2026-03-24 02:46:40 ng_pi05.py:1102 Vision embedding key might need handling: paligemma_with_expert.paligemma.model.vision_tower.vision_model.embeddings.patch_embedding.weight
159
- INFO 2026-03-24 02:46:46 _client.py:1025 HTTP Request: HEAD https://huggingface.co/lerobot/pi05_base/resolve/main/policy_preprocessor.json "HTTP/1.1 307 Temporary Redirect"
160
- INFO 2026-03-24 02:46:47 _client.py:1025 HTTP Request: HEAD https://huggingface.co/api/resolve-cache/models/lerobot/pi05_base/9e55186ad36e66b95cda57bc47818d9e6237ae30/policy_preprocessor.json "HTTP/1.1 200 OK"
161
- INFO 2026-03-24 02:46:47 _client.py:1025 HTTP Request: GET https://huggingface.co/api/resolve-cache/models/lerobot/pi05_base/9e55186ad36e66b95cda57bc47818d9e6237ae30/policy_preprocessor.json "HTTP/1.1 200 OK"
162
- INFO 2026-03-24 02:46:47 _client.py:1025 HTTP Request: HEAD https://huggingface.co/google/paligemma-3b-pt-224/resolve/main/config.json "HTTP/1.1 200 OK"
163
- INFO 2026-03-24 02:46:47 _client.py:1025 HTTP Request: GET https://huggingface.co/google/paligemma-3b-pt-224/resolve/main/config.json "HTTP/1.1 200 OK"
164
- INFO 2026-03-24 02:46:47 _client.py:1025 HTTP Request: HEAD https://huggingface.co/google/paligemma-3b-pt-224/resolve/main/tokenizer_config.json "HTTP/1.1 200 OK"
165
- INFO 2026-03-24 02:46:47 _client.py:1025 HTTP Request: GET https://huggingface.co/google/paligemma-3b-pt-224/resolve/main/tokenizer_config.json "HTTP/1.1 200 OK"
166
- INFO 2026-03-24 02:46:47 _client.py:1025 HTTP Request: GET https://huggingface.co/api/models/google/paligemma-3b-pt-224/tree/main/additional_chat_templates?recursive=false&expand=false "HTTP/1.1 404 Not Found"
167
- INFO 2026-03-24 02:46:47 _client.py:1025 HTTP Request: GET https://huggingface.co/api/models/google/paligemma-3b-pt-224/tree/main?recursive=true&expand=false "HTTP/1.1 200 OK"
168
- INFO 2026-03-24 02:46:47 _client.py:1025 HTTP Request: HEAD https://huggingface.co/google/paligemma-3b-pt-224/resolve/main/tokenizer.json "HTTP/1.1 302 Found"
169
- INFO 2026-03-24 02:46:47 _client.py:1025 HTTP Request: GET https://huggingface.co/api/models/google/paligemma-3b-pt-224/xet-read-token/35e4f46485b4d07967e7e9935bc3786aad50687c "HTTP/1.1 200 OK"
170
- INFO 2026-03-24 02:46:48 _client.py:1025 HTTP Request: HEAD https://huggingface.co/google/paligemma-3b-pt-224/resolve/main/added_tokens.json "HTTP/1.1 200 OK"
171
- INFO 2026-03-24 02:46:48 _client.py:1025 HTTP Request: GET https://huggingface.co/google/paligemma-3b-pt-224/resolve/main/added_tokens.json "HTTP/1.1 200 OK"
172
- INFO 2026-03-24 02:46:48 _client.py:1025 HTTP Request: HEAD https://huggingface.co/google/paligemma-3b-pt-224/resolve/main/special_tokens_map.json "HTTP/1.1 200 OK"
173
- INFO 2026-03-24 02:46:48 _client.py:1025 HTTP Request: GET https://huggingface.co/google/paligemma-3b-pt-224/resolve/main/special_tokens_map.json "HTTP/1.1 200 OK"
174
- INFO 2026-03-24 02:46:48 _client.py:1025 HTTP Request: HEAD https://huggingface.co/google/paligemma-3b-pt-224/resolve/main/chat_template.jinja "HTTP/1.1 404 Not Found"
175
- INFO 2026-03-24 02:46:49 _client.py:1025 HTTP Request: GET https://huggingface.co/api/models/google/paligemma-3b-pt-224 "HTTP/1.1 200 OK"
176
- INFO 2026-03-24 02:46:49 _client.py:1025 HTTP Request: HEAD https://huggingface.co/lerobot/pi05_base/resolve/main/policy_postprocessor.json "HTTP/1.1 307 Temporary Redirect"
177
- INFO 2026-03-24 02:46:49 _client.py:1025 HTTP Request: HEAD https://huggingface.co/api/resolve-cache/models/lerobot/pi05_base/9e55186ad36e66b95cda57bc47818d9e6237ae30/policy_postprocessor.json "HTTP/1.1 200 OK"
178
- INFO 2026-03-24 02:46:49 _client.py:1025 HTTP Request: GET https://huggingface.co/api/resolve-cache/models/lerobot/pi05_base/9e55186ad36e66b95cda57bc47818d9e6237ae30/policy_postprocessor.json "HTTP/1.1 200 OK"
179
- INFO 2026-03-24 02:46:49 ot_train.py:295 Creating optimizer and scheduler
180
- INFO 2026-03-24 02:46:49 ot_train.py:330 Output dir: /ephemeral/production_run
181
- INFO 2026-03-24 02:46:49 ot_train.py:337 cfg.steps=170000 (170K)
182
- INFO 2026-03-24 02:46:49 ot_train.py:338 dataset.num_frames=4923840 (5M)
183
- INFO 2026-03-24 02:46:49 ot_train.py:339 dataset.num_episodes=10155
184
- INFO 2026-03-24 02:46:49 ot_train.py:342 Effective batch size: 32 x 1 = 32
185
- INFO 2026-03-24 02:46:49 ot_train.py:343 num_learnable_params=693422112 (693M)
186
- INFO 2026-03-24 02:46:49 ot_train.py:344 num_total_params=4143404816 (4B)
187
- The PI05 model is a direct port of the OpenPI implementation.
188
- This implementation follows the original OpenPI structure for compatibility.
189
- Original implementation: https://github.com/Physical-Intelligence/openpi
190
- Loading model from: lerobot/pi05_base
191
- ✓ Loaded state dict from model.safetensors
192
- Remapped 812 state dict keys
193
- All keys loaded successfully!
194
-
195
- Traceback (most recent call last):
196
- File "<frozen runpy>", line 198, in _run_module_as_main
197
- File "<frozen runpy>", line 88, in _run_code
198
- File "/workspace/pi05-so100-diverse/lerobot/src/lerobot/scripts/lerobot_train.py", line 575, in <module>
199
- main()
200
- File "/workspace/pi05-so100-diverse/lerobot/src/lerobot/scripts/lerobot_train.py", line 571, in main
201
- train()
202
- File "/workspace/pi05-so100-diverse/lerobot/src/lerobot/configs/parser.py", line 233, in wrapper_inner
203
- response = fn(cfg, *args, **kwargs)
204
- ^^^^^^^^^^^^^^^^^^^^^^^^
205
- File "/workspace/pi05-so100-diverse/lerobot/src/lerobot/scripts/lerobot_train.py", line 419, in train
206
- train_tracker, output_dict = update_policy(
207
- ^^^^^^^^^^^^^^
208
- File "/workspace/pi05-so100-diverse/lerobot/src/lerobot/scripts/lerobot_train.py", line 118, in update_policy
209
- loss, output_dict = policy.forward(batch)
210
- ^^^^^^^^^^^^^^^^^^^^^
211
- File "/workspace/pi05-so100-diverse/lerobot/src/lerobot/policies/pi05/modeling_pi05.py", line 1264, in forward
212
- losses = self.model.forward(images, img_masks, tokens, masks, actions)
213
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
214
- File "/workspace/pi05-so100-diverse/lerobot/src/lerobot/policies/pi05/modeling_pi05.py", line 771, in forward
215
- suffix_out = self._apply_checkpoint(
216
- ^^^^^^^^^^^^^^^^^^^^^^^
217
- File "/workspace/pi05-so100-diverse/lerobot/src/lerobot/policies/pi05/modeling_pi05.py", line 617, in _apply_checkpoint
218
- return func(*args, **kwargs)
219
- ^^^^^^^^^^^^^^^^^^^^^
220
- File "/workspace/pi05-so100-diverse/lerobot/src/lerobot/policies/pi05/modeling_pi05.py", line 761, in forward_func
221
- (_, suffix_out), _ = self.paligemma_with_expert.forward(
222
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
223
- File "/workspace/pi05-so100-diverse/lerobot/src/lerobot/policies/pi05/modeling_pi05.py", line 513, in forward
224
- inputs_embeds = compute_layer_complete(
225
- ^^^^^^^^^^^^^^^^^^^^^^^
226
- File "/workspace/pi05-so100-diverse/lerobot/src/lerobot/policies/pi05/modeling_pi05.py", line 264, in compute_layer_complete
227
- att_output, _ = modeling_gemma.eager_attention_forward(
228
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
229
- File "/usr/local/lib/python3.12/dist-packages/transformers/models/gemma/modeling_gemma.py", line 210, in eager_attention_forward
230
- attn_weights = attn_weights + attention_mask
231
- ~~~~~~~~~~~~~^~~~~~~~~~~~~~~~
232
- torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 1014.00 MiB. GPU 0 has a total capacity of 79.19 GiB of which 222.50 MiB is free. Including non-PyTorch memory, this process has 78.96 GiB memory in use. Of the allocated memory 76.51 GiB is allocated by PyTorch, and 1.86 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
233
- Traceback (most recent call last):
234
- File "<frozen runpy>", line 198, in _run_module_as_main
235
- File "<frozen runpy>", line 88, in _run_code
236
- File "/usr/local/lib/python3.12/dist-packages/accelerate/commands/launch.py", line 1415, in <module>
237
- main()
238
- File "/usr/local/lib/python3.12/dist-packages/accelerate/commands/launch.py", line 1411, in main
239
- launch_command(args)
240
- File "/usr/local/lib/python3.12/dist-packages/accelerate/commands/launch.py", line 1405, in launch_command
241
- simple_launcher(args)
242
- File "/usr/local/lib/python3.12/dist-packages/accelerate/commands/launch.py", line 993, in simple_launcher
243
- raise subprocess.CalledProcessError(returncode=process.returncode, cmd=cmd)
244
- subprocess.CalledProcessError: Command '['/usr/bin/python3.12', '-m', 'lerobot.scripts.lerobot_train', '--dataset.repo_id=so100:/ephemeral/community_dataset_v3:/workspace/pi05-so100-diverse/filtered_index.json:/workspace/pi05-so100-diverse/norm_stats.json', '--policy.path=lerobot/pi05_base', '--policy.train_expert_only=true', '--policy.dtype=bfloat16', '--policy.gradient_checkpointing=false', '--policy.push_to_hub=true', '--policy.repo_id=StrongRoboticsLab/pi05-so100-diverse', '--policy.normalization_mapping={"VISUAL": "IDENTITY", "STATE": "MEAN_STD", "ACTION": "MEAN_STD"}', '--policy.scheduler_warmup_steps=1000', '--policy.scheduler_decay_steps=170000', '--rename_map={"observation.images.image": "observation.images.base_0_rgb", "observation.images.image2": "observation.images.left_wrist_0_rgb"}', '--batch_size=32', '--steps=170000', '--save_freq=500', '--log_freq=50', '--num_workers=4', '--wandb.enable=true', '--wandb.project=pi05-so100-diverse', '--output_dir=/ephemeral/production_run']' returned non-zero exit status 1.
245
- === Training Complete (exit: 1) ===
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
logs/training_20260324_025549.log DELETED
@@ -1,1136 +0,0 @@
1
- === Starting Training ===
2
- The following values were not passed to `accelerate launch` and had defaults used instead:
3
- `--num_processes` was set to a value of `1`
4
- `--num_machines` was set to a value of `1`
5
- `--mixed_precision` was set to a value of `'no'`
6
- `--dynamo_backend` was set to a value of `'no'`
7
- To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`.
8
- WARNING:lerobot.configs.policies:Device 'mps' is not available. Switching to 'cuda'.
9
- WARNING:lerobot.configs.policies:Device 'mps' is not available. Switching to 'cuda'.
10
- INFO 2026-03-24 02:55:56 ot_train.py:198 {'batch_size': 16,
11
- 'checkpoint_path': None,
12
- 'cudnn_deterministic': False,
13
- 'dataset': {'episodes': None,
14
- 'image_transforms': {'enable': False,
15
- 'max_num_transforms': 3,
16
- 'random_order': False,
17
- 'tfs': {'affine': {'kwargs': {'degrees': [-5.0,
18
- 5.0],
19
- 'translate': [0.05,
20
- 0.05]},
21
- 'type': 'RandomAffine',
22
- 'weight': 1.0},
23
- 'brightness': {'kwargs': {'brightness': [0.8,
24
- 1.2]},
25
- 'type': 'ColorJitter',
26
- 'weight': 1.0},
27
- 'contrast': {'kwargs': {'contrast': [0.8,
28
- 1.2]},
29
- 'type': 'ColorJitter',
30
- 'weight': 1.0},
31
- 'hue': {'kwargs': {'hue': [-0.05,
32
- 0.05]},
33
- 'type': 'ColorJitter',
34
- 'weight': 1.0},
35
- 'saturation': {'kwargs': {'saturation': [0.5,
36
- 1.5]},
37
- 'type': 'ColorJitter',
38
- 'weight': 1.0},
39
- 'sharpness': {'kwargs': {'sharpness': [0.5,
40
- 1.5]},
41
- 'type': 'SharpnessJitter',
42
- 'weight': 1.0}}},
43
- 'repo_id': 'so100:/ephemeral/community_dataset_v3:/workspace/pi05-so100-diverse/filtered_index.json:/workspace/pi05-so100-diverse/norm_stats.json',
44
- 'revision': None,
45
- 'root': None,
46
- 'streaming': False,
47
- 'use_imagenet_stats': True,
48
- 'video_backend': 'torchcodec'},
49
- 'env': None,
50
- 'eval': {'batch_size': 50, 'n_episodes': 50, 'use_async_envs': False},
51
- 'eval_freq': 20000,
52
- 'job_name': 'pi05',
53
- 'log_freq': 50,
54
- 'num_workers': 4,
55
- 'optimizer': {'betas': [0.9, 0.95],
56
- 'eps': 1e-08,
57
- 'grad_clip_norm': 1.0,
58
- 'lr': 2.5e-05,
59
- 'type': 'adamw',
60
- 'weight_decay': 0.01},
61
- 'output_dir': '/ephemeral/production_run',
62
- 'peft': None,
63
- 'policy': {'action_expert_variant': 'gemma_300m',
64
- 'chunk_size': 50,
65
- 'compile_mode': 'max-autotune',
66
- 'compile_model': False,
67
- 'device': 'cuda',
68
- 'dtype': 'bfloat16',
69
- 'empty_cameras': 0,
70
- 'freeze_vision_encoder': False,
71
- 'gradient_checkpointing': False,
72
- 'image_resolution': [224, 224],
73
- 'input_features': {'observation.images.base_0_rgb': {'shape': [3,
74
- 224,
75
- 224],
76
- 'type': <FeatureType.VISUAL: 'VISUAL'>},
77
- 'observation.images.left_wrist_0_rgb': {'shape': [3,
78
- 224,
79
- 224],
80
- 'type': <FeatureType.VISUAL: 'VISUAL'>},
81
- 'observation.images.right_wrist_0_rgb': {'shape': [3,
82
- 224,
83
- 224],
84
- 'type': <FeatureType.VISUAL: 'VISUAL'>},
85
- 'observation.state': {'shape': [32],
86
- 'type': <FeatureType.STATE: 'STATE'>}},
87
- 'license': None,
88
- 'max_action_dim': 32,
89
- 'max_period': 4.0,
90
- 'max_state_dim': 32,
91
- 'min_period': 0.004,
92
- 'n_action_steps': 50,
93
- 'n_obs_steps': 1,
94
- 'normalization_mapping': {'ACTION': <NormalizationMode.MEAN_STD: 'MEAN_STD'>,
95
- 'STATE': <NormalizationMode.MEAN_STD: 'MEAN_STD'>,
96
- 'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>},
97
- 'num_inference_steps': 10,
98
- 'optimizer_betas': [0.9, 0.95],
99
- 'optimizer_eps': 1e-08,
100
- 'optimizer_grad_clip_norm': 1.0,
101
- 'optimizer_lr': 2.5e-05,
102
- 'optimizer_weight_decay': 0.01,
103
- 'output_features': {'action': {'shape': [32],
104
- 'type': <FeatureType.ACTION: 'ACTION'>}},
105
- 'paligemma_variant': 'gemma_2b',
106
- 'pretrained_path': 'lerobot/pi05_base',
107
- 'private': None,
108
- 'push_to_hub': True,
109
- 'repo_id': 'StrongRoboticsLab/pi05-so100-diverse',
110
- 'rtc_config': None,
111
- 'scheduler_decay_lr': 2.5e-06,
112
- 'scheduler_decay_steps': 340000,
113
- 'scheduler_warmup_steps': 1000,
114
- 'tags': None,
115
- 'time_sampling_beta_alpha': 1.5,
116
- 'time_sampling_beta_beta': 1.0,
117
- 'time_sampling_offset': 0.001,
118
- 'time_sampling_scale': 0.999,
119
- 'tokenizer_max_length': 200,
120
- 'train_expert_only': True,
121
- 'type': 'pi05',
122
- 'use_amp': False,
123
- 'use_peft': False},
124
- 'rabc_epsilon': 1e-06,
125
- 'rabc_head_mode': 'sparse',
126
- 'rabc_kappa': 0.01,
127
- 'rabc_progress_path': None,
128
- 'rename_map': {'observation.images.image': 'observation.images.base_0_rgb',
129
- 'observation.images.image2': 'observation.images.left_wrist_0_rgb'},
130
- 'resume': False,
131
- 'save_checkpoint': True,
132
- 'save_freq': 500,
133
- 'scheduler': {'decay_lr': 2.5e-06,
134
- 'num_decay_steps': 340000,
135
- 'num_warmup_steps': 1000,
136
- 'peak_lr': 2.5e-05,
137
- 'type': 'cosine_decay_with_warmup'},
138
- 'seed': 1000,
139
- 'steps': 340000,
140
- 'tolerance_s': 0.0001,
141
- 'use_policy_training_preset': True,
142
- 'use_rabc': False,
143
- 'wandb': {'add_tags': True,
144
- 'disable_artifact': False,
145
- 'enable': True,
146
- 'entity': None,
147
- 'mode': None,
148
- 'notes': None,
149
- 'project': 'pi05-so100-diverse',
150
- 'run_id': None}}
151
- INFO 2026-03-24 02:55:58 db_utils.py:117 Logs will be synced with wandb.
152
- INFO 2026-03-24 02:55:58 db_utils.py:118 Track this run --> https://wandb.ai/gptjustin-strong-robotics-lab/pi05-so100-diverse/runs/7zuftzeh
153
- INFO 2026-03-24 02:55:58 ot_train.py:222 Creating dataset
154
- INFO 2026-03-24 02:56:01 ot_train.py:240 Creating policy
155
- INFO 2026-03-24 02:57:56 _client.py:1025 HTTP Request: HEAD https://huggingface.co/lerobot/pi05_base/resolve/main/model.safetensors "HTTP/1.1 302 Found"
156
- INFO 2026-03-24 02:57:56 _client.py:1025 HTTP Request: GET https://huggingface.co/api/models/lerobot/pi05_base/xet-read-token/9e55186ad36e66b95cda57bc47818d9e6237ae30 "HTTP/1.1 200 OK"
157
- WARNING 2026-03-24 02:58:13 ng_pi05.py:1102 Vision embedding key might need handling: paligemma_with_expert.paligemma.model.vision_tower.vision_model.embeddings.patch_embedding.bias
158
- WARNING 2026-03-24 02:58:13 ng_pi05.py:1102 Vision embedding key might need handling: paligemma_with_expert.paligemma.model.vision_tower.vision_model.embeddings.patch_embedding.weight
159
- INFO 2026-03-24 02:58:18 _client.py:1025 HTTP Request: HEAD https://huggingface.co/lerobot/pi05_base/resolve/main/policy_preprocessor.json "HTTP/1.1 307 Temporary Redirect"
160
- INFO 2026-03-24 02:58:18 _client.py:1025 HTTP Request: HEAD https://huggingface.co/api/resolve-cache/models/lerobot/pi05_base/9e55186ad36e66b95cda57bc47818d9e6237ae30/policy_preprocessor.json "HTTP/1.1 200 OK"
161
- INFO 2026-03-24 02:58:18 _client.py:1025 HTTP Request: GET https://huggingface.co/api/resolve-cache/models/lerobot/pi05_base/9e55186ad36e66b95cda57bc47818d9e6237ae30/policy_preprocessor.json "HTTP/1.1 200 OK"
162
- INFO 2026-03-24 02:58:18 _client.py:1025 HTTP Request: HEAD https://huggingface.co/google/paligemma-3b-pt-224/resolve/main/config.json "HTTP/1.1 200 OK"
163
- INFO 2026-03-24 02:58:18 _client.py:1025 HTTP Request: GET https://huggingface.co/google/paligemma-3b-pt-224/resolve/main/config.json "HTTP/1.1 200 OK"
164
- INFO 2026-03-24 02:58:18 _client.py:1025 HTTP Request: HEAD https://huggingface.co/google/paligemma-3b-pt-224/resolve/main/tokenizer_config.json "HTTP/1.1 200 OK"
165
- INFO 2026-03-24 02:58:19 _client.py:1025 HTTP Request: GET https://huggingface.co/google/paligemma-3b-pt-224/resolve/main/tokenizer_config.json "HTTP/1.1 200 OK"
166
- INFO 2026-03-24 02:58:19 _client.py:1025 HTTP Request: GET https://huggingface.co/api/models/google/paligemma-3b-pt-224/tree/main/additional_chat_templates?recursive=false&expand=false "HTTP/1.1 404 Not Found"
167
- INFO 2026-03-24 02:58:19 _client.py:1025 HTTP Request: GET https://huggingface.co/api/models/google/paligemma-3b-pt-224/tree/main?recursive=true&expand=false "HTTP/1.1 200 OK"
168
- INFO 2026-03-24 02:58:19 _client.py:1025 HTTP Request: HEAD https://huggingface.co/google/paligemma-3b-pt-224/resolve/main/tokenizer.json "HTTP/1.1 302 Found"
169
- INFO 2026-03-24 02:58:19 _client.py:1025 HTTP Request: GET https://huggingface.co/api/models/google/paligemma-3b-pt-224/xet-read-token/35e4f46485b4d07967e7e9935bc3786aad50687c "HTTP/1.1 200 OK"
170
- INFO 2026-03-24 02:58:19 _client.py:1025 HTTP Request: HEAD https://huggingface.co/google/paligemma-3b-pt-224/resolve/main/added_tokens.json "HTTP/1.1 200 OK"
171
- INFO 2026-03-24 02:58:19 _client.py:1025 HTTP Request: GET https://huggingface.co/google/paligemma-3b-pt-224/resolve/main/added_tokens.json "HTTP/1.1 200 OK"
172
- INFO 2026-03-24 02:58:19 _client.py:1025 HTTP Request: HEAD https://huggingface.co/google/paligemma-3b-pt-224/resolve/main/special_tokens_map.json "HTTP/1.1 200 OK"
173
- INFO 2026-03-24 02:58:19 _client.py:1025 HTTP Request: GET https://huggingface.co/google/paligemma-3b-pt-224/resolve/main/special_tokens_map.json "HTTP/1.1 200 OK"
174
- INFO 2026-03-24 02:58:19 _client.py:1025 HTTP Request: HEAD https://huggingface.co/google/paligemma-3b-pt-224/resolve/main/chat_template.jinja "HTTP/1.1 404 Not Found"
175
- INFO 2026-03-24 02:58:21 _client.py:1025 HTTP Request: GET https://huggingface.co/api/models/google/paligemma-3b-pt-224 "HTTP/1.1 200 OK"
176
- INFO 2026-03-24 02:58:21 _client.py:1025 HTTP Request: HEAD https://huggingface.co/lerobot/pi05_base/resolve/main/policy_postprocessor.json "HTTP/1.1 307 Temporary Redirect"
177
- INFO 2026-03-24 02:58:21 _client.py:1025 HTTP Request: HEAD https://huggingface.co/api/resolve-cache/models/lerobot/pi05_base/9e55186ad36e66b95cda57bc47818d9e6237ae30/policy_postprocessor.json "HTTP/1.1 200 OK"
178
- INFO 2026-03-24 02:58:21 _client.py:1025 HTTP Request: GET https://huggingface.co/api/resolve-cache/models/lerobot/pi05_base/9e55186ad36e66b95cda57bc47818d9e6237ae30/policy_postprocessor.json "HTTP/1.1 200 OK"
179
- INFO 2026-03-24 02:58:21 ot_train.py:295 Creating optimizer and scheduler
180
- INFO 2026-03-24 02:58:21 ot_train.py:330 Output dir: /ephemeral/production_run
181
- INFO 2026-03-24 02:58:21 ot_train.py:337 cfg.steps=340000 (340K)
182
- INFO 2026-03-24 02:58:21 ot_train.py:338 dataset.num_frames=4923840 (5M)
183
- INFO 2026-03-24 02:58:21 ot_train.py:339 dataset.num_episodes=10155
184
- INFO 2026-03-24 02:58:21 ot_train.py:342 Effective batch size: 16 x 1 = 16
185
- INFO 2026-03-24 02:58:21 ot_train.py:343 num_learnable_params=693422112 (693M)
186
- INFO 2026-03-24 02:58:21 ot_train.py:344 num_total_params=4143404816 (4B)
187
- The PI05 model is a direct port of the OpenPI implementation.
188
- This implementation follows the original OpenPI structure for compatibility.
189
- Original implementation: https://github.com/Physical-Intelligence/openpi
190
- Loading model from: lerobot/pi05_base
191
- ✓ Loaded state dict from model.safetensors
192
- Remapped 812 state dict keys
193
- All keys loaded successfully!
194
-
195
-
196
- WARNING 2026-03-24 02:59:05 db_utils.py:181 WandB logging of key "loss_per_dim" was ignored as its type "<class 'list'>" is not handled by this wrapper.
197
-
198
- WARNING 2026-03-24 02:59:47 db_utils.py:181 WandB logging of key "loss_per_dim" was ignored as its type "<class 'list'>" is not handled by this wrapper.
199
-
200
- WARNING 2026-03-24 03:00:30 db_utils.py:181 WandB logging of key "loss_per_dim" was ignored as its type "<class 'list'>" is not handled by this wrapper.
201
-
202
- WARNING 2026-03-24 03:01:13 db_utils.py:181 WandB logging of key "loss_per_dim" was ignored as its type "<class 'list'>" is not handled by this wrapper.
203
-
204
- WARNING 2026-03-24 03:01:56 db_utils.py:181 WandB logging of key "loss_per_dim" was ignored as its type "<class 'list'>" is not handled by this wrapper.
205
-
206
- WARNING 2026-03-24 03:02:39 db_utils.py:181 WandB logging of key "loss_per_dim" was ignored as its type "<class 'list'>" is not handled by this wrapper.
207
-
208
- WARNING 2026-03-24 03:03:22 db_utils.py:181 WandB logging of key "loss_per_dim" was ignored as its type "<class 'list'>" is not handled by this wrapper.
209
-
210
- WARNING 2026-03-24 03:04:05 db_utils.py:181 WandB logging of key "loss_per_dim" was ignored as its type "<class 'list'>" is not handled by this wrapper.
211
-
212
- WARNING 2026-03-24 03:04:48 db_utils.py:181 WandB logging of key "loss_per_dim" was ignored as its type "<class 'list'>" is not handled by this wrapper.
213
-
214
- WARNING 2026-03-24 03:05:31 db_utils.py:181 WandB logging of key "loss_per_dim" was ignored as its type "<class 'list'>" is not handled by this wrapper.
215
- INFO 2026-03-24 03:05:31 ot_train.py:461 Checkpoint policy after step 500
216
- INFO 2026-03-24 03:05:55 _client.py:1025 HTTP Request: POST https://huggingface.co/api/models/StrongRoboticsLab/pi05-so100-diverse/preupload/main "HTTP/1.1 200 OK"
217
- INFO 2026-03-24 03:05:55 _client.py:1025 HTTP Request: POST https://huggingface.co/StrongRoboticsLab/pi05-so100-diverse.git/info/lfs/objects/batch "HTTP/1.1 200 OK"
218
- INFO 2026-03-24 03:05:55 _client.py:1025 HTTP Request: GET https://huggingface.co/api/models/StrongRoboticsLab/pi05-so100-diverse/xet-write-token/main "HTTP/1.1 200 OK"
219
-
220
-
221
-
222
-
223
  ...timizer_state.safetensors: 0%| | 28.1kB / 2.20GB 
224
-
225
-
226
-
227
  ...d_model/model.safetensors: 0%| | 98.2kB / 9.35GB 
228
-
229
-
230
  ...timizer_state.safetensors: 0%| | 28.1kB / 2.20GB 
231
-
232
-
233
-
234
  ...d_model/model.safetensors: 0%| | 98.2kB / 9.35GB 
235
-
236
-
237
  ...timizer_state.safetensors: 1%| | 14.9MB / 2.20GB 
238
-
239
-
240
-
241
  ...d_model/model.safetensors: 0%| | 98.2kB / 9.35GB 
242
-
243
-
244
-
245
  ...timizer_state.safetensors: 3%|▎ | 64.0MB / 2.20GB 
246
-
247
-
248
-
249
  ...d_model/model.safetensors: 0%| | 98.2kB / 9.35GB 
250
-
251
-
252
-
253
  ...timizer_state.safetensors: 3%|▎ | 66.7MB / 2.20GB 
254
-
255
-
256
-
257
  ...d_model/model.safetensors: 0%| | 98.2kB / 9.35GB 
258
-
259
-
260
-
261
  ...timizer_state.safetensors: 3%|▎ | 66.7MB / 2.20GB 
262
-
263
-
264
-
265
  ...d_model/model.safetensors: 0%| | 98.2kB / 9.35GB 
266
-
267
-
268
  ...timizer_state.safetensors: 3%|▎ | 66.7MB / 2.20GB 
269
-
270
-
271
-
272
  ...d_model/model.safetensors: 0%| | 98.2kB / 9.35GB 
273
-
274
-
275
  ...timizer_state.safetensors: 3%|▎ | 66.7MB / 2.20GB 
276
-
277
-
278
-
279
  ...d_model/model.safetensors: 0%| | 98.2kB / 9.35GB 
280
-
281
-
282
  ...timizer_state.safetensors: 3%|▎ | 66.7MB / 2.20GB 
283
-
284
-
285
-
286
  ...d_model/model.safetensors: 0%| | 98.2kB / 9.35GB 
287
-
288
-
289
  ...timizer_state.safetensors: 3%|▎ | 66.7MB / 2.20GB 
290
-
291
-
292
-
293
  ...d_model/model.safetensors: 0%| | 98.2kB / 9.35GB 
294
-
295
-
296
  ...timizer_state.safetensors: 3%|▎ | 66.7MB / 2.20GB 
297
-
298
-
299
-
300
  ...d_model/model.safetensors: 0%| | 98.2kB / 9.35GB 
301
-
302
-
303
  ...timizer_state.safetensors: 3%|▎ | 66.7MB / 2.20GB 
304
-
305
-
306
-
307
  ...d_model/model.safetensors: 0%| | 98.2kB / 9.35GB 
308
-
309
-
310
  ...timizer_state.safetensors: 3%|▎ | 66.7MB / 2.20GB 
311
-
312
-
313
-
314
  ...d_model/model.safetensors: 0%| | 98.2kB / 9.35GB 
315
-
316
-
317
  ...timizer_state.safetensors: 3%|▎ | 71.5MB / 2.20GB 
318
-
319
-
320
-
321
  ...d_model/model.safetensors: 0%| | 5.09MB / 9.35GB 
322
-
323
-
324
-
325
  ...timizer_state.safetensors: 5%|▌ | 119MB / 2.20GB 
326
-
327
-
328
-
329
  ...d_model/model.safetensors: 1%| | 47.8MB / 9.35GB 
330
-
331
-
332
-
333
  ...timizer_state.safetensors: 6%|▌ | 134MB / 2.20GB 
334
-
335
-
336
-
337
  ...d_model/model.safetensors: 1%| | 66.7MB / 9.35GB 
338
-
339
-
340
-
341
  ...timizer_state.safetensors: 6%|▌ | 134MB / 2.20GB 
342
-
343
-
344
-
345
  ...d_model/model.safetensors: 1%| | 66.7MB / 9.35GB 
346
-
347
-
348
  ...timizer_state.safetensors: 6%|▌ | 134MB / 2.20GB 
349
-
350
-
351
-
352
  ...d_model/model.safetensors: 1%| | 66.7MB / 9.35GB 
353
-
354
-
355
  ...timizer_state.safetensors: 6%|▌ | 134MB / 2.20GB 
356
-
357
-
358
-
359
  ...d_model/model.safetensors: 1%| | 67.1MB / 9.35GB 
360
-
361
-
362
-
363
  ...timizer_state.safetensors: 7%|▋ | 146MB / 2.20GB 
364
-
365
-
366
-
367
  ...d_model/model.safetensors: 1%| | 78.3MB / 9.35GB 
368
-
369
-
370
-
371
  ...timizer_state.safetensors: 9%|▊ | 191MB / 2.20GB 
372
-
373
-
374
-
375
  ...d_model/model.safetensors: 1%|▏ | 125MB / 9.35GB 
376
-
377
-
378
-
379
  ...timizer_state.safetensors: 9%|▉ | 201MB / 2.20GB 
380
-
381
-
382
-
383
  ...d_model/model.safetensors: 2%|▏ | 169MB / 9.35GB 
384
-
385
-
386
-
387
  ...timizer_state.safetensors: 9%|▉ | 207MB / 2.20GB 
388
-
389
-
390
-
391
  ...d_model/model.safetensors: 2%|▏ | 201MB / 9.35GB 
392
-
393
-
394
-
395
  ...timizer_state.safetensors: 12%|█▏ | 256MB / 2.20GB 
396
-
397
-
398
-
399
  ...d_model/model.safetensors: 2%|▏ | 201MB / 9.35GB 
400
-
401
-
402
-
403
  ...timizer_state.safetensors: 12%|█▏ | 267MB / 2.20GB 
404
-
405
-
406
-
407
  ...d_model/model.safetensors: 2%|▏ | 201MB / 9.35GB 
408
-
409
-
410
-
411
  ...timizer_state.safetensors: 12%|█▏ | 272MB / 2.20GB 
412
-
413
-
414
-
415
  ...d_model/model.safetensors: 2%|▏ | 204MB / 9.35GB 
416
-
417
-
418
-
419
  ...timizer_state.safetensors: 14%|█▍ | 307MB / 2.20GB 
420
-
421
-
422
-
423
  ...d_model/model.safetensors: 2%|▏ | 214MB / 9.35GB 
424
-
425
-
426
-
427
  ...timizer_state.safetensors: 15%|█▌ | 334MB / 2.20GB 
428
-
429
-
430
-
431
  ...d_model/model.safetensors: 3%|▎ | 264MB / 9.35GB 
432
-
433
-
434
-
435
  ...timizer_state.safetensors: 15%|█▌ | 334MB / 2.20GB 
436
-
437
-
438
-
439
  ...d_model/model.safetensors: 3%|▎ | 295MB / 9.35GB 
440
-
441
-
442
-
443
  ...timizer_state.safetensors: 16%|█▌ | 348MB / 2.20GB 
444
-
445
-
446
-
447
  ...d_model/model.safetensors: 3%|▎ | 306MB / 9.35GB 
448
-
449
-
450
-
451
  ...timizer_state.safetensors: 18%|█▊ | 397MB / 2.20GB 
452
-
453
-
454
-
455
  ...d_model/model.safetensors: 4%|▎ | 344MB / 9.35GB 
456
-
457
-
458
-
459
  ...timizer_state.safetensors: 18%|█▊ | 402MB / 2.20GB 
460
-
461
-
462
-
463
  ...d_model/model.safetensors: 4%|▍ | 392MB / 9.35GB 
464
-
465
-
466
-
467
  ...timizer_state.safetensors: 19%|█▉ | 425MB / 2.20GB 
468
-
469
-
470
-
471
  ...d_model/model.safetensors: 4%|▍ | 402MB / 9.35GB 
472
-
473
-
474
-
475
  ...timizer_state.safetensors: 23%|██▎ | 498MB / 2.20GB 
476
-
477
-
478
-
479
  ...d_model/model.safetensors: 4%|▍ | 402MB / 9.35GB 
480
-
481
-
482
-
483
  ...timizer_state.safetensors: 26%|██▌ | 560MB / 2.20GB 
484
-
485
-
486
-
487
  ...d_model/model.safetensors: 5%|▌ | 495MB / 9.35GB 
488
-
489
-
490
-
491
  ...timizer_state.safetensors: 29%|██▉ | 637MB / 2.20GB 
492
-
493
-
494
-
495
  ...d_model/model.safetensors: 8%|▊ | 779MB / 9.35GB 
496
-
497
-
498
-
499
  ...timizer_state.safetensors: 32%|███▏ | 706MB / 2.20GB 
500
-
501
-
502
-
503
  ...d_model/model.safetensors: 11%|█ | 1.02GB / 9.35GB 
504
-
505
-
506
-
507
  ...timizer_state.safetensors: 36%|███▌ | 793MB / 2.20GB 
508
-
509
-
510
-
511
  ...d_model/model.safetensors: 13%|█▎ | 1.24GB / 9.35GB 
512
-
513
-
514
-
515
  ...timizer_state.safetensors: 38%|███▊ | 824MB / 2.20GB 
516
-
517
-
518
-
519
  ...d_model/model.safetensors: 15%|█▌ | 1.43GB / 9.35GB 
520
-
521
-
522
-
523
  ...timizer_state.safetensors: 40%|████ | 884MB / 2.20GB 
524
-
525
-
526
-
527
  ...d_model/model.safetensors: 17%|█▋ | 1.62GB / 9.35GB 
528
-
529
-
530
-
531
  ...timizer_state.safetensors: 43%|████▎ | 940MB / 2.20GB 
532
-
533
-
534
-
535
  ...d_model/model.safetensors: 19%|█▉ | 1.82GB / 9.35GB 
536
-
537
-
538
-
539
  ...timizer_state.safetensors: 46%|████▋ | 1.02GB / 2.20GB 
540
-
541
-
542
-
543
  ...d_model/model.safetensors: 21%|██▏ | 2.01GB / 9.35GB 
544
-
545
-
546
-
547
  ...timizer_state.safetensors: 50%|████▉ | 1.09GB / 2.20GB 
548
-
549
-
550
-
551
  ...d_model/model.safetensors: 22%|██▏ | 2.08GB / 9.35GB 
552
-
553
-
554
-
555
  ...timizer_state.safetensors: 53%|█████▎ | 1.17GB / 2.20GB 
556
-
557
-
558
-
559
  ...d_model/model.safetensors: 22%|██▏ | 2.08GB / 9.35GB 
560
-
561
-
562
-
563
  ...timizer_state.safetensors: 56%|█████▌ | 1.23GB / 2.20GB 
564
-
565
-
566
-
567
  ...d_model/model.safetensors: 24%|██▎ | 2.22GB / 9.35GB 
568
-
569
-
570
-
571
  ...timizer_state.safetensors: 59%|█████▉ | 1.30GB / 2.20GB 
572
-
573
-
574
-
575
  ...d_model/model.safetensors: 24%|██▎ | 2.22GB / 9.35GB 
576
-
577
-
578
-
579
  ...timizer_state.safetensors: 63%|██████▎ | 1.38GB / 2.20GB 
580
-
581
-
582
-
583
  ...d_model/model.safetensors: 24%|██▍ | 2.27GB / 9.35GB 
584
-
585
-
586
-
587
  ...timizer_state.safetensors: 66%|██████▌ | 1.44GB / 2.20GB 
588
-
589
-
590
-
591
  ...d_model/model.safetensors: 27%|██▋ | 2.50GB / 9.35GB 
592
-
593
-
594
-
595
  ...timizer_state.safetensors: 68%|██████▊ | 1.50GB / 2.20GB 
596
-
597
-
598
-
599
  ...d_model/model.safetensors: 28%|██▊ | 2.61GB / 9.35GB 
600
-
601
-
602
-
603
  ...timizer_state.safetensors: 71%|███████ | 1.55GB / 2.20GB 
604
-
605
-
606
-
607
  ...d_model/model.safetensors: 28%|██▊ | 2.62GB / 9.35GB 
608
-
609
-
610
-
611
  ...timizer_state.safetensors: 74%|███████▍ | 1.63GB / 2.20GB 
612
-
613
-
614
-
615
  ...d_model/model.safetensors: 28%|██▊ | 2.64GB / 9.35GB 
616
-
617
-
618
-
619
  ...timizer_state.safetensors: 78%|███████▊ | 1.71GB / 2.20GB 
620
-
621
-
622
-
623
  ...d_model/model.safetensors: 29%|██▉ | 2.71GB / 9.35GB 
624
-
625
-
626
-
627
  ...timizer_state.safetensors: 81%|████████ | 1.77GB / 2.20GB 
628
-
629
-
630
-
631
  ...d_model/model.safetensors: 30%|██▉ | 2.78GB / 9.35GB 
632
-
633
-
634
-
635
  ...timizer_state.safetensors: 84%|████████▍ | 1.84GB / 2.20GB 
636
-
637
-
638
-
639
  ...d_model/model.safetensors: 31%|███ | 2.86GB / 9.35GB 
640
-
641
-
642
-
643
  ...timizer_state.safetensors: 87%|████████▋ | 1.90GB / 2.20GB 
644
-
645
-
646
-
647
  ...d_model/model.safetensors: 31%|███ | 2.92GB / 9.35GB 
648
-
649
-
650
-
651
  ...timizer_state.safetensors: 90%|████████▉ | 1.97GB / 2.20GB 
652
-
653
-
654
-
655
  ...d_model/model.safetensors: 32%|███▏ | 2.98GB / 9.35GB 
656
-
657
-
658
  ...timizer_state.safetensors: 91%|█████████ | 1.99GB / 2.20GB 
659
-
660
-
661
-
662
  ...d_model/model.safetensors: 32%|███▏ | 3.02GB / 9.35GB 
663
-
664
-
665
  ...timizer_state.safetensors: 93%|█████████▎| 2.04GB / 2.20GB 
666
-
667
-
668
-
669
  ...d_model/model.safetensors: 33%|███▎ | 3.08GB / 9.35GB 
670
-
671
-
672
-
673
  ...timizer_state.safetensors: 96%|█████████▌| 2.10GB / 2.20GB 
674
-
675
-
676
-
677
  ...d_model/model.safetensors: 33%|███▎ | 3.09GB / 9.35GB 
678
-
679
-
680
-
681
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
682
-
683
-
684
-
685
  ...d_model/model.safetensors: 35%|███▍ | 3.23GB / 9.35GB 
686
-
687
-
688
-
689
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
690
-
691
-
692
-
693
  ...d_model/model.safetensors: 38%|███▊ | 3.53GB / 9.35GB 
694
-
695
-
696
-
697
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
698
-
699
-
700
-
701
  ...d_model/model.safetensors: 40%|████ | 3.78GB / 9.35GB 
702
-
703
-
704
-
705
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
706
-
707
-
708
-
709
  ...d_model/model.safetensors: 43%|████▎ | 4.03GB / 9.35GB 
710
-
711
-
712
-
713
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
714
-
715
-
716
-
717
  ...d_model/model.safetensors: 45%|████▌ | 4.23GB / 9.35GB 
718
-
719
-
720
-
721
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
722
-
723
-
724
-
725
  ...d_model/model.safetensors: 48%|████▊ | 4.46GB / 9.35GB 
726
-
727
-
728
-
729
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
730
-
731
-
732
-
733
  ...d_model/model.safetensors: 50%|█████ | 4.68GB / 9.35GB 
734
-
735
-
736
-
737
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
738
-
739
-
740
-
741
  ...d_model/model.safetensors: 52%|█████▏ | 4.86GB / 9.35GB 
742
-
743
-
744
-
745
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
746
-
747
-
748
-
749
  ...d_model/model.safetensors: 54%|█████▍ | 5.03GB / 9.35GB 
750
-
751
-
752
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
753
-
754
-
755
-
756
  ...d_model/model.safetensors: 56%|█████▌ | 5.22GB / 9.35GB 
757
-
758
-
759
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
760
-
761
-
762
-
763
  ...d_model/model.safetensors: 58%|█████▊ | 5.38GB / 9.35GB 
764
-
765
-
766
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
767
-
768
-
769
-
770
  ...d_model/model.safetensors: 58%|█████▊ | 5.38GB / 9.35GB 
771
-
772
-
773
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
774
-
775
-
776
-
777
  ...d_model/model.safetensors: 58%|█████▊ | 5.38GB / 9.35GB 
778
-
779
-
780
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
781
-
782
-
783
-
784
  ...d_model/model.safetensors: 58%|█████▊ | 5.39GB / 9.35GB 
785
-
786
-
787
-
788
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
789
-
790
-
791
-
792
  ...d_model/model.safetensors: 58%|█████▊ | 5.41GB / 9.35GB 
793
-
794
-
795
-
796
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
797
-
798
-
799
-
800
  ...d_model/model.safetensors: 60%|██████ | 5.64GB / 9.35GB 
801
-
802
-
803
-
804
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
805
-
806
-
807
-
808
  ...d_model/model.safetensors: 63%|██████▎ | 5.87GB / 9.35GB 
809
-
810
-
811
-
812
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
813
-
814
-
815
-
816
  ...d_model/model.safetensors: 65%|██████▌ | 6.10GB / 9.35GB 
817
-
818
-
819
-
820
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
821
-
822
-
823
-
824
  ...d_model/model.safetensors: 67%|██████▋ | 6.31GB / 9.35GB 
825
-
826
-
827
-
828
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
829
-
830
-
831
-
832
  ...d_model/model.safetensors: 70%|██████▉ | 6.51GB / 9.35GB 
833
-
834
-
835
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
836
-
837
-
838
-
839
  ...d_model/model.safetensors: 72%|███████▏ | 6.70GB / 9.35GB 
840
-
841
-
842
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
843
-
844
-
845
-
846
  ...d_model/model.safetensors: 74%|███████▎ | 6.89GB / 9.35GB 
847
-
848
-
849
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
850
-
851
-
852
-
853
  ...d_model/model.safetensors: 75%|███████▍ | 6.98GB / 9.35GB 
854
-
855
-
856
-
857
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
858
-
859
-
860
-
861
  ...d_model/model.safetensors: 75%|███████▍ | 6.99GB / 9.35GB 
862
-
863
-
864
-
865
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
866
-
867
-
868
-
869
  ...d_model/model.safetensors: 75%|███████▍ | 7.01GB / 9.35GB 
870
-
871
-
872
-
873
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
874
-
875
-
876
-
877
  ...d_model/model.safetensors: 75%|███████▌ | 7.06GB / 9.35GB 
878
-
879
-
880
-
881
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
882
-
883
-
884
-
885
  ...d_model/model.safetensors: 77%|███████▋ | 7.24GB / 9.35GB 
886
-
887
-
888
-
889
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
890
-
891
-
892
-
893
  ...d_model/model.safetensors: 80%|███████▉ | 7.45GB / 9.35GB 
894
-
895
-
896
-
897
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
898
-
899
-
900
-
901
  ...d_model/model.safetensors: 82%|████████▏ | 7.65GB / 9.35GB 
902
-
903
-
904
-
905
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
906
-
907
-
908
-
909
  ...d_model/model.safetensors: 84%|████████▍ | 7.85GB / 9.35GB 
910
-
911
-
912
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
913
-
914
-
915
-
916
  ...d_model/model.safetensors: 86%|████████▌ | 8.05GB / 9.35GB 
917
-
918
-
919
-
920
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
921
-
922
-
923
-
924
  ...d_model/model.safetensors: 88%|████████▊ | 8.25GB / 9.35GB 
925
-
926
-
927
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
928
-
929
-
930
-
931
  ...d_model/model.safetensors: 90%|█████████ | 8.45GB / 9.35GB 
932
-
933
-
934
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
935
-
936
-
937
-
938
  ...d_model/model.safetensors: 93%|█████████▎| 8.67GB / 9.35GB 
939
-
940
-
941
-
942
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
943
-
944
-
945
-
946
  ...d_model/model.safetensors: 95%|█████████▍| 8.86GB / 9.35GB 
947
-
948
-
949
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
950
-
951
-
952
-
953
  ...d_model/model.safetensors: 97%|█████████▋| 9.07GB / 9.35GB 
954
-
955
-
956
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
957
-
958
-
959
-
960
  ...d_model/model.safetensors: 99%|█████████▉| 9.27GB / 9.35GB 
961
-
962
-
963
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
964
-
965
-
966
-
967
  ...d_model/model.safetensors: 100%|█████████▉| 9.34GB / 9.35GB 
968
-
969
-
970
-
971
-
972
  ...zer_processor.safetensors: 14%|█▍ | 109B / 768B 
973
-
974
-
975
-
976
-
977
-
978
  ...zer_processor.safetensors: 14%|█▍ | 109B / 768B 
979
-
980
-
981
-
982
-
983
-
984
-
985
  ...ate/rng_state.safetensors: 14%|█▍ | 2.23kB / 15.7kB 
986
-
987
-
988
  ...timizer_state.safetensors: 98%|█████████▊| 2.16GB / 2.20GB 
989
-
990
-
991
-
992
  ...d_model/model.safetensors: 100%|█████████▉| 9.34GB / 9.35GB 
993
-
994
-
995
-
996
-
997
  ...zer_processor.safetensors: 14%|█▍ | 109B / 768B 
998
-
999
-
1000
-
1001
-
1002
-
1003
  ...zer_processor.safetensors: 14%|█▍ | 109B / 768B 
1004
-
1005
-
1006
-
1007
-
1008
-
1009
-
1010
  ...ate/rng_state.safetensors: 14%|█▍ | 2.23kB / 15.7kB 
1011
-
1012
-
1013
-
1014
  ...timizer_state.safetensors: 99%|█████████▉| 2.17GB / 2.20GB 
1015
-
1016
-
1017
-
1018
  ...d_model/model.safetensors: 100%|█████████▉| 9.34GB / 9.35GB 
1019
-
1020
-
1021
-
1022
-
1023
  ...zer_processor.safetensors: 43%|████▎ | 327B / 768B 
1024
-
1025
-
1026
-
1027
-
1028
-
1029
  ...zer_processor.safetensors: 43%|████▎ | 327B / 768B 
1030
-
1031
-
1032
-
1033
-
1034
-
1035
-
1036
  ...ate/rng_state.safetensors: 43%|████▎ | 6.69kB / 15.7kB 
1037
-
1038
-
1039
-
1040
  ...timizer_state.safetensors: 99%|█████████▉| 2.18GB / 2.20GB 
1041
-
1042
-
1043
-
1044
  ...d_model/model.safetensors: 100%|█████████▉| 9.35GB / 9.35GB 
1045
-
1046
-
1047
-
1048
-
1049
  ...zer_processor.safetensors: 70%|██████▉ | 537B / 768B 
1050
-
1051
-
1052
-
1053
-
1054
-
1055
  ...zer_processor.safetensors: 70%|██████▉ | 537B / 768B 
1056
-
1057
-
1058
-
1059
-
1060
-
1061
-
1062
  ...ate/rng_state.safetensors: 70%|██████▉ | 11.0kB / 15.7kB 
1063
-
1064
-
1065
-
1066
  ...timizer_state.safetensors: 100%|█████████▉| 2.19GB / 2.20GB 
1067
-
1068
-
1069
-
1070
  ...d_model/model.safetensors: 100%|█████████▉| 9.35GB / 9.35GB 
1071
-
1072
-
1073
-
1074
-
1075
  ...zer_processor.safetensors: 98%|█████████▊| 755B / 768B 
1076
-
1077
-
1078
-
1079
-
1080
-
1081
  ...zer_processor.safetensors: 98%|█████████▊| 755B / 768B 
1082
-
1083
-
1084
-
1085
-
1086
-
1087
-
1088
  ...ate/rng_state.safetensors: 98%|█████████▊| 15.4kB / 15.7kB 
1089
-
1090
-
1091
-
1092
  ...timizer_state.safetensors: 100%|█████████▉| 2.19GB / 2.20GB 
1093
-
1094
-
1095
-
1096
  ...d_model/model.safetensors: 100%|█████████▉| 9.35GB / 9.35GB 
1097
-
1098
-
1099
-
1100
-
1101
  ...zer_processor.safetensors: 99%|█████████▉| 763B / 768B 
1102
-
1103
-
1104
-
1105
-
1106
-
1107
  ...zer_processor.safetensors: 99%|█████████▉| 763B / 768B 
1108
-
1109
-
1110
-
1111
-
1112
-
1113
-
1114
  ...ate/rng_state.safetensors: 99%|█████████▉| 15.6kB / 15.7kB 
1115
-
1116
-
1117
-
1118
  ...timizer_state.safetensors: 100%|█████████▉| 2.19GB / 2.20GB 
1119
-
1120
-
1121
-
1122
  ...d_model/model.safetensors: 100%|█████████▉| 9.35GB / 9.35GB 
1123
-
1124
-
1125
-
1126
-
1127
  ...zer_processor.safetensors: 99%|█████████▉| 763B / 768B 
1128
-
1129
-
1130
-
1131
-
1132
-
1133
  ...zer_processor.safetensors: 99%|█████████▉| 763B / 768B 
1134
-
1135
-
1136
-
1137
-
1138
-
1139
-
1140
  ...ate/rng_state.safetensors: 99%|█████████▉| 15.6kB / 15.7kB 
1141
-
1142
-
1143
  ...timizer_state.safetensors: 100%|█████████▉| 2.19GB / 2.20GB 
1144
-
1145
-
1146
-
1147
  ...d_model/model.safetensors: 100%|█████████▉| 9.35GB / 9.35GB 
1148
-
1149
-
1150
-
1151
-
1152
  ...zer_processor.safetensors: 99%|█████████▉| 763B / 768B 
1153
-
1154
-
1155
-
1156
-
1157
-
1158
  ...zer_processor.safetensors: 99%|█████████▉| 763B / 768B 
1159
-
1160
-
1161
-
1162
-
1163
-
1164
-
1165
  ...ate/rng_state.safetensors: 99%|█████████▉| 15.6kB / 15.7kB 
1166
-
1167
-
1168
  ...timizer_state.safetensors: 100%|█████████▉| 2.19GB / 2.20GB 
1169
-
1170
-
1171
-
1172
  ...d_model/model.safetensors: 100%|█████████▉| 9.35GB / 9.35GB 
1173
-
1174
-
1175
-
1176
-
1177
  ...zer_processor.safetensors: 99%|█████████▉| 763B / 768B 
1178
-
1179
-
1180
-
1181
-
1182
-
1183
  ...zer_processor.safetensors: 99%|█████████▉| 763B / 768B 
1184
-
1185
-
1186
-
1187
-
1188
-
1189
-
1190
  ...ate/rng_state.safetensors: 99%|█████████▉| 15.6kB / 15.7kB 
1191
-
1192
-
1193
  ...timizer_state.safetensors: 100%|██████████| 2.20GB / 2.20GB 
1194
-
1195
-
1196
-
1197
  ...d_model/model.safetensors: 100%|█████████▉| 9.35GB / 9.35GB 
1198
-
1199
-
1200
-
1201
-
1202
  ...zer_processor.safetensors: 100%|██████████| 768B / 768B 
1203
-
1204
-
1205
-
1206
-
1207
-
1208
  ...zer_processor.safetensors: 100%|██████████| 768B / 768B 
1209
-
1210
-
1211
-
1212
-
1213
-
1214
-
1215
  ...ate/rng_state.safetensors: 100%|██████████| 15.7kB / 15.7kB 
1216
-
1217
-
1218
-
1219
  ...timizer_state.safetensors: 100%|██████████| 2.20GB / 2.20GB 
1220
-
1221
-
1222
-
1223
  ...d_model/model.safetensors: 100%|█████████▉| 9.35GB / 9.35GB 
1224
-
1225
-
1226
-
1227
-
1228
  ...zer_processor.safetensors: 100%|██████████| 768B / 768B 
1229
-
1230
-
1231
-
1232
-
1233
-
1234
  ...zer_processor.safetensors: 100%|██████████| 768B / 768B 
1235
-
1236
-
1237
-
1238
-
1239
-
1240
-
1241
  ...ate/rng_state.safetensors: 100%|██████████| 15.7kB / 15.7kB 
1242
-
1243
-
1244
  ...timizer_state.safetensors: 100%|██████████| 2.20GB / 2.20GB 
1245
-
1246
-
1247
-
1248
  ...d_model/model.safetensors: 100%|█████████▉| 9.35GB / 9.35GB 
1249
-
1250
-
1251
-
1252
-
1253
  ...zer_processor.safetensors: 100%|██████████| 768B / 768B 
1254
-
1255
-
1256
-
1257
-
1258
-
1259
  ...zer_processor.safetensors: 100%|██████████| 768B / 768B 
1260
-
1261
-
1262
-
1263
-
1264
-
1265
-
1266
  ...ate/rng_state.safetensors: 100%|██████████| 15.7kB / 15.7kB 
1267
-
1268
-
1269
  ...timizer_state.safetensors: 100%|██████████| 2.20GB / 2.20GB 
1270
-
1271
-
1272
-
1273
  ...d_model/model.safetensors: 100%|█████████▉| 9.35GB / 9.35GB 
1274
-
1275
-
1276
-
1277
-
1278
  ...zer_processor.safetensors: 100%|██████████| 768B / 768B 
1279
-
1280
-
1281
-
1282
-
1283
-
1284
  ...zer_processor.safetensors: 100%|██████████| 768B / 768B 
1285
-
1286
-
1287
-
1288
-
1289
-
1290
-
1291
  ...ate/rng_state.safetensors: 100%|██████████| 15.7kB / 15.7kB 
1292
-
1293
-
1294
  ...timizer_state.safetensors: 100%|██████████| 2.20GB / 2.20GB 
1295
-
1296
-
1297
-
1298
  ...d_model/model.safetensors: 100%|█████████▉| 9.35GB / 9.35GB 
1299
-
1300
-
1301
-
1302
-
1303
  ...zer_processor.safetensors: 100%|██████████| 768B / 768B 
1304
-
1305
-
1306
-
1307
-
1308
-
1309
  ...zer_processor.safetensors: 100%|██████████| 768B / 768B 
1310
-
1311
-
1312
-
1313
-
1314
-
1315
-
1316
  ...ate/rng_state.safetensors: 100%|██████████| 15.7kB / 15.7kB 
1317
-
1318
-
1319
  ...timizer_state.safetensors: 100%|██████████| 2.20GB / 2.20GB 
1320
-
1321
-
1322
-
1323
  ...d_model/model.safetensors: 100%|█████████▉| 9.35GB / 9.35GB 
1324
-
1325
-
1326
-
1327
-
1328
  ...zer_processor.safetensors: 100%|██████████| 768B / 768B 
1329
-
1330
-
1331
-
1332
-
1333
-
1334
  ...zer_processor.safetensors: 100%|██████████| 768B / 768B 
1335
-
1336
-
1337
-
1338
-
1339
-
1340
-
1341
  ...ate/rng_state.safetensors: 100%|██████████| 15.7kB / 15.7kB 
1342
-
1343
-
1344
  ...timizer_state.safetensors: 100%|██████████| 2.20GB / 2.20GB 
1345
-
1346
-
1347
-
1348
  ...d_model/model.safetensors: 100%|█████████▉| 9.35GB / 9.35GB 
1349
-
1350
-
1351
-
1352
-
1353
  ...zer_processor.safetensors: 100%|██████████| 768B / 768B 
1354
-
1355
-
1356
-
1357
-
1358
-
1359
  ...zer_processor.safetensors: 100%|██████████| 768B / 768B 
1360
-
1361
-
1362
-
1363
-
1364
-
1365
-
1366
  ...ate/rng_state.safetensors: 100%|██████████| 15.7kB / 15.7kB 
1367
-
1368
-
1369
  ...timizer_state.safetensors: 100%|██████████| 2.20GB / 2.20GB
1370
-
1371
  ...d_model/model.safetensors: 100%|█████████▉| 9.35GB / 9.35GB
1372
-
1373
  ...zer_processor.safetensors: 100%|██████████| 768B / 768B
1374
-
1375
  ...zer_processor.safetensors: 100%|██████████| 768B / 768B
1376
-
1377
  ...ate/rng_state.safetensors: 100%|██████████| 15.7kB / 15.7kB
1378
- INFO 2026-03-24 03:06:18 _client.py:1025 HTTP Request: POST https://huggingface.co/api/models/StrongRoboticsLab/pi05-so100-diverse/commit/main "HTTP/1.1 200 OK"
1379
- INFO 2026-03-24 03:06:18 ot_train.py:485 Uploaded checkpoint step 500 to HF
1380
- Traceback (most recent call last):
1381
- File "<frozen runpy>", line 198, in _run_module_as_main
1382
- File "<frozen runpy>", line 88, in _run_code
1383
- File "/workspace/pi05-so100-diverse/lerobot/src/lerobot/scripts/lerobot_train.py", line 575, in <module>
1384
- main()
1385
- File "/workspace/pi05-so100-diverse/lerobot/src/lerobot/scripts/lerobot_train.py", line 571, in main
1386
- train()
1387
- File "/workspace/pi05-so100-diverse/lerobot/src/lerobot/configs/parser.py", line 233, in wrapper_inner
1388
- response = fn(cfg, *args, **kwargs)
1389
- ^^^^^^^^^^^^^^^^^^^^^^^^
1390
- File "/workspace/pi05-so100-diverse/lerobot/src/lerobot/scripts/lerobot_train.py", line 493, in train
1391
- wandb_logger.log_policy(checkpoint_dir)
1392
- File "/workspace/pi05-so100-diverse/lerobot/src/lerobot/rl/wandb_utils.py", line 129, in log_policy
1393
- artifact = self._wandb.Artifact(artifact_name, type="model")
1394
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
1395
- File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/artifacts/artifact.py", line 201, in __init__
1396
- self._name: str = validate_artifact_name(name) # includes version after saving
1397
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
1398
- File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/artifacts/_validators.py", line 72, in validate_artifact_name
1399
- raise ValueError(
1400
- ValueError: Artifact name is longer than 128 characters: 'policy_pi05-seed_1000-dataset_so100__ephemeral_community_dataset_v3__workspace_pi05-so100-diverse_filtered_index.json__workspace ...'
1401
- Traceback (most recent call last):
1402
- File "<frozen runpy>", line 198, in _run_module_as_main
1403
- File "<frozen runpy>", line 88, in _run_code
1404
- File "/usr/local/lib/python3.12/dist-packages/accelerate/commands/launch.py", line 1415, in <module>
1405
- main()
1406
- File "/usr/local/lib/python3.12/dist-packages/accelerate/commands/launch.py", line 1411, in main
1407
- launch_command(args)
1408
- File "/usr/local/lib/python3.12/dist-packages/accelerate/commands/launch.py", line 1405, in launch_command
1409
- simple_launcher(args)
1410
- File "/usr/local/lib/python3.12/dist-packages/accelerate/commands/launch.py", line 993, in simple_launcher
1411
- raise subprocess.CalledProcessError(returncode=process.returncode, cmd=cmd)
1412
- subprocess.CalledProcessError: Command '['/usr/bin/python3.12', '-m', 'lerobot.scripts.lerobot_train', '--dataset.repo_id=so100:/ephemeral/community_dataset_v3:/workspace/pi05-so100-diverse/filtered_index.json:/workspace/pi05-so100-diverse/norm_stats.json', '--policy.path=lerobot/pi05_base', '--policy.train_expert_only=true', '--policy.dtype=bfloat16', '--policy.gradient_checkpointing=false', '--policy.push_to_hub=true', '--policy.repo_id=StrongRoboticsLab/pi05-so100-diverse', '--policy.normalization_mapping={"VISUAL": "IDENTITY", "STATE": "MEAN_STD", "ACTION": "MEAN_STD"}', '--policy.scheduler_warmup_steps=1000', '--policy.scheduler_decay_steps=340000', '--rename_map={"observation.images.image": "observation.images.base_0_rgb", "observation.images.image2": "observation.images.left_wrist_0_rgb"}', '--batch_size=16', '--steps=340000', '--save_freq=500', '--log_freq=50', '--num_workers=4', '--wandb.enable=true', '--wandb.project=pi05-so100-diverse', '--output_dir=/ephemeral/production_run']' returned non-zero exit status 1.
1413
- === Training Complete (exit: 1) ===
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
0
  ...timizer_state.safetensors: 0%| | 28.1kB / 2.20GB 
 
 
 
1
  ...d_model/model.safetensors: 0%| | 98.2kB / 9.35GB 
 
 
2
  ...timizer_state.safetensors: 0%| | 28.1kB / 2.20GB 
 
 
 
3
  ...d_model/model.safetensors: 0%| | 98.2kB / 9.35GB 
 
 
4
  ...timizer_state.safetensors: 1%| | 14.9MB / 2.20GB 
 
 
 
5
  ...d_model/model.safetensors: 0%| | 98.2kB / 9.35GB 
 
 
 
6
  ...timizer_state.safetensors: 3%|▎ | 64.0MB / 2.20GB 
 
 
 
7
  ...d_model/model.safetensors: 0%| | 98.2kB / 9.35GB 
 
 
 
8
  ...timizer_state.safetensors: 3%|▎ | 66.7MB / 2.20GB 
 
 
 
9
  ...d_model/model.safetensors: 0%| | 98.2kB / 9.35GB 
 
 
 
10
  ...timizer_state.safetensors: 3%|▎ | 66.7MB / 2.20GB 
 
 
 
11
  ...d_model/model.safetensors: 0%| | 98.2kB / 9.35GB 
 
 
12
  ...timizer_state.safetensors: 3%|▎ | 66.7MB / 2.20GB 
 
 
 
13
  ...d_model/model.safetensors: 0%| | 98.2kB / 9.35GB 
 
 
14
  ...timizer_state.safetensors: 3%|▎ | 66.7MB / 2.20GB 
 
 
 
15
  ...d_model/model.safetensors: 0%| | 98.2kB / 9.35GB 
 
 
16
  ...timizer_state.safetensors: 3%|▎ | 66.7MB / 2.20GB 
 
 
 
17
  ...d_model/model.safetensors: 0%| | 98.2kB / 9.35GB 
 
 
18
  ...timizer_state.safetensors: 3%|▎ | 66.7MB / 2.20GB 
 
 
 
19
  ...d_model/model.safetensors: 0%| | 98.2kB / 9.35GB 
 
 
20
  ...timizer_state.safetensors: 3%|▎ | 66.7MB / 2.20GB 
 
 
 
21
  ...d_model/model.safetensors: 0%| | 98.2kB / 9.35GB 
 
 
22
  ...timizer_state.safetensors: 3%|▎ | 66.7MB / 2.20GB 
 
 
 
23
  ...d_model/model.safetensors: 0%| | 98.2kB / 9.35GB 
 
 
24
  ...timizer_state.safetensors: 3%|▎ | 66.7MB / 2.20GB 
 
 
 
25
  ...d_model/model.safetensors: 0%| | 98.2kB / 9.35GB 
 
 
26
  ...timizer_state.safetensors: 3%|▎ | 71.5MB / 2.20GB 
 
 
 
27
  ...d_model/model.safetensors: 0%| | 5.09MB / 9.35GB 
 
 
 
28
  ...timizer_state.safetensors: 5%|▌ | 119MB / 2.20GB 
 
 
 
29
  ...d_model/model.safetensors: 1%| | 47.8MB / 9.35GB 
 
 
 
30
  ...timizer_state.safetensors: 6%|▌ | 134MB / 2.20GB 
 
 
 
31
  ...d_model/model.safetensors: 1%| | 66.7MB / 9.35GB 
 
 
 
32
  ...timizer_state.safetensors: 6%|▌ | 134MB / 2.20GB 
 
 
 
33
  ...d_model/model.safetensors: 1%| | 66.7MB / 9.35GB 
 
 
34
  ...timizer_state.safetensors: 6%|▌ | 134MB / 2.20GB 
 
 
 
35
  ...d_model/model.safetensors: 1%| | 66.7MB / 9.35GB 
 
 
36
  ...timizer_state.safetensors: 6%|▌ | 134MB / 2.20GB 
 
 
 
37
  ...d_model/model.safetensors: 1%| | 67.1MB / 9.35GB 
 
 
 
38
  ...timizer_state.safetensors: 7%|▋ | 146MB / 2.20GB 
 
 
 
39
  ...d_model/model.safetensors: 1%| | 78.3MB / 9.35GB 
 
 
 
40
  ...timizer_state.safetensors: 9%|▊ | 191MB / 2.20GB 
 
 
 
41
  ...d_model/model.safetensors: 1%|▏ | 125MB / 9.35GB 
 
 
 
42
  ...timizer_state.safetensors: 9%|▉ | 201MB / 2.20GB 
 
 
 
43
  ...d_model/model.safetensors: 2%|▏ | 169MB / 9.35GB 
 
 
 
44
  ...timizer_state.safetensors: 9%|▉ | 207MB / 2.20GB 
 
 
 
45
  ...d_model/model.safetensors: 2%|▏ | 201MB / 9.35GB 
 
 
 
46
  ...timizer_state.safetensors: 12%|█▏ | 256MB / 2.20GB 
 
 
 
47
  ...d_model/model.safetensors: 2%|▏ | 201MB / 9.35GB 
 
 
 
48
  ...timizer_state.safetensors: 12%|█▏ | 267MB / 2.20GB 
 
 
 
49
  ...d_model/model.safetensors: 2%|▏ | 201MB / 9.35GB 
 
 
 
50
  ...timizer_state.safetensors: 12%|█▏ | 272MB / 2.20GB 
 
 
 
51
  ...d_model/model.safetensors: 2%|▏ | 204MB / 9.35GB 
 
 
 
52
  ...timizer_state.safetensors: 14%|█▍ | 307MB / 2.20GB 
 
 
 
53
  ...d_model/model.safetensors: 2%|▏ | 214MB / 9.35GB 
 
 
 
54
  ...timizer_state.safetensors: 15%|█▌ | 334MB / 2.20GB 
 
 
 
55
  ...d_model/model.safetensors: 3%|▎ | 264MB / 9.35GB 
 
 
 
56
  ...timizer_state.safetensors: 15%|█▌ | 334MB / 2.20GB 
 
 
 
57
  ...d_model/model.safetensors: 3%|▎ | 295MB / 9.35GB 
 
 
 
58
  ...timizer_state.safetensors: 16%|█▌ | 348MB / 2.20GB 
 
 
 
59
  ...d_model/model.safetensors: 3%|▎ | 306MB / 9.35GB 
 
 
 
60
  ...timizer_state.safetensors: 18%|█▊ | 397MB / 2.20GB 
 
 
 
61
  ...d_model/model.safetensors: 4%|▎ | 344MB / 9.35GB 
 
 
 
62
  ...timizer_state.safetensors: 18%|█▊ | 402MB / 2.20GB 
 
 
 
63
  ...d_model/model.safetensors: 4%|▍ | 392MB / 9.35GB 
 
 
 
64
  ...timizer_state.safetensors: 19%|█▉ | 425MB / 2.20GB 
 
 
 
65
  ...d_model/model.safetensors: 4%|▍ | 402MB / 9.35GB 
 
 
 
66
  ...timizer_state.safetensors: 23%|██▎ | 498MB / 2.20GB 
 
 
 
67
  ...d_model/model.safetensors: 4%|▍ | 402MB / 9.35GB 
 
 
 
68
  ...timizer_state.safetensors: 26%|██▌ | 560MB / 2.20GB 
 
 
 
69
  ...d_model/model.safetensors: 5%|▌ | 495MB / 9.35GB 
 
 
 
70
  ...timizer_state.safetensors: 29%|██▉ | 637MB / 2.20GB 
 
 
 
71
  ...d_model/model.safetensors: 8%|▊ | 779MB / 9.35GB 
 
 
 
72
  ...timizer_state.safetensors: 32%|███▏ | 706MB / 2.20GB 
 
 
 
73
  ...d_model/model.safetensors: 11%|█ | 1.02GB / 9.35GB 
 
 
 
74
  ...timizer_state.safetensors: 36%|███▌ | 793MB / 2.20GB 
 
 
 
75
  ...d_model/model.safetensors: 13%|█▎ | 1.24GB / 9.35GB 
 
 
 
76
  ...timizer_state.safetensors: 38%|███▊ | 824MB / 2.20GB 
 
 
 
77
  ...d_model/model.safetensors: 15%|█▌ | 1.43GB / 9.35GB 
 
 
 
78
  ...timizer_state.safetensors: 40%|████ | 884MB / 2.20GB 
 
 
 
79
  ...d_model/model.safetensors: 17%|█▋ | 1.62GB / 9.35GB 
 
 
 
80
  ...timizer_state.safetensors: 43%|████▎ | 940MB / 2.20GB 
 
 
 
81
  ...d_model/model.safetensors: 19%|█▉ | 1.82GB / 9.35GB 
 
 
 
82
  ...timizer_state.safetensors: 46%|████▋ | 1.02GB / 2.20GB 
 
 
 
83
  ...d_model/model.safetensors: 21%|██▏ | 2.01GB / 9.35GB 
 
 
 
84
  ...timizer_state.safetensors: 50%|████▉ | 1.09GB / 2.20GB 
 
 
 
85
  ...d_model/model.safetensors: 22%|██▏ | 2.08GB / 9.35GB 
 
 
 
86
  ...timizer_state.safetensors: 53%|█████▎ | 1.17GB / 2.20GB 
 
 
 
87
  ...d_model/model.safetensors: 22%|██▏ | 2.08GB / 9.35GB 
 
 
 
88
  ...timizer_state.safetensors: 56%|█████▌ | 1.23GB / 2.20GB 
 
 
 
89
  ...d_model/model.safetensors: 24%|██▎ | 2.22GB / 9.35GB 
 
 
 
90
  ...timizer_state.safetensors: 59%|█████▉ | 1.30GB / 2.20GB 
 
 
 
91
  ...d_model/model.safetensors: 24%|██▎ | 2.22GB / 9.35GB 
 
 
 
92
  ...timizer_state.safetensors: 63%|██████▎ | 1.38GB / 2.20GB 
 
 
 
93
  ...d_model/model.safetensors: 24%|██▍ | 2.27GB / 9.35GB 
 
 
 
94
  ...timizer_state.safetensors: 66%|██████▌ | 1.44GB / 2.20GB 
 
 
 
95
  ...d_model/model.safetensors: 27%|██▋ | 2.50GB / 9.35GB 
 
 
 
96
  ...timizer_state.safetensors: 68%|██████▊ | 1.50GB / 2.20GB 
 
 
 
97
  ...d_model/model.safetensors: 28%|██▊ | 2.61GB / 9.35GB 
 
 
 
98
  ...timizer_state.safetensors: 71%|███████ | 1.55GB / 2.20GB 
 
 
 
99
  ...d_model/model.safetensors: 28%|██▊ | 2.62GB / 9.35GB 
 
 
 
100
  ...timizer_state.safetensors: 74%|███████▍ | 1.63GB / 2.20GB 
 
 
 
101
  ...d_model/model.safetensors: 28%|██▊ | 2.64GB / 9.35GB 
 
 
 
102
  ...timizer_state.safetensors: 78%|███████▊ | 1.71GB / 2.20GB 
 
 
 
103
  ...d_model/model.safetensors: 29%|██▉ | 2.71GB / 9.35GB 
 
 
 
104
  ...timizer_state.safetensors: 81%|████████ | 1.77GB / 2.20GB 
 
 
 
105
  ...d_model/model.safetensors: 30%|██▉ | 2.78GB / 9.35GB 
 
 
 
106
  ...timizer_state.safetensors: 84%|████████▍ | 1.84GB / 2.20GB 
 
 
 
107
  ...d_model/model.safetensors: 31%|███ | 2.86GB / 9.35GB 
 
 
 
108
  ...timizer_state.safetensors: 87%|████████▋ | 1.90GB / 2.20GB 
 
 
 
109
  ...d_model/model.safetensors: 31%|███ | 2.92GB / 9.35GB 
 
 
 
110
  ...timizer_state.safetensors: 90%|████████▉ | 1.97GB / 2.20GB 
 
 
 
111
  ...d_model/model.safetensors: 32%|███▏ | 2.98GB / 9.35GB 
 
 
112
  ...timizer_state.safetensors: 91%|█████████ | 1.99GB / 2.20GB 
 
 
 
113
  ...d_model/model.safetensors: 32%|███▏ | 3.02GB / 9.35GB 
 
 
114
  ...timizer_state.safetensors: 93%|█████████▎| 2.04GB / 2.20GB 
 
 
 
115
  ...d_model/model.safetensors: 33%|███▎ | 3.08GB / 9.35GB 
 
 
 
116
  ...timizer_state.safetensors: 96%|█████████▌| 2.10GB / 2.20GB 
 
 
 
117
  ...d_model/model.safetensors: 33%|███▎ | 3.09GB / 9.35GB 
 
 
 
118
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
 
 
 
119
  ...d_model/model.safetensors: 35%|███▍ | 3.23GB / 9.35GB 
 
 
 
120
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
 
 
 
121
  ...d_model/model.safetensors: 38%|███▊ | 3.53GB / 9.35GB 
 
 
 
122
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
 
 
 
123
  ...d_model/model.safetensors: 40%|████ | 3.78GB / 9.35GB 
 
 
 
124
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
 
 
 
125
  ...d_model/model.safetensors: 43%|████▎ | 4.03GB / 9.35GB 
 
 
 
126
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
 
 
 
127
  ...d_model/model.safetensors: 45%|████▌ | 4.23GB / 9.35GB 
 
 
 
128
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
 
 
 
129
  ...d_model/model.safetensors: 48%|████▊ | 4.46GB / 9.35GB 
 
 
 
130
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
 
 
 
131
  ...d_model/model.safetensors: 50%|█████ | 4.68GB / 9.35GB 
 
 
 
132
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
 
 
 
133
  ...d_model/model.safetensors: 52%|█████▏ | 4.86GB / 9.35GB 
 
 
 
134
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
 
 
 
135
  ...d_model/model.safetensors: 54%|█████▍ | 5.03GB / 9.35GB 
 
 
136
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
 
 
 
137
  ...d_model/model.safetensors: 56%|█████▌ | 5.22GB / 9.35GB 
 
 
138
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
 
 
 
139
  ...d_model/model.safetensors: 58%|█████▊ | 5.38GB / 9.35GB 
 
 
140
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
 
 
 
141
  ...d_model/model.safetensors: 58%|█████▊ | 5.38GB / 9.35GB 
 
 
142
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
 
 
 
143
  ...d_model/model.safetensors: 58%|█████▊ | 5.38GB / 9.35GB 
 
 
144
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
 
 
 
145
  ...d_model/model.safetensors: 58%|█████▊ | 5.39GB / 9.35GB 
 
 
 
146
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
 
 
 
147
  ...d_model/model.safetensors: 58%|█████▊ | 5.41GB / 9.35GB 
 
 
 
148
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
 
 
 
149
  ...d_model/model.safetensors: 60%|██████ | 5.64GB / 9.35GB 
 
 
 
150
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
 
 
 
151
  ...d_model/model.safetensors: 63%|██████▎ | 5.87GB / 9.35GB 
 
 
 
152
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
 
 
 
153
  ...d_model/model.safetensors: 65%|██████▌ | 6.10GB / 9.35GB 
 
 
 
154
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
 
 
 
155
  ...d_model/model.safetensors: 67%|██████▋ | 6.31GB / 9.35GB 
 
 
 
156
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
 
 
 
157
  ...d_model/model.safetensors: 70%|██████▉ | 6.51GB / 9.35GB 
 
 
158
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
 
 
 
159
  ...d_model/model.safetensors: 72%|███████▏ | 6.70GB / 9.35GB 
 
 
160
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
 
 
 
161
  ...d_model/model.safetensors: 74%|███████▎ | 6.89GB / 9.35GB 
 
 
162
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
 
 
 
163
  ...d_model/model.safetensors: 75%|███████▍ | 6.98GB / 9.35GB 
 
 
 
164
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
 
 
 
165
  ...d_model/model.safetensors: 75%|███████▍ | 6.99GB / 9.35GB 
 
 
 
166
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
 
 
 
167
  ...d_model/model.safetensors: 75%|███████▍ | 7.01GB / 9.35GB 
 
 
 
168
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
 
 
 
169
  ...d_model/model.safetensors: 75%|███████▌ | 7.06GB / 9.35GB 
 
 
 
170
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
 
 
 
171
  ...d_model/model.safetensors: 77%|███████▋ | 7.24GB / 9.35GB 
 
 
 
172
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
 
 
 
173
  ...d_model/model.safetensors: 80%|███████▉ | 7.45GB / 9.35GB 
 
 
 
174
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
 
 
 
175
  ...d_model/model.safetensors: 82%|████████▏ | 7.65GB / 9.35GB 
 
 
 
176
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
 
 
 
177
  ...d_model/model.safetensors: 84%|████████▍ | 7.85GB / 9.35GB 
 
 
178
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
 
 
 
179
  ...d_model/model.safetensors: 86%|████████▌ | 8.05GB / 9.35GB 
 
 
 
180
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
 
 
 
181
  ...d_model/model.safetensors: 88%|████████▊ | 8.25GB / 9.35GB 
 
 
182
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
 
 
 
183
  ...d_model/model.safetensors: 90%|█████████ | 8.45GB / 9.35GB 
 
 
184
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
 
 
 
185
  ...d_model/model.safetensors: 93%|█████████▎| 8.67GB / 9.35GB 
 
 
 
186
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
 
 
 
187
  ...d_model/model.safetensors: 95%|█████████▍| 8.86GB / 9.35GB 
 
 
188
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
 
 
 
189
  ...d_model/model.safetensors: 97%|█████████▋| 9.07GB / 9.35GB 
 
 
190
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
 
 
 
191
  ...d_model/model.safetensors: 99%|█████████▉| 9.27GB / 9.35GB 
 
 
192
  ...timizer_state.safetensors: 98%|█████████▊| 2.15GB / 2.20GB 
 
 
 
193
  ...d_model/model.safetensors: 100%|█████████▉| 9.34GB / 9.35GB 
 
 
 
 
194
  ...zer_processor.safetensors: 14%|█▍ | 109B / 768B 
 
 
 
 
 
195
  ...zer_processor.safetensors: 14%|█▍ | 109B / 768B 
 
 
 
 
 
 
196
  ...ate/rng_state.safetensors: 14%|█▍ | 2.23kB / 15.7kB 
 
 
197
  ...timizer_state.safetensors: 98%|█████████▊| 2.16GB / 2.20GB 
 
 
 
198
  ...d_model/model.safetensors: 100%|█████████▉| 9.34GB / 9.35GB 
 
 
 
 
199
  ...zer_processor.safetensors: 14%|█▍ | 109B / 768B 
 
 
 
 
 
200
  ...zer_processor.safetensors: 14%|█▍ | 109B / 768B 
 
 
 
 
 
 
201
  ...ate/rng_state.safetensors: 14%|█▍ | 2.23kB / 15.7kB 
 
 
 
202
  ...timizer_state.safetensors: 99%|█████████▉| 2.17GB / 2.20GB 
 
 
 
203
  ...d_model/model.safetensors: 100%|█████████▉| 9.34GB / 9.35GB 
 
 
 
 
204
  ...zer_processor.safetensors: 43%|████▎ | 327B / 768B 
 
 
 
 
 
205
  ...zer_processor.safetensors: 43%|████▎ | 327B / 768B 
 
 
 
 
 
 
206
  ...ate/rng_state.safetensors: 43%|████▎ | 6.69kB / 15.7kB 
 
 
 
207
  ...timizer_state.safetensors: 99%|█████████▉| 2.18GB / 2.20GB 
 
 
 
208
  ...d_model/model.safetensors: 100%|█████████▉| 9.35GB / 9.35GB 
 
 
 
 
209
  ...zer_processor.safetensors: 70%|██████▉ | 537B / 768B 
 
 
 
 
 
210
  ...zer_processor.safetensors: 70%|██████▉ | 537B / 768B 
 
 
 
 
 
 
211
  ...ate/rng_state.safetensors: 70%|██████▉ | 11.0kB / 15.7kB 
 
 
 
212
  ...timizer_state.safetensors: 100%|█████████▉| 2.19GB / 2.20GB 
 
 
 
213
  ...d_model/model.safetensors: 100%|█████████▉| 9.35GB / 9.35GB 
 
 
 
 
214
  ...zer_processor.safetensors: 98%|█████████▊| 755B / 768B 
 
 
 
 
 
215
  ...zer_processor.safetensors: 98%|█████████▊| 755B / 768B 
 
 
 
 
 
 
216
  ...ate/rng_state.safetensors: 98%|█████████▊| 15.4kB / 15.7kB 
 
 
 
217
  ...timizer_state.safetensors: 100%|█████████▉| 2.19GB / 2.20GB 
 
 
 
218
  ...d_model/model.safetensors: 100%|█████████▉| 9.35GB / 9.35GB 
 
 
 
 
219
  ...zer_processor.safetensors: 99%|█████████▉| 763B / 768B 
 
 
 
 
 
220
  ...zer_processor.safetensors: 99%|█████████▉| 763B / 768B 
 
 
 
 
 
 
221
  ...ate/rng_state.safetensors: 99%|█████████▉| 15.6kB / 15.7kB 
 
 
 
222
  ...timizer_state.safetensors: 100%|█████████▉| 2.19GB / 2.20GB 
 
 
 
223
  ...d_model/model.safetensors: 100%|█████████▉| 9.35GB / 9.35GB 
 
 
 
 
224
  ...zer_processor.safetensors: 99%|█████████▉| 763B / 768B 
 
 
 
 
 
225
  ...zer_processor.safetensors: 99%|█████████▉| 763B / 768B 
 
 
 
 
 
 
226
  ...ate/rng_state.safetensors: 99%|█████████▉| 15.6kB / 15.7kB 
 
 
227
  ...timizer_state.safetensors: 100%|█████████▉| 2.19GB / 2.20GB 
 
 
 
228
  ...d_model/model.safetensors: 100%|█████████▉| 9.35GB / 9.35GB 
 
 
 
 
229
  ...zer_processor.safetensors: 99%|█████████▉| 763B / 768B 
 
 
 
 
 
230
  ...zer_processor.safetensors: 99%|█████████▉| 763B / 768B 
 
 
 
 
 
 
231
  ...ate/rng_state.safetensors: 99%|█████████▉| 15.6kB / 15.7kB 
 
 
232
  ...timizer_state.safetensors: 100%|█████████▉| 2.19GB / 2.20GB 
 
 
 
233
  ...d_model/model.safetensors: 100%|█████████▉| 9.35GB / 9.35GB 
 
 
 
 
234
  ...zer_processor.safetensors: 99%|█████████▉| 763B / 768B 
 
 
 
 
 
235
  ...zer_processor.safetensors: 99%|█████████▉| 763B / 768B 
 
 
 
 
 
 
236
  ...ate/rng_state.safetensors: 99%|█████████▉| 15.6kB / 15.7kB 
 
 
237
  ...timizer_state.safetensors: 100%|██████████| 2.20GB / 2.20GB 
 
 
 
238
  ...d_model/model.safetensors: 100%|█████████▉| 9.35GB / 9.35GB 
 
 
 
 
239
  ...zer_processor.safetensors: 100%|██████████| 768B / 768B 
 
 
 
 
 
240
  ...zer_processor.safetensors: 100%|██████████| 768B / 768B 
 
 
 
 
 
 
241
  ...ate/rng_state.safetensors: 100%|██████████| 15.7kB / 15.7kB 
 
 
 
242
  ...timizer_state.safetensors: 100%|██████████| 2.20GB / 2.20GB 
 
 
 
243
  ...d_model/model.safetensors: 100%|█████████▉| 9.35GB / 9.35GB 
 
 
 
 
244
  ...zer_processor.safetensors: 100%|██████████| 768B / 768B 
 
 
 
 
 
245
  ...zer_processor.safetensors: 100%|██████████| 768B / 768B 
 
 
 
 
 
 
246
  ...ate/rng_state.safetensors: 100%|██████████| 15.7kB / 15.7kB 
 
 
247
  ...timizer_state.safetensors: 100%|██████████| 2.20GB / 2.20GB 
 
 
 
248
  ...d_model/model.safetensors: 100%|█████████▉| 9.35GB / 9.35GB 
 
 
 
 
249
  ...zer_processor.safetensors: 100%|██████████| 768B / 768B 
 
 
 
 
 
250
  ...zer_processor.safetensors: 100%|██████████| 768B / 768B 
 
 
 
 
 
 
251
  ...ate/rng_state.safetensors: 100%|██████████| 15.7kB / 15.7kB 
 
 
252
  ...timizer_state.safetensors: 100%|██████████| 2.20GB / 2.20GB 
 
 
 
253
  ...d_model/model.safetensors: 100%|█████████▉| 9.35GB / 9.35GB 
 
 
 
 
254
  ...zer_processor.safetensors: 100%|██████████| 768B / 768B 
 
 
 
 
 
255
  ...zer_processor.safetensors: 100%|██████████| 768B / 768B 
 
 
 
 
 
 
256
  ...ate/rng_state.safetensors: 100%|██████████| 15.7kB / 15.7kB 
 
 
257
  ...timizer_state.safetensors: 100%|██████████| 2.20GB / 2.20GB 
 
 
 
258
  ...d_model/model.safetensors: 100%|█████████▉| 9.35GB / 9.35GB 
 
 
 
 
259
  ...zer_processor.safetensors: 100%|██████████| 768B / 768B 
 
 
 
 
 
260
  ...zer_processor.safetensors: 100%|██████████| 768B / 768B 
 
 
 
 
 
 
261
  ...ate/rng_state.safetensors: 100%|██████████| 15.7kB / 15.7kB 
 
 
262
  ...timizer_state.safetensors: 100%|██████████| 2.20GB / 2.20GB 
 
 
 
263
  ...d_model/model.safetensors: 100%|█████████▉| 9.35GB / 9.35GB 
 
 
 
 
264
  ...zer_processor.safetensors: 100%|██████████| 768B / 768B 
 
 
 
 
 
265
  ...zer_processor.safetensors: 100%|██████████| 768B / 768B 
 
 
 
 
 
 
266
  ...ate/rng_state.safetensors: 100%|██████████| 15.7kB / 15.7kB 
 
 
267
  ...timizer_state.safetensors: 100%|██████████| 2.20GB / 2.20GB 
 
 
 
268
  ...d_model/model.safetensors: 100%|█████████▉| 9.35GB / 9.35GB 
 
 
 
 
269
  ...zer_processor.safetensors: 100%|██████████| 768B / 768B 
 
 
 
 
 
270
  ...zer_processor.safetensors: 100%|██████████| 768B / 768B 
 
 
 
 
 
 
271
  ...ate/rng_state.safetensors: 100%|██████████| 15.7kB / 15.7kB 
 
 
272
  ...timizer_state.safetensors: 100%|██████████| 2.20GB / 2.20GB
 
273
  ...d_model/model.safetensors: 100%|█████████▉| 9.35GB / 9.35GB
 
274
  ...zer_processor.safetensors: 100%|██████████| 768B / 768B
 
275
  ...zer_processor.safetensors: 100%|██████████| 768B / 768B
 
276
  ...ate/rng_state.safetensors: 100%|██████████| 15.7kB / 15.7kB