diff --git "a/checkpoints/math_operations/primitive_atomic_full_sft_50k_lr1e4_t20260308/debug.log" "b/checkpoints/math_operations/primitive_atomic_full_sft_50k_lr1e4_t20260308/debug.log"
new file mode 100644--- /dev/null
+++ "b/checkpoints/math_operations/primitive_atomic_full_sft_50k_lr1e4_t20260308/debug.log"
@@ -0,0 +1,964 @@
+[2026-03-08 15:53:42,476] [DEBUG] [axolotl.utils.config.resolve_dtype:66] [PID:1659682] bf16 support detected, enabling for this configuration.
+[2026-03-08 15:53:42,506] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:1659682] baseline 0.000GB ()
+[2026-03-08 15:53:42,507] [INFO] [axolotl.cli.config.load_cfg:259] [PID:1659682] config:
+{
+  "activation_offloading": false,
+  "axolotl_config_path": "/home/jiaruil5/math_rl/mix_teachers/mix_teachers/train/math_operations_sft/configs/.tmp_jo6r_r7k.yaml",
+  "base_model": "/home/jiaruil5/math_rl/mix_teachers/r3lit_rl/models/Qwen/Qwen3-4B-Instruct-2507",
+  "base_model_config": "/home/jiaruil5/math_rl/mix_teachers/r3lit_rl/models/Qwen/Qwen3-4B-Instruct-2507",
+  "batch_size": 8,
+  "bf16": true,
+  "capabilities": {
+    "bf16": true,
+    "compute_capability": "sm_86",
+    "fp8": false,
+    "n_gpu": 1,
+    "n_node": 1
+  },
+  "chat_template": "chatml",
+  "context_parallel_size": 1,
+  "dataloader_num_workers": 1,
+  "dataloader_pin_memory": true,
+  "dataloader_prefetch_factor": 256,
+  "dataset_num_proc": 128,
+  "datasets": [
+    {
+      "message_property_mappings": {
+        "content": "content",
+        "role": "role"
+      },
+      "path": "/home/jiaruil5/math_rl/mix_teachers/r3lit_rl/mix_teachers/data/math_operations/primitive_atomic_balanced_sft_50k/balanced_train_alpaca.jsonl",
+      "trust_remote_code": false,
+      "type": "alpaca"
+    }
+  ],
+  "ddp": false,
+  "device": "cuda:0",
+  "device_map": "auto",
+  "dion_rank_fraction": 1.0,
+  "dion_rank_multiple_of": 1,
+  "eaft_alpha": 1.0,
+  "eaft_k": 20,
+  "env_capabilities": {
+    "torch_version": "2.8.0"
+  },
+  "eval_batch_size": 1,
+  "eval_causal_lm_metrics": [
+    "sacrebleu",
+    "comet",
+    "ter",
+    "chrf"
+  ],
+  "eval_max_new_tokens": 128,
+  "eval_sample_packing": true,
+  "eval_steps": 0.16666666666666666,
+  "eval_table_size": 0,
+  "evals_per_epoch": 2,
+  "experimental_skip_move_to_device": true,
+  "flash_attention": true,
+  "fp16": false,
+  "gradient_accumulation_steps": 8,
+  "gradient_checkpointing": true,
+  "gradient_checkpointing_kwargs": {
+    "use_reentrant": false
+  },
+  "include_tkps": true,
+  "learning_rate": 0.0001,
+  "lisa_layers_attribute": "model.layers",
+  "load_best_model_at_end": true,
+  "load_in_4bit": false,
+  "load_in_8bit": false,
+  "local_rank": 0,
+  "logging_steps": 10,
+  "lora_dropout": 0.0,
+  "loraplus_lr_embedding": 1e-06,
+  "lr_scheduler": "constant_with_warmup",
+  "mean_resizing_embeddings": false,
+  "micro_batch_size": 1,
+  "model_config_type": "qwen3",
+  "num_epochs": 3.0,
+  "optimizer": "adamw_torch_fused",
+  "otel_metrics_host": "localhost",
+  "otel_metrics_port": 8000,
+  "output_dir": "/home/jiaruil5/math_rl/mix_teachers/r3lit_rl/mix_teachers/checkpoints/math_operations/primitive_atomic_full_sft_50k_lr1e4_t20260308",
+  "pad_to_sequence_len": true,
+  "pretrain_multipack_attn": true,
+  "profiler_steps_start": 0,
+  "qlora_sharded_model_loading": false,
+  "ray_num_workers": 1,
+  "resources_per_worker": {
+    "GPU": 1
+  },
+  "sample_packing": true,
+  "sample_packing_bin_size": 200,
+  "sample_packing_group_size": 100000,
+  "save_only_model": false,
+  "save_safetensors": true,
+  "save_steps": 0.3333333333333333,
+  "save_total_limit": 3,
+  "saves_per_epoch": 1,
+  "sequence_len": 2048,
+  "shuffle_before_merging_datasets": false,
+  "shuffle_merged_datasets": true,
+  "skip_prepare_dataset": false,
+  "streaming_multipack_buffer_size": 10000,
+  "strict": false,
+  "tensor_parallel_size": 1,
+  "test_datasets": [
+    {
+      "message_property_mappings": {
+        "content": "content",
+        "role": "role"
+      },
+      "path": "/home/jiaruil5/math_rl/mix_teachers/r3lit_rl/mix_teachers/data/math_operations/primitive_atomic_balanced_sft_50k/balanced_val_alpaca.jsonl",
+      "trust_remote_code": false,
+      "type": "alpaca"
+    }
+  ],
+  "tf32": true,
+  "tiled_mlp_use_original_mlp": true,
+  "tokenizer_config": "/home/jiaruil5/math_rl/mix_teachers/r3lit_rl/models/Qwen/Qwen3-4B-Instruct-2507",
+  "tokenizer_save_jinja_files": true,
+  "torch_dtype": "torch.bfloat16",
+  "train_on_inputs": false,
+  "trl": {
+    "log_completions": false,
+    "mask_truncated_completions": false,
+    "ref_model_mixup_alpha": 0.9,
+    "ref_model_sync_steps": 64,
+    "scale_rewards": true,
+    "sync_ref_model": false,
+    "use_vllm": false,
+    "vllm_server_host": "0.0.0.0",
+    "vllm_server_port": 8000
+  },
+  "use_otel_metrics": false,
+  "use_ray": false,
+  "use_wandb": true,
+  "val_set_size": 0.0,
+  "vllm": {
+    "device": "auto",
+    "dtype": "auto",
+    "gpu_memory_utilization": 0.9,
+    "host": "0.0.0.0",
+    "port": 8000
+  },
+  "wandb_log_model": "false",
+  "wandb_name": "primitive-atomic-full-sft-50k-lr1e4-t20260308",
+  "wandb_project": "math_operations_sft",
+  "warmup_ratio": 0.1,
+  "weight_decay": 0.01,
+  "world_size": 1
+}
+[2026-03-08 15:53:43,978] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:285] [PID:1659682] EOS: 151645 / <|im_end|>
+[2026-03-08 15:53:43,979] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:286] [PID:1659682] BOS: None / None
+[2026-03-08 15:53:43,979] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:287] [PID:1659682] PAD: 151643 / <|endoftext|>
+[2026-03-08 15:53:43,979] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:288] [PID:1659682] UNK: None / None
+[2026-03-08 15:53:43,981] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:481] [PID:1659682] Unable to find prepared dataset in last_run_prepared/c5b2dcf69e91df42a1bb115dcc61090d
+[2026-03-08 15:53:43,981] [INFO] [axolotl.utils.data.sft._load_raw_datasets:320] [PID:1659682] Loading raw datasets...
+[2026-03-08 15:53:43,981] [WARNING] [axolotl.utils.data.sft._load_raw_datasets:322] [PID:1659682] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`.
+[2026-03-08 15:53:44,834] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:1659682] Loading dataset: /home/jiaruil5/math_rl/mix_teachers/r3lit_rl/mix_teachers/data/math_operations/primitive_atomic_balanced_sft_50k/balanced_train_alpaca.jsonl with base_type: alpaca and prompt_style: None
+[2026-03-08 15:53:48,061] [INFO] [axolotl.utils.data.utils.handle_long_seq_in_dataset:224] [PID:1659682] min_input_len: 257
+[2026-03-08 15:53:48,062] [INFO] [axolotl.utils.data.utils.handle_long_seq_in_dataset:226] [PID:1659682] max_input_len: 371
+
Dropping Long Sequences (>2048) (num_proc=128):   0%|          | 0/50000 [00:00<?, ? examples/s]
Dropping Long Sequences (>2048) (num_proc=128):   1%|          | 391/50000 [00:01<03:08, 262.70 examples/s]
Dropping Long Sequences (>2048) (num_proc=128):   7%|▋         | 3519/50000 [00:01<00:15, 2950.35 examples/s]
Dropping Long Sequences (>2048) (num_proc=128):  13%|█▎        | 6647/50000 [00:01<00:07, 5827.45 examples/s]
Dropping Long Sequences (>2048) (num_proc=128):  21%|██        | 10557/50000 [00:01<00:03, 9923.09 examples/s]
Dropping Long Sequences (>2048) (num_proc=128):  31%|███▏      | 15640/50000 [00:01<00:02, 15964.67 examples/s]
Dropping Long Sequences (>2048) (num_proc=128):  38%|███▊      | 19159/50000 [00:02<00:01, 18319.56 examples/s]
Dropping Long Sequences (>2048) (num_proc=128):  46%|████▌     | 23068/50000 [00:02<00:01, 21593.10 examples/s]
Dropping Long Sequences (>2048) (num_proc=128):  52%|█████▏    | 26196/50000 [00:02<00:01, 21007.95 examples/s]
Dropping Long Sequences (>2048) (num_proc=128):  77%|███████▋  | 38689/50000 [00:02<00:00, 42705.24 examples/s]
Dropping Long Sequences (>2048) (num_proc=128): 100%|██████████| 50000/50000 [00:03<00:00, 16564.97 examples/s]
+
Drop Samples with Zero Trainable Tokens (num_proc=128):   0%|          | 0/50000 [00:00<?, ? examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=128):   1%|          | 391/50000 [00:01<03:43, 222.19 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=128):   6%|▋         | 3128/50000 [00:01<00:20, 2249.60 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=128):  11%|█         | 5474/50000 [00:01<00:10, 4280.75 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=128):  24%|██▍       | 12121/50000 [00:02<00:03, 11605.77 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=128):  33%|███▎      | 16422/50000 [00:02<00:02, 15795.06 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=128):  49%|████▉     | 24633/50000 [00:02<00:00, 26180.38 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=128):  59%|█████▉    | 29716/50000 [00:02<00:00, 28588.25 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=128):  69%|██████▉   | 34400/50000 [00:02<00:00, 31008.81 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=128):  77%|███████▋  | 38690/50000 [00:02<00:00, 29625.65 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=128):  85%|████████▌ | 42590/50000 [00:02<00:00, 26782.58 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=128): 100%|██████████| 50000/50000 [00:03<00:00, 14360.63 examples/s]
+
Add position_id column (Sample Packing) (num_proc=128):   0%|          | 0/50000 [00:00<?, ? examples/s]
Add position_id column (Sample Packing) (num_proc=128):   1%|          | 391/50000 [00:01<03:50, 215.15 examples/s]
Add position_id column (Sample Packing) (num_proc=128):   3%|▎         | 1564/50000 [00:01<00:45, 1054.10 examples/s]
Add position_id column (Sample Packing) (num_proc=128):   5%|▌         | 2737/50000 [00:02<00:24, 1933.29 examples/s]
Add position_id column (Sample Packing) (num_proc=128):   9%|▉         | 4692/50000 [00:02<00:11, 3871.69 examples/s]
Add position_id column (Sample Packing) (num_proc=128):  12%|█▏        | 5865/50000 [00:02<00:09, 4724.63 examples/s]
Add position_id column (Sample Packing) (num_proc=128):  14%|█▍        | 7038/50000 [00:02<00:07, 5525.46 examples/s]
Add position_id column (Sample Packing) (num_proc=128):  18%|█▊        | 8993/50000 [00:02<00:05, 7171.19 examples/s]
Add position_id column (Sample Packing) (num_proc=128):  21%|██        | 10556/50000 [00:02<00:04, 8488.87 examples/s]
Add position_id column (Sample Packing) (num_proc=128):  33%|███▎      | 16420/50000 [00:02<00:01, 18467.37 examples/s]
Add position_id column (Sample Packing) (num_proc=128):  63%|██████▎   | 31276/50000 [00:02<00:00, 47473.25 examples/s]
Add position_id column (Sample Packing) (num_proc=128):  90%|████████▉ | 44928/50000 [00:03<00:00, 68819.84 examples/s]
Add position_id column (Sample Packing) (num_proc=128): 100%|██████████| 50000/50000 [00:03<00:00, 13959.49 examples/s]
+
Saving the dataset (0/128 shards):   0%|          | 0/50000 [00:00<?, ? examples/s]
Saving the dataset (0/128 shards):   1%|          | 391/50000 [00:01<03:38, 226.76 examples/s]
Saving the dataset (1/128 shards):   1%|          | 391/50000 [00:01<03:38, 226.76 examples/s]
Saving the dataset (2/128 shards):   2%|▏         | 1173/50000 [00:01<03:35, 226.76 examples/s]
Saving the dataset (3/128 shards):   2%|▏         | 1173/50000 [00:01<03:35, 226.76 examples/s]
Saving the dataset (4/128 shards):   5%|▍         | 2346/50000 [00:01<03:30, 226.76 examples/s]
Saving the dataset (5/128 shards):   5%|▍         | 2346/50000 [00:01<03:30, 226.76 examples/s]
Saving the dataset (6/128 shards):   5%|▍         | 2346/50000 [00:01<03:30, 226.76 examples/s]
Saving the dataset (7/128 shards):   7%|▋         | 3519/50000 [00:01<03:24, 226.76 examples/s]
Saving the dataset (8/128 shards):   7%|▋         | 3519/50000 [00:01<03:24, 226.76 examples/s]
Saving the dataset (9/128 shards):   7%|▋         | 3519/50000 [00:01<03:24, 226.76 examples/s]
Saving the dataset (10/128 shards):   8%|▊         | 3910/50000 [00:01<03:23, 226.76 examples/s]
Saving the dataset (11/128 shards):  11%|█         | 5474/50000 [00:01<03:16, 226.76 examples/s]
Saving the dataset (12/128 shards):  11%|█         | 5474/50000 [00:01<03:16, 226.76 examples/s]
Saving the dataset (13/128 shards):  11%|█         | 5474/50000 [00:01<03:16, 226.76 examples/s]
Saving the dataset (14/128 shards):  11%|█         | 5474/50000 [00:01<03:16, 226.76 examples/s]
Saving the dataset (15/128 shards):  12%|█▏        | 5865/50000 [00:01<03:14, 226.76 examples/s]
Saving the dataset (16/128 shards):  13%|█▎        | 6647/50000 [00:01<03:11, 226.76 examples/s]
Saving the dataset (17/128 shards):  14%|█▍        | 7038/50000 [00:01<03:09, 226.76 examples/s]
Saving the dataset (18/128 shards):  16%|█▌        | 7820/50000 [00:01<03:06, 226.76 examples/s]
Saving the dataset (19/128 shards):  16%|█▌        | 7820/50000 [00:01<03:06, 226.76 examples/s]
Saving the dataset (20/128 shards):  16%|█▌        | 7820/50000 [00:01<03:06, 226.76 examples/s]
Saving the dataset (21/128 shards):  16%|█▋        | 8211/50000 [00:01<03:04, 226.76 examples/s]
Saving the dataset (22/128 shards):  17%|█▋        | 8602/50000 [00:01<03:02, 226.76 examples/s]
Saving the dataset (23/128 shards):  18%|█▊        | 8993/50000 [00:01<03:00, 226.76 examples/s]
Saving the dataset (24/128 shards):  20%|█▉        | 9775/50000 [00:01<02:57, 226.76 examples/s]
Saving the dataset (25/128 shards):  20%|█▉        | 9775/50000 [00:01<02:57, 226.76 examples/s]
Saving the dataset (26/128 shards):  22%|██▏       | 10948/50000 [00:01<02:52, 226.76 examples/s]
Saving the dataset (27/128 shards):  22%|██▏       | 10948/50000 [00:01<02:52, 226.76 examples/s]
Saving the dataset (28/128 shards):  22%|██▏       | 10948/50000 [00:01<02:52, 226.76 examples/s]
Saving the dataset (29/128 shards):  23%|██▎       | 11339/50000 [00:01<02:50, 226.76 examples/s]
Saving the dataset (30/128 shards):  23%|██▎       | 11730/50000 [00:01<02:48, 226.76 examples/s]
Saving the dataset (31/128 shards):  26%|██▌       | 12903/50000 [00:01<02:43, 226.76 examples/s]
Saving the dataset (32/128 shards):  26%|██▌       | 12903/50000 [00:01<02:43, 226.76 examples/s]
Saving the dataset (33/128 shards):  26%|██▌       | 12903/50000 [00:01<02:43, 226.76 examples/s]
Saving the dataset (34/128 shards):  28%|██▊       | 14076/50000 [00:01<02:38, 226.76 examples/s]
Saving the dataset (35/128 shards):  28%|██▊       | 14076/50000 [00:01<02:38, 226.76 examples/s]
Saving the dataset (36/128 shards):  28%|██▊       | 14076/50000 [00:01<02:38, 226.76 examples/s]
Saving the dataset (37/128 shards):  30%|██▉       | 14858/50000 [00:01<02:34, 226.76 examples/s]
Saving the dataset (38/128 shards):  30%|███       | 15249/50000 [00:01<02:33, 226.76 examples/s]
Saving the dataset (39/128 shards):  30%|███       | 15249/50000 [00:01<02:33, 226.76 examples/s]
Saving the dataset (40/128 shards):  31%|███▏      | 15640/50000 [00:01<02:31, 226.76 examples/s]
Saving the dataset (41/128 shards):  33%|███▎      | 16422/50000 [00:01<02:28, 226.76 examples/s]
Saving the dataset (42/128 shards):  34%|███▎      | 16813/50000 [00:01<02:26, 226.76 examples/s]
Saving the dataset (43/128 shards):  34%|███▎      | 16813/50000 [00:01<02:26, 226.76 examples/s]
Saving the dataset (44/128 shards):  35%|███▌      | 17595/50000 [00:01<02:22, 226.76 examples/s]
Saving the dataset (45/128 shards):  35%|███▌      | 17595/50000 [00:01<02:22, 226.76 examples/s]
Saving the dataset (46/128 shards):  36%|███▌      | 17986/50000 [00:01<02:21, 226.76 examples/s]
Saving the dataset (47/128 shards):  37%|███▋      | 18377/50000 [00:01<02:19, 226.76 examples/s]
Saving the dataset (48/128 shards):  38%|███▊      | 18768/50000 [00:01<02:17, 226.76 examples/s]
Saving the dataset (49/128 shards):  39%|███▉      | 19550/50000 [00:01<02:14, 226.76 examples/s]
Saving the dataset (50/128 shards):  39%|███▉      | 19550/50000 [00:01<02:14, 226.76 examples/s]
Saving the dataset (51/128 shards):  41%|████      | 20332/50000 [00:01<02:10, 226.76 examples/s]
Saving the dataset (52/128 shards):  41%|████      | 20332/50000 [00:01<02:10, 226.76 examples/s]
Saving the dataset (53/128 shards):  42%|████▏     | 21114/50000 [00:01<02:07, 226.76 examples/s]
Saving the dataset (54/128 shards):  43%|████▎     | 21505/50000 [00:01<02:05, 226.76 examples/s]
Saving the dataset (55/128 shards):  43%|████▎     | 21505/50000 [00:01<02:05, 226.76 examples/s]
Saving the dataset (56/128 shards):  44%|████▍     | 21896/50000 [00:01<02:03, 226.76 examples/s]
Saving the dataset (57/128 shards):  45%|████▌     | 22678/50000 [00:01<02:00, 226.76 examples/s]
Saving the dataset (58/128 shards):  45%|████▌     | 22678/50000 [00:01<02:00, 226.76 examples/s]
Saving the dataset (59/128 shards):  46%|████▌     | 23069/50000 [00:01<01:58, 226.76 examples/s]
Saving the dataset (60/128 shards):  47%|████▋     | 23460/50000 [00:01<01:57, 226.76 examples/s]
Saving the dataset (61/128 shards):  48%|████▊     | 24242/50000 [00:01<01:53, 226.76 examples/s]
Saving the dataset (62/128 shards):  48%|████▊     | 24242/50000 [00:01<01:53, 226.76 examples/s]
Saving the dataset (63/128 shards):  50%|█████     | 25024/50000 [00:01<01:50, 226.76 examples/s]
Saving the dataset (64/128 shards):  50%|█████     | 25024/50000 [00:01<01:50, 226.76 examples/s]
Saving the dataset (65/128 shards):  51%|█████     | 25415/50000 [00:01<01:48, 226.76 examples/s]
Saving the dataset (66/128 shards):  52%|█████▏    | 25806/50000 [00:01<01:46, 226.76 examples/s]
Saving the dataset (67/128 shards):  53%|█████▎    | 26588/50000 [00:01<01:43, 226.76 examples/s]
Saving the dataset (68/128 shards):  53%|█████▎    | 26588/50000 [00:01<01:43, 226.76 examples/s]
Saving the dataset (69/128 shards):  55%|█████▍    | 27370/50000 [00:01<01:39, 226.76 examples/s]
Saving the dataset (70/128 shards):  56%|█████▌    | 27761/50000 [00:01<01:38, 226.76 examples/s]
Saving the dataset (71/128 shards):  56%|█████▌    | 27761/50000 [00:01<01:38, 226.76 examples/s]
Saving the dataset (72/128 shards):  56%|█████▋    | 28152/50000 [00:01<01:36, 226.76 examples/s]
Saving the dataset (73/128 shards):  57%|█████▋    | 28543/50000 [00:01<01:34, 226.76 examples/s]
Saving the dataset (74/128 shards):  58%|█████▊    | 28934/50000 [00:01<01:32, 226.76 examples/s]
Saving the dataset (75/128 shards):  59%|█████▊    | 29325/50000 [00:01<01:31, 226.76 examples/s]
Saving the dataset (76/128 shards):  60%|██████    | 30107/50000 [00:01<01:27, 226.76 examples/s]
Saving the dataset (77/128 shards):  60%|██████    | 30107/50000 [00:01<01:27, 226.76 examples/s]
Saving the dataset (78/128 shards):  62%|██████▏   | 30889/50000 [00:01<01:24, 226.76 examples/s]
Saving the dataset (79/128 shards):  62%|██████▏   | 30889/50000 [00:01<01:24, 226.76 examples/s]
Saving the dataset (80/128 shards):  64%|██████▍   | 32060/50000 [00:01<01:19, 226.76 examples/s]
Saving the dataset (81/128 shards):  64%|██████▍   | 32060/50000 [00:01<01:19, 226.76 examples/s]
Saving the dataset (82/128 shards):  65%|██████▍   | 32450/50000 [00:01<01:17, 226.76 examples/s]
Saving the dataset (83/128 shards):  65%|██████▍   | 32450/50000 [00:01<01:17, 226.76 examples/s]
Saving the dataset (84/128 shards):  67%|██████▋   | 33620/50000 [00:01<01:12, 226.76 examples/s]
Saving the dataset (85/128 shards):  67%|██████▋   | 33620/50000 [00:01<01:12, 226.76 examples/s]
Saving the dataset (86/128 shards):  67%|██████▋   | 33620/50000 [00:01<01:12, 226.76 examples/s]
Saving the dataset (87/128 shards):  69%|██████▉   | 34400/50000 [00:01<01:08, 226.76 examples/s]
Saving the dataset (88/128 shards):  69%|██████▉   | 34400/50000 [00:01<01:08, 226.76 examples/s]
Saving the dataset (89/128 shards):  70%|███████   | 35180/50000 [00:01<01:05, 226.76 examples/s]
Saving the dataset (90/128 shards):  71%|███████   | 35570/50000 [00:01<01:03, 226.76 examples/s]
Saving the dataset (91/128 shards):  71%|███████   | 35570/50000 [00:01<01:03, 226.76 examples/s]
Saving the dataset (92/128 shards):  72%|███████▏  | 35960/50000 [00:01<01:01, 226.76 examples/s]
Saving the dataset (93/128 shards):  73%|███████▎  | 36350/50000 [00:01<01:00, 226.76 examples/s]
Saving the dataset (94/128 shards):  74%|███████▍  | 37130/50000 [00:01<00:56, 226.76 examples/s]
Saving the dataset (95/128 shards):  74%|███████▍  | 37130/50000 [00:01<00:56, 226.76 examples/s]
Saving the dataset (96/128 shards):  75%|███████▌  | 37520/50000 [00:01<00:55, 226.76 examples/s]
Saving the dataset (97/128 shards):  77%|███████▋  | 38300/50000 [00:01<00:51, 226.76 examples/s]
Saving the dataset (98/128 shards):  77%|███████▋  | 38690/50000 [00:01<00:49, 226.76 examples/s]
Saving the dataset (99/128 shards):  77%|███████▋  | 38690/50000 [00:01<00:49, 226.76 examples/s]
Saving the dataset (100/128 shards):  78%|███████▊  | 39080/50000 [00:01<00:48, 226.76 examples/s]
Saving the dataset (101/128 shards):  79%|███████▉  | 39470/50000 [00:01<00:46, 226.76 examples/s]
Saving the dataset (102/128 shards):  80%|████████  | 40250/50000 [00:01<00:42, 226.76 examples/s]
Saving the dataset (103/128 shards):  80%|████████  | 40250/50000 [00:01<00:42, 226.76 examples/s]
Saving the dataset (104/128 shards):  82%|████████▏ | 41030/50000 [00:01<00:39, 226.76 examples/s]
Saving the dataset (105/128 shards):  82%|████████▏ | 41030/50000 [00:01<00:39, 226.76 examples/s]
Saving the dataset (106/128 shards):  83%|████████▎ | 41420/50000 [00:01<00:37, 226.76 examples/s]
Saving the dataset (107/128 shards):  84%|████████▍ | 42200/50000 [00:01<00:34, 226.76 examples/s]
Saving the dataset (108/128 shards):  84%|████████▍ | 42200/50000 [00:01<00:34, 226.76 examples/s]
Saving the dataset (109/128 shards):  86%|████████▌ | 42980/50000 [00:01<00:30, 226.76 examples/s]
Saving the dataset (110/128 shards):  86%|████████▌ | 42980/50000 [00:01<00:30, 226.76 examples/s]
Saving the dataset (111/128 shards):  87%|████████▋ | 43370/50000 [00:01<00:29, 226.76 examples/s]
Saving the dataset (112/128 shards):  88%|████████▊ | 44150/50000 [00:01<00:25, 226.76 examples/s]
Saving the dataset (113/128 shards):  88%|████████▊ | 44150/50000 [00:01<00:25, 226.76 examples/s]
Saving the dataset (114/128 shards):  89%|████████▉ | 44540/50000 [00:01<00:24, 226.76 examples/s]
Saving the dataset (115/128 shards):  91%|█████████▏| 45710/50000 [00:01<00:18, 226.76 examples/s]
Saving the dataset (116/128 shards):  91%|█████████▏| 45710/50000 [00:01<00:18, 226.76 examples/s]
Saving the dataset (117/128 shards):  91%|█████████▏| 45710/50000 [00:01<00:18, 226.76 examples/s]
Saving the dataset (118/128 shards):  92%|█████████▏| 46100/50000 [00:01<00:17, 226.76 examples/s]
Saving the dataset (119/128 shards):  93%|█████████▎| 46490/50000 [00:01<00:15, 226.76 examples/s]
Saving the dataset (120/128 shards):  95%|█████████▌| 47660/50000 [00:01<00:10, 226.76 examples/s]
Saving the dataset (121/128 shards):  95%|█████████▌| 47660/50000 [00:01<00:10, 226.76 examples/s]
Saving the dataset (122/128 shards):  95%|█████████▌| 47660/50000 [00:01<00:10, 226.76 examples/s]
Saving the dataset (123/128 shards):  96%|█████████▌| 48050/50000 [00:01<00:08, 226.76 examples/s]
Saving the dataset (124/128 shards):  98%|█████████▊| 48830/50000 [00:01<00:05, 226.76 examples/s]
Saving the dataset (125/128 shards):  98%|█████████▊| 48830/50000 [00:01<00:05, 226.76 examples/s]
Saving the dataset (126/128 shards):  98%|█████████▊| 49220/50000 [00:01<00:03, 226.76 examples/s]
Saving the dataset (127/128 shards): 100%|██████████| 50000/50000 [00:01<00:00, 226.76 examples/s]
Saving the dataset (128/128 shards): 100%|██████████| 50000/50000 [00:01<00:00, 226.76 examples/s]
Saving the dataset (128/128 shards): 100%|██████████| 50000/50000 [00:01<00:00, 27051.49 examples/s]
+[2026-03-08 15:54:01,260] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:481] [PID:1659682] Unable to find prepared dataset in last_run_prepared/6ef0c0270a5f2e04de2b8e4deededd5a
+[2026-03-08 15:54:01,261] [INFO] [axolotl.utils.data.sft._load_raw_datasets:320] [PID:1659682] Loading raw datasets...
+[2026-03-08 15:54:01,261] [WARNING] [axolotl.utils.data.sft._load_raw_datasets:322] [PID:1659682] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`.
+[2026-03-08 15:54:01,477] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:1659682] Loading dataset: /home/jiaruil5/math_rl/mix_teachers/r3lit_rl/mix_teachers/data/math_operations/primitive_atomic_balanced_sft_50k/balanced_val_alpaca.jsonl with base_type: alpaca and prompt_style: None
+[2026-03-08 15:54:03,528] [INFO] [axolotl.utils.data.utils.handle_long_seq_in_dataset:224] [PID:1659682] min_input_len: 281
+[2026-03-08 15:54:03,528] [INFO] [axolotl.utils.data.utils.handle_long_seq_in_dataset:226] [PID:1659682] max_input_len: 367
+
Dropping Long Sequences (>2048) (num_proc=128):   0%|          | 0/200 [00:00<?, ? examples/s]
Dropping Long Sequences (>2048) (num_proc=128):   1%|          | 2/200 [00:01<02:16,  1.45 examples/s]
Dropping Long Sequences (>2048) (num_proc=128):  13%|█▎        | 26/200 [00:01<00:07, 23.70 examples/s]
Dropping Long Sequences (>2048) (num_proc=128):  25%|██▌       | 50/200 [00:01<00:03, 48.06 examples/s]
Dropping Long Sequences (>2048) (num_proc=128):  36%|███▌      | 72/200 [00:01<00:01, 72.15 examples/s]
Dropping Long Sequences (>2048) (num_proc=128):  74%|███████▍  | 149/200 [00:01<00:00, 187.42 examples/s]
Dropping Long Sequences (>2048) (num_proc=128): 100%|██████████| 200/200 [00:02<00:00, 87.41 examples/s] 
+
Drop Samples with Zero Trainable Tokens (num_proc=128):   0%|          | 0/200 [00:00<?, ? examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=128):   1%|          | 2/200 [00:01<02:20,  1.41 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=128):   3%|▎         | 6/200 [00:01<00:40,  4.81 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=128):   4%|▍         | 8/200 [00:01<00:29,  6.49 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=128):   6%|▌         | 12/200 [00:01<00:17, 10.98 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=128):   8%|▊         | 16/200 [00:01<00:12, 14.30 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=128):  12%|█▏        | 24/200 [00:02<00:07, 24.91 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=128):  15%|█▌        | 30/200 [00:02<00:05, 30.75 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=128):  18%|█▊        | 36/200 [00:02<00:04, 35.25 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=128):  21%|██        | 42/200 [00:02<00:04, 34.60 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=128):  24%|██▍       | 48/200 [00:02<00:04, 35.98 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=128):  96%|█████████▋| 193/200 [00:02<00:00, 340.29 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=128): 100%|██████████| 200/200 [00:03<00:00, 63.20 examples/s] 
+
Add position_id column (Sample Packing) (num_proc=128):   0%|          | 0/200 [00:00<?, ? examples/s]
Add position_id column (Sample Packing) (num_proc=128):   1%|          | 2/200 [00:01<02:20,  1.41 examples/s]
Add position_id column (Sample Packing) (num_proc=128):  11%|█         | 22/200 [00:01<00:09, 19.45 examples/s]
Add position_id column (Sample Packing) (num_proc=128):  21%|██        | 42/200 [00:01<00:04, 38.43 examples/s]
Add position_id column (Sample Packing) (num_proc=128):  42%|████▏     | 84/200 [00:01<00:01, 89.65 examples/s]
Add position_id column (Sample Packing) (num_proc=128):  78%|███████▊  | 156/200 [00:01<00:00, 190.13 examples/s]
Add position_id column (Sample Packing) (num_proc=128): 100%|██████████| 200/200 [00:02<00:00, 82.33 examples/s] 
+
Saving the dataset (0/1 shards):   0%|          | 0/200 [00:00<?, ? examples/s]
Saving the dataset (0/1 shards): 100%|██████████| 200/200 [00:00<00:00, 1654.36 examples/s]
Saving the dataset (1/1 shards): 100%|██████████| 200/200 [00:00<00:00, 1654.36 examples/s]
Saving the dataset (1/1 shards): 100%|██████████| 200/200 [00:00<00:00, 960.52 examples/s] 
+[2026-03-08 15:54:12,503] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:417] [PID:1659682] total_num_tokens: 64_780
+[2026-03-08 15:54:12,522] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:435] [PID:1659682] `total_supervised_tokens: 21_095`
+[2026-03-08 15:54:12,586] [DEBUG] [axolotl.utils.samplers.multipack.pack_parallel:177] [PID:1659682] Using single process for pack_parallel, running sequentially.
+[2026-03-08 15:54:14,254] [DEBUG] [axolotl.utils.samplers.multipack.pack_parallel:177] [PID:1659682] Using single process for pack_parallel, running sequentially.
+[2026-03-08 15:54:14,422] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:1659682] generate_batches time: 0.16837859153747559
+[2026-03-08 15:54:14,422] [DEBUG] [axolotl.utils.samplers.multipack.pack_parallel:177] [PID:1659682] Using single process for pack_parallel, running sequentially.
+[2026-03-08 15:54:14,589] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:1659682] generate_batches time: 0.16713953018188477
+[2026-03-08 15:54:14,590] [DEBUG] [axolotl.utils.samplers.multipack.pack_parallel:177] [PID:1659682] Using single process for pack_parallel, running sequentially.
+[2026-03-08 15:54:14,759] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:1659682] generate_batches time: 0.16975831985473633
+[2026-03-08 15:54:14,760] [DEBUG] [axolotl.utils.samplers.multipack.pack_parallel:177] [PID:1659682] Using single process for pack_parallel, running sequentially.
+[2026-03-08 15:54:14,925] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:1659682] generate_batches time: 0.16611099243164062
+[2026-03-08 15:54:15,041] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:1659682] gather_len_batches: [34]
+[2026-03-08 15:54:15,041] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:494] [PID:1659682] data_loader_len: 4
+[2026-03-08 15:54:15,041] [INFO] [axolotl.utils.trainer.calc_sample_packing_eff_est:510] [PID:1659682] sample_packing_eff_est across ranks: [0.9303193933823529]
+[2026-03-08 15:54:15,041] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:522] [PID:1659682] sample_packing_eff_est: None
+[2026-03-08 15:54:15,041] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:533] [PID:1659682] total_num_steps: 12
+[2026-03-08 15:54:15,100] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:417] [PID:1659682] total_num_tokens: 16_272_381
+[2026-03-08 15:54:15,393] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:435] [PID:1659682] `total_supervised_tokens: 5_346_651`
+[2026-03-08 15:54:15,966] [DEBUG] [axolotl.utils.samplers.multipack.pack_parallel:177] [PID:1659682] Using single process for pack_parallel, running sequentially.
+[2026-03-08 15:54:16,339] [DEBUG] [axolotl.utils.samplers.multipack.pack_parallel:177] [PID:1659682] Using single process for pack_parallel, running sequentially.
+[2026-03-08 15:54:16,669] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:1659682] generate_batches time: 0.3653264045715332
+[2026-03-08 15:54:16,683] [DEBUG] [axolotl.utils.samplers.multipack.pack_parallel:177] [PID:1659682] Using single process for pack_parallel, running sequentially.
+[2026-03-08 15:54:17,005] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:1659682] generate_batches time: 0.33499598503112793
+[2026-03-08 15:54:17,012] [DEBUG] [axolotl.utils.samplers.multipack.pack_parallel:177] [PID:1659682] Using single process for pack_parallel, running sequentially.
+[2026-03-08 15:54:17,329] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:1659682] generate_batches time: 0.3225700855255127
+[2026-03-08 15:54:17,336] [DEBUG] [axolotl.utils.samplers.multipack.pack_parallel:177] [PID:1659682] Using single process for pack_parallel, running sequentially.
+[2026-03-08 15:54:17,657] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:1659682] generate_batches time: 0.3276090621948242
+[2026-03-08 15:54:17,658] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:1659682] gather_len_batches: [8334]
+[2026-03-08 15:54:17,658] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:494] [PID:1659682] data_loader_len: 1041
+[2026-03-08 15:54:17,658] [INFO] [axolotl.utils.trainer.calc_sample_packing_eff_est:510] [PID:1659682] sample_packing_eff_est across ranks: [0.9533835535344672]
+[2026-03-08 15:54:17,658] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:522] [PID:1659682] sample_packing_eff_est: 0.96
+[2026-03-08 15:54:17,658] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:533] [PID:1659682] total_num_steps: 3123
+[2026-03-08 15:54:17,659] [INFO] [axolotl.utils.data.sft._prepare_standard_dataset:121] [PID:1659682] Maximum number of steps set at 3123
+[2026-03-08 15:54:17,704] [DEBUG] [axolotl.train.setup_model_and_tokenizer:70] [PID:1659682] loading tokenizer... /home/jiaruil5/math_rl/mix_teachers/r3lit_rl/models/Qwen/Qwen3-4B-Instruct-2507
+[2026-03-08 15:54:18,529] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:285] [PID:1659682] EOS: 151645 / <|im_end|>
+[2026-03-08 15:54:18,529] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:286] [PID:1659682] BOS: None / None
+[2026-03-08 15:54:18,529] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:287] [PID:1659682] PAD: 151643 / <|endoftext|>
+[2026-03-08 15:54:18,529] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:288] [PID:1659682] UNK: None / None
+[2026-03-08 15:54:18,529] [DEBUG] [axolotl.train.setup_model_and_tokenizer:82] [PID:1659682] Loading model
+[2026-03-08 15:54:18,574] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:87] [PID:1659682] Patched Trainer.evaluation_loop with nanmean loss calculation
+[2026-03-08 15:54:18,576] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:138] [PID:1659682] Patched Trainer._maybe_log_save_evaluate with nanmean loss calculation
+[2026-03-08 15:54:18,576] [INFO] [axolotl.loaders.patch_manager._apply_multipack_patches:346] [PID:1659682] Applying multipack dataloader patch for sample packing...
+
Fetching 22 files:   0%|          | 0/22 [00:00<?, ?it/s]
Fetching 22 files: 100%|██████████| 22/22 [00:00<00:00, 891.79it/s]
+
Loading weights:   0%|          | 0/398 [00:00<?, ?it/s]
Loading weights:   0%|          | 1/398 [00:00<00:00, 16710.37it/s, Materializing param=model.embed_tokens.weight]
Loading weights:   0%|          | 1/398 [00:00<00:00, 7269.16it/s, Materializing param=model.embed_tokens.weight] 
Loading weights:   1%|          | 2/398 [00:27<1:30:11, 13.67s/it, Materializing param=model.embed_tokens.weight]
Loading weights:   1%|          | 2/398 [00:27<1:30:11, 13.67s/it, Materializing param=model.layers.0.input_layernorm.weight]
Loading weights:   1%|          | 2/398 [00:27<1:30:11, 13.67s/it, Materializing param=model.layers.0.input_layernorm.weight]
Loading weights:   1%|          | 3/398 [00:27<1:29:58, 13.67s/it, Materializing param=model.layers.0.mlp.down_proj.weight]  
Loading weights:   1%|          | 3/398 [00:27<1:29:58, 13.67s/it, Materializing param=model.layers.0.mlp.down_proj.weight]
Loading weights:   1%|          | 4/398 [00:27<1:29:44, 13.67s/it, Materializing param=model.layers.0.mlp.gate_proj.weight]
Loading weights:   1%|          | 4/398 [00:27<1:29:44, 13.67s/it, Materializing param=model.layers.0.mlp.gate_proj.weight]
Loading weights:   1%|▏         | 5/398 [00:27<1:29:30, 13.67s/it, Materializing param=model.layers.0.mlp.up_proj.weight]  
Loading weights:   1%|▏         | 5/398 [00:27<1:29:30, 13.67s/it, Materializing param=model.layers.0.mlp.up_proj.weight]
Loading weights:   2%|▏         | 6/398 [00:27<23:40,  3.62s/it, Materializing param=model.layers.0.mlp.up_proj.weight]  
Loading weights:   2%|▏         | 6/398 [00:27<23:40,  3.62s/it, Materializing param=model.layers.0.post_attention_layernorm.weight]
Loading weights:   2%|▏         | 6/398 [00:27<23:40,  3.62s/it, Materializing param=model.layers.0.post_attention_layernorm.weight]
Loading weights:   2%|▏         | 7/398 [00:27<23:36,  3.62s/it, Materializing param=model.layers.0.self_attn.k_norm.weight]        
Loading weights:   2%|▏         | 7/398 [00:27<23:36,  3.62s/it, Materializing param=model.layers.0.self_attn.k_norm.weight]
Loading weights:   2%|▏         | 8/398 [00:27<23:32,  3.62s/it, Materializing param=model.layers.0.self_attn.k_proj.weight]
Loading weights:   2%|▏         | 8/398 [00:27<23:32,  3.62s/it, Materializing param=model.layers.0.self_attn.k_proj.weight]
Loading weights:   2%|▏         | 9/398 [00:27<23:29,  3.62s/it, Materializing param=model.layers.0.self_attn.o_proj.weight]
Loading weights:   2%|▏         | 9/398 [00:27<23:29,  3.62s/it, Materializing param=model.layers.0.self_attn.o_proj.weight]
Loading weights:   3%|▎         | 10/398 [00:28<11:40,  1.81s/it, Materializing param=model.layers.0.self_attn.o_proj.weight]
Loading weights:   3%|▎         | 10/398 [00:28<11:40,  1.81s/it, Materializing param=model.layers.0.self_attn.q_norm.weight]
Loading weights:   3%|▎         | 10/398 [00:28<11:40,  1.81s/it, Materializing param=model.layers.0.self_attn.q_norm.weight]
Loading weights:   3%|▎         | 11/398 [00:28<11:38,  1.81s/it, Materializing param=model.layers.0.self_attn.q_proj.weight]
Loading weights:   3%|▎         | 11/398 [00:28<11:38,  1.81s/it, Materializing param=model.layers.0.self_attn.q_proj.weight]
Loading weights:   3%|▎         | 12/398 [00:28<08:38,  1.34s/it, Materializing param=model.layers.0.self_attn.q_proj.weight]
Loading weights:   3%|▎         | 12/398 [00:28<08:38,  1.34s/it, Materializing param=model.layers.0.self_attn.v_proj.weight]
Loading weights:   3%|▎         | 12/398 [00:28<08:38,  1.34s/it, Materializing param=model.layers.0.self_attn.v_proj.weight]
Loading weights:   3%|▎         | 13/398 [00:28<08:37,  1.34s/it, Materializing param=model.layers.1.input_layernorm.weight] 
Loading weights:   3%|▎         | 13/398 [00:28<08:37,  1.34s/it, Materializing param=model.layers.1.input_layernorm.weight]
Loading weights:   4%|▎         | 14/398 [00:28<06:31,  1.02s/it, Materializing param=model.layers.1.input_layernorm.weight]
Loading weights:   4%|▎         | 14/398 [00:28<06:31,  1.02s/it, Materializing param=model.layers.1.mlp.down_proj.weight]  
Loading weights:   4%|▎         | 14/398 [00:28<06:31,  1.02s/it, Materializing param=model.layers.1.mlp.down_proj.weight]
Loading weights:   4%|▍         | 15/398 [00:28<06:30,  1.02s/it, Materializing param=model.layers.1.mlp.gate_proj.weight]
Loading weights:   4%|▍         | 15/398 [00:28<06:30,  1.02s/it, Materializing param=model.layers.1.mlp.gate_proj.weight]
Loading weights:   4%|▍         | 16/398 [00:29<05:47,  1.10it/s, Materializing param=model.layers.1.mlp.gate_proj.weight]
Loading weights:   4%|▍         | 16/398 [00:30<05:47,  1.10it/s, Materializing param=model.layers.1.mlp.up_proj.weight]  
Loading weights:   4%|▍         | 16/398 [00:30<05:47,  1.10it/s, Materializing param=model.layers.1.mlp.up_proj.weight]
Loading weights:   4%|▍         | 17/398 [00:30<05:42,  1.11it/s, Materializing param=model.layers.1.mlp.up_proj.weight]
Loading weights:   4%|▍         | 17/398 [00:30<05:42,  1.11it/s, Materializing param=model.layers.1.post_attention_layernorm.weight]
Loading weights:   4%|▍         | 17/398 [00:30<05:42,  1.11it/s, Materializing param=model.layers.1.post_attention_layernorm.weight]
Loading weights:   5%|▍         | 18/398 [00:30<05:41,  1.11it/s, Materializing param=model.layers.1.self_attn.k_norm.weight]        
Loading weights:   5%|▍         | 18/398 [00:30<05:41,  1.11it/s, Materializing param=model.layers.1.self_attn.k_norm.weight]
Loading weights:   5%|▍         | 19/398 [00:30<05:40,  1.11it/s, Materializing param=model.layers.1.self_attn.k_proj.weight]
Loading weights:   5%|▍         | 19/398 [00:30<05:40,  1.11it/s, Materializing param=model.layers.1.self_attn.k_proj.weight]
Loading weights:   5%|▌         | 20/398 [00:30<03:29,  1.81it/s, Materializing param=model.layers.1.self_attn.k_proj.weight]
Loading weights:   5%|▌         | 20/398 [00:30<03:29,  1.81it/s, Materializing param=model.layers.1.self_attn.o_proj.weight]
Loading weights:   5%|▌         | 20/398 [00:30<03:29,  1.81it/s, Materializing param=model.layers.1.self_attn.o_proj.weight]
Loading weights:   5%|▌         | 21/398 [00:31<03:26,  1.83it/s, Materializing param=model.layers.1.self_attn.o_proj.weight]
Loading weights:   5%|▌         | 21/398 [00:31<03:26,  1.83it/s, Materializing param=model.layers.1.self_attn.q_norm.weight]
Loading weights:   5%|▌         | 21/398 [00:31<03:26,  1.83it/s, Materializing param=model.layers.1.self_attn.q_norm.weight]
Loading weights:   6%|▌         | 22/398 [00:31<02:58,  2.11it/s, Materializing param=model.layers.1.self_attn.q_norm.weight]
Loading weights:   6%|▌         | 22/398 [00:31<02:58,  2.11it/s, Materializing param=model.layers.1.self_attn.q_proj.weight]
Loading weights:   6%|▌         | 22/398 [00:31<02:58,  2.11it/s, Materializing param=model.layers.1.self_attn.q_proj.weight]
Loading weights:   6%|▌         | 23/398 [00:31<02:57,  2.11it/s, Materializing param=model.layers.1.self_attn.v_proj.weight]
Loading weights:   6%|▌         | 23/398 [00:31<02:57,  2.11it/s, Materializing param=model.layers.1.self_attn.v_proj.weight]
Loading weights:   6%|▌         | 24/398 [00:31<02:01,  3.09it/s, Materializing param=model.layers.1.self_attn.v_proj.weight]
Loading weights:   6%|▌         | 24/398 [00:31<02:01,  3.09it/s, Materializing param=model.layers.2.input_layernorm.weight] 
Loading weights:   6%|▌         | 24/398 [00:31<02:01,  3.09it/s, Materializing param=model.layers.2.input_layernorm.weight]
Loading weights:   6%|▋         | 25/398 [00:31<02:00,  3.09it/s, Materializing param=model.layers.2.mlp.down_proj.weight]  
Loading weights:   6%|▋         | 25/398 [00:31<02:00,  3.09it/s, Materializing param=model.layers.2.mlp.down_proj.weight]
Loading weights:   7%|▋         | 26/398 [00:31<02:00,  3.09it/s, Materializing param=model.layers.2.mlp.gate_proj.weight]
Loading weights:   7%|▋         | 26/398 [00:31<02:00,  3.09it/s, Materializing param=model.layers.2.mlp.gate_proj.weight]
Loading weights:   7%|▋         | 27/398 [00:32<01:37,  3.80it/s, Materializing param=model.layers.2.mlp.gate_proj.weight]
Loading weights:   7%|▋         | 27/398 [00:32<01:37,  3.80it/s, Materializing param=model.layers.2.mlp.up_proj.weight]  
Loading weights:   7%|▋         | 27/398 [00:32<01:37,  3.80it/s, Materializing param=model.layers.2.mlp.up_proj.weight]
Loading weights:   7%|▋         | 28/398 [00:32<01:42,  3.59it/s, Materializing param=model.layers.2.mlp.up_proj.weight]
Loading weights:   7%|▋         | 28/398 [00:32<01:42,  3.59it/s, Materializing param=model.layers.2.post_attention_layernorm.weight]
Loading weights:   7%|▋         | 28/398 [00:32<01:42,  3.59it/s, Materializing param=model.layers.2.post_attention_layernorm.weight]
Loading weights:   7%|▋         | 29/398 [00:32<01:42,  3.59it/s, Materializing param=model.layers.2.self_attn.k_norm.weight]        
Loading weights:   7%|▋         | 29/398 [00:32<01:42,  3.59it/s, Materializing param=model.layers.2.self_attn.k_norm.weight]
Loading weights:   8%|▊         | 30/398 [00:32<01:17,  4.74it/s, Materializing param=model.layers.2.self_attn.k_norm.weight]
Loading weights:   8%|▊         | 30/398 [00:32<01:17,  4.74it/s, Materializing param=model.layers.2.self_attn.k_proj.weight]
Loading weights:   8%|▊         | 30/398 [00:32<01:17,  4.74it/s, Materializing param=model.layers.2.self_attn.k_proj.weight]
Loading weights:   8%|▊         | 31/398 [00:32<01:17,  4.74it/s, Materializing param=model.layers.2.self_attn.o_proj.weight]
Loading weights:   8%|▊         | 31/398 [00:32<01:17,  4.74it/s, Materializing param=model.layers.2.self_attn.o_proj.weight]
Loading weights:   8%|▊         | 32/398 [00:32<00:59,  6.16it/s, Materializing param=model.layers.2.self_attn.o_proj.weight]
Loading weights:   8%|▊         | 32/398 [00:32<00:59,  6.16it/s, Materializing param=model.layers.2.self_attn.q_norm.weight]
Loading weights:   8%|▊         | 32/398 [00:32<00:59,  6.16it/s, Materializing param=model.layers.2.self_attn.q_norm.weight]
Loading weights:   8%|▊         | 33/398 [00:33<00:59,  6.16it/s, Materializing param=model.layers.2.self_attn.q_proj.weight]
Loading weights:   8%|▊         | 33/398 [00:33<00:59,  6.16it/s, Materializing param=model.layers.2.self_attn.q_proj.weight]
Loading weights:   9%|▊         | 34/398 [00:33<00:53,  6.77it/s, Materializing param=model.layers.2.self_attn.q_proj.weight]
Loading weights:   9%|▊         | 34/398 [00:33<00:53,  6.77it/s, Materializing param=model.layers.2.self_attn.v_proj.weight]
Loading weights:   9%|▊         | 34/398 [00:33<00:53,  6.77it/s, Materializing param=model.layers.2.self_attn.v_proj.weight]
Loading weights:   9%|▉         | 35/398 [00:33<00:53,  6.77it/s, Materializing param=model.layers.3.input_layernorm.weight] 
Loading weights:   9%|▉         | 35/398 [00:33<00:53,  6.77it/s, Materializing param=model.layers.3.input_layernorm.weight]
Loading weights:   9%|▉         | 36/398 [00:33<00:58,  6.21it/s, Materializing param=model.layers.3.input_layernorm.weight]
Loading weights:   9%|▉         | 36/398 [00:33<00:58,  6.21it/s, Materializing param=model.layers.3.mlp.down_proj.weight]  
Loading weights:   9%|▉         | 36/398 [00:33<00:58,  6.21it/s, Materializing param=model.layers.3.mlp.down_proj.weight]
Loading weights:   9%|▉         | 37/398 [00:33<01:03,  5.65it/s, Materializing param=model.layers.3.mlp.down_proj.weight]
Loading weights:   9%|▉         | 37/398 [00:33<01:03,  5.65it/s, Materializing param=model.layers.3.mlp.gate_proj.weight]
Loading weights:   9%|▉         | 37/398 [00:33<01:03,  5.65it/s, Materializing param=model.layers.3.mlp.gate_proj.weight]
Loading weights:  10%|▉         | 38/398 [00:34<01:16,  4.70it/s, Materializing param=model.layers.3.mlp.gate_proj.weight]
Loading weights:  10%|▉         | 38/398 [00:34<01:16,  4.70it/s, Materializing param=model.layers.3.mlp.up_proj.weight]  
Loading weights:  10%|▉         | 38/398 [00:34<01:16,  4.70it/s, Materializing param=model.layers.3.mlp.up_proj.weight]
Loading weights:  10%|▉         | 39/398 [00:34<01:38,  3.66it/s, Materializing param=model.layers.3.mlp.up_proj.weight]
Loading weights:  10%|▉         | 39/398 [00:34<01:38,  3.66it/s, Materializing param=model.layers.3.post_attention_layernorm.weight]
Loading weights:  10%|▉         | 39/398 [00:34<01:38,  3.66it/s, Materializing param=model.layers.3.post_attention_layernorm.weight]
Loading weights:  10%|█         | 40/398 [00:34<01:37,  3.66it/s, Materializing param=model.layers.3.self_attn.k_norm.weight]        
Loading weights:  10%|█         | 40/398 [00:34<01:37,  3.66it/s, Materializing param=model.layers.3.self_attn.k_norm.weight]
Loading weights:  10%|█         | 41/398 [00:34<01:08,  5.19it/s, Materializing param=model.layers.3.self_attn.k_norm.weight]
Loading weights:  10%|█         | 41/398 [00:34<01:08,  5.19it/s, Materializing param=model.layers.3.self_attn.k_proj.weight]
Loading weights:  10%|█         | 41/398 [00:34<01:08,  5.19it/s, Materializing param=model.layers.3.self_attn.k_proj.weight]
Loading weights:  11%|█         | 42/398 [00:34<01:08,  5.19it/s, Materializing param=model.layers.3.self_attn.o_proj.weight]
Loading weights:  11%|█         | 42/398 [00:34<01:08,  5.19it/s, Materializing param=model.layers.3.self_attn.o_proj.weight]
Loading weights:  11%|█         | 43/398 [00:35<01:00,  5.90it/s, Materializing param=model.layers.3.self_attn.o_proj.weight]
Loading weights:  11%|█         | 43/398 [00:35<01:00,  5.90it/s, Materializing param=model.layers.3.self_attn.q_norm.weight]
Loading weights:  11%|█         | 43/398 [00:35<01:00,  5.90it/s, Materializing param=model.layers.3.self_attn.q_norm.weight]
Loading weights:  11%|█         | 44/398 [00:35<01:09,  5.12it/s, Materializing param=model.layers.3.self_attn.q_norm.weight]
Loading weights:  11%|█         | 44/398 [00:35<01:09,  5.12it/s, Materializing param=model.layers.3.self_attn.q_proj.weight]
Loading weights:  11%|█         | 44/398 [00:35<01:09,  5.12it/s, Materializing param=model.layers.3.self_attn.q_proj.weight]
Loading weights:  11%|█▏        | 45/398 [00:35<01:09,  5.12it/s, Materializing param=model.layers.3.self_attn.v_proj.weight]
Loading weights:  11%|█▏        | 45/398 [00:35<01:09,  5.12it/s, Materializing param=model.layers.3.self_attn.v_proj.weight]
Loading weights:  12%|█▏        | 46/398 [00:35<01:08,  5.12it/s, Materializing param=model.layers.4.input_layernorm.weight] 
Loading weights:  12%|█▏        | 46/398 [00:35<01:08,  5.12it/s, Materializing param=model.layers.4.input_layernorm.weight]
Loading weights:  12%|█▏        | 47/398 [00:35<01:08,  5.12it/s, Materializing param=model.layers.4.mlp.down_proj.weight]  
Loading weights:  12%|█▏        | 47/398 [00:35<01:08,  5.12it/s, Materializing param=model.layers.4.mlp.down_proj.weight]
Loading weights:  12%|█▏        | 48/398 [00:35<01:08,  5.12it/s, Materializing param=model.layers.4.mlp.gate_proj.weight]
Loading weights:  12%|█▏        | 48/398 [00:35<01:08,  5.12it/s, Materializing param=model.layers.4.mlp.gate_proj.weight]
Loading weights:  12%|█▏        | 49/398 [00:35<00:44,  7.85it/s, Materializing param=model.layers.4.mlp.gate_proj.weight]
Loading weights:  12%|█▏        | 49/398 [00:35<00:44,  7.85it/s, Materializing param=model.layers.4.mlp.up_proj.weight]  
Loading weights:  12%|█▏        | 49/398 [00:35<00:44,  7.85it/s, Materializing param=model.layers.4.mlp.up_proj.weight]
Loading weights:  13%|█▎        | 50/398 [00:36<01:09,  5.04it/s, Materializing param=model.layers.4.mlp.up_proj.weight]
Loading weights:  13%|█▎        | 50/398 [00:36<01:09,  5.04it/s, Materializing param=model.layers.4.post_attention_layernorm.weight]
Loading weights:  13%|█▎        | 50/398 [00:36<01:09,  5.04it/s, Materializing param=model.layers.4.post_attention_layernorm.weight]
Loading weights:  13%|█▎        | 51/398 [00:36<01:07,  5.11it/s, Materializing param=model.layers.4.post_attention_layernorm.weight]
Loading weights:  13%|█▎        | 51/398 [00:36<01:07,  5.11it/s, Materializing param=model.layers.4.self_attn.k_norm.weight]        
Loading weights:  13%|█▎        | 51/398 [00:36<01:07,  5.11it/s, Materializing param=model.layers.4.self_attn.k_norm.weight]
Loading weights:  13%|█▎        | 52/398 [00:36<01:07,  5.11it/s, Materializing param=model.layers.4.self_attn.k_proj.weight]
Loading weights:  13%|█▎        | 52/398 [00:36<01:07,  5.11it/s, Materializing param=model.layers.4.self_attn.k_proj.weight]
Loading weights:  13%|█▎        | 53/398 [00:36<01:07,  5.11it/s, Materializing param=model.layers.4.self_attn.o_proj.weight]
Loading weights:  13%|█▎        | 53/398 [00:36<01:07,  5.11it/s, Materializing param=model.layers.4.self_attn.o_proj.weight]
Loading weights:  14%|█▎        | 54/398 [00:36<00:46,  7.47it/s, Materializing param=model.layers.4.self_attn.o_proj.weight]
Loading weights:  14%|█▎        | 54/398 [00:36<00:46,  7.47it/s, Materializing param=model.layers.4.self_attn.q_norm.weight]
Loading weights:  14%|█▎        | 54/398 [00:36<00:46,  7.47it/s, Materializing param=model.layers.4.self_attn.q_norm.weight]
Loading weights:  14%|█▍        | 55/398 [00:36<00:45,  7.47it/s, Materializing param=model.layers.4.self_attn.q_proj.weight]
Loading weights:  14%|█▍        | 55/398 [00:36<00:45,  7.47it/s, Materializing param=model.layers.4.self_attn.q_proj.weight]
Loading weights:  14%|█▍        | 56/398 [00:36<00:42,  8.02it/s, Materializing param=model.layers.4.self_attn.q_proj.weight]
Loading weights:  14%|█▍        | 56/398 [00:36<00:42,  8.02it/s, Materializing param=model.layers.4.self_attn.v_proj.weight]
Loading weights:  14%|█▍        | 56/398 [00:36<00:42,  8.02it/s, Materializing param=model.layers.4.self_attn.v_proj.weight]
Loading weights:  14%|█▍        | 57/398 [00:37<00:42,  8.02it/s, Materializing param=model.layers.5.input_layernorm.weight] 
Loading weights:  14%|█▍        | 57/398 [00:37<00:42,  8.02it/s, Materializing param=model.layers.5.input_layernorm.weight]
Loading weights:  15%|█▍        | 58/398 [00:37<00:41,  8.16it/s, Materializing param=model.layers.5.input_layernorm.weight]
Loading weights:  15%|█▍        | 58/398 [00:37<00:41,  8.16it/s, Materializing param=model.layers.5.mlp.down_proj.weight]  
Loading weights:  15%|█▍        | 58/398 [00:37<00:41,  8.16it/s, Materializing param=model.layers.5.mlp.down_proj.weight]
Loading weights:  15%|█▍        | 59/398 [00:37<00:49,  6.86it/s, Materializing param=model.layers.5.mlp.down_proj.weight]
Loading weights:  15%|█▍        | 59/398 [00:37<00:49,  6.86it/s, Materializing param=model.layers.5.mlp.gate_proj.weight]
Loading weights:  15%|█▍        | 59/398 [00:37<00:49,  6.86it/s, Materializing param=model.layers.5.mlp.gate_proj.weight]
Loading weights:  15%|█▌        | 60/398 [00:38<02:26,  2.31it/s, Materializing param=model.layers.5.mlp.gate_proj.weight]
Loading weights:  15%|█▌        | 60/398 [00:38<02:26,  2.31it/s, Materializing param=model.layers.5.mlp.up_proj.weight]  
Loading weights:  15%|█▌        | 60/398 [00:38<02:26,  2.31it/s, Materializing param=model.layers.5.mlp.up_proj.weight]
Loading weights:  15%|█▌        | 61/398 [00:39<02:32,  2.21it/s, Materializing param=model.layers.5.mlp.up_proj.weight]
Loading weights:  15%|█▌        | 61/398 [00:39<02:32,  2.21it/s, Materializing param=model.layers.5.post_attention_layernorm.weight]
Loading weights:  15%|█▌        | 61/398 [00:39<02:32,  2.21it/s, Materializing param=model.layers.5.post_attention_layernorm.weight]
Loading weights:  16%|█▌        | 62/398 [00:39<02:31,  2.21it/s, Materializing param=model.layers.5.self_attn.k_norm.weight]        
Loading weights:  16%|█▌        | 62/398 [00:39<02:31,  2.21it/s, Materializing param=model.layers.5.self_attn.k_norm.weight]
Loading weights:  16%|█▌        | 63/398 [00:39<02:31,  2.21it/s, Materializing param=model.layers.5.self_attn.k_proj.weight]
Loading weights:  16%|█▌        | 63/398 [00:39<02:31,  2.21it/s, Materializing param=model.layers.5.self_attn.k_proj.weight]
Loading weights:  16%|█▌        | 64/398 [00:39<01:23,  3.99it/s, Materializing param=model.layers.5.self_attn.k_proj.weight]
Loading weights:  16%|█▌        | 64/398 [00:39<01:23,  3.99it/s, Materializing param=model.layers.5.self_attn.o_proj.weight]
Loading weights:  16%|█▌        | 64/398 [00:39<01:23,  3.99it/s, Materializing param=model.layers.5.self_attn.o_proj.weight]
Loading weights:  16%|█▋        | 65/398 [00:39<01:23,  3.99it/s, Materializing param=model.layers.5.self_attn.q_norm.weight]
Loading weights:  16%|█▋        | 65/398 [00:39<01:23,  3.99it/s, Materializing param=model.layers.5.self_attn.q_norm.weight]
Loading weights:  17%|█▋        | 66/398 [00:39<01:08,  4.83it/s, Materializing param=model.layers.5.self_attn.q_norm.weight]
Loading weights:  17%|█▋        | 66/398 [00:39<01:08,  4.83it/s, Materializing param=model.layers.5.self_attn.q_proj.weight]
Loading weights:  17%|█▋        | 66/398 [00:39<01:08,  4.83it/s, Materializing param=model.layers.5.self_attn.q_proj.weight]
Loading weights:  17%|█▋        | 67/398 [00:39<01:08,  4.83it/s, Materializing param=model.layers.5.self_attn.v_proj.weight]
Loading weights:  17%|█▋        | 67/398 [00:39<01:08,  4.83it/s, Materializing param=model.layers.5.self_attn.v_proj.weight]
Loading weights:  17%|█▋        | 68/398 [00:39<01:08,  4.83it/s, Materializing param=model.layers.6.input_layernorm.weight] 
Loading weights:  17%|█▋        | 68/398 [00:39<01:08,  4.83it/s, Materializing param=model.layers.6.input_layernorm.weight]
Loading weights:  17%|█▋        | 69/398 [00:40<00:57,  5.77it/s, Materializing param=model.layers.6.input_layernorm.weight]
Loading weights:  17%|█▋        | 69/398 [00:40<00:57,  5.77it/s, Materializing param=model.layers.6.mlp.down_proj.weight]  
Loading weights:  17%|█▋        | 69/398 [00:40<00:57,  5.77it/s, Materializing param=model.layers.6.mlp.down_proj.weight]
Loading weights:  18%|█▊        | 70/398 [00:40<00:56,  5.77it/s, Materializing param=model.layers.6.mlp.gate_proj.weight]
Loading weights:  18%|█▊        | 70/398 [00:40<00:56,  5.77it/s, Materializing param=model.layers.6.mlp.gate_proj.weight]
Loading weights:  18%|█▊        | 71/398 [00:40<01:04,  5.07it/s, Materializing param=model.layers.6.mlp.gate_proj.weight]
Loading weights:  18%|█▊        | 71/398 [00:40<01:04,  5.07it/s, Materializing param=model.layers.6.mlp.up_proj.weight]  
Loading weights:  18%|█▊        | 71/398 [00:40<01:04,  5.07it/s, Materializing param=model.layers.6.mlp.up_proj.weight]
Loading weights:  18%|█▊        | 72/398 [00:40<01:04,  5.07it/s, Materializing param=model.layers.6.post_attention_layernorm.weight]
Loading weights:  18%|█▊        | 72/398 [00:40<01:04,  5.07it/s, Materializing param=model.layers.6.post_attention_layernorm.weight]
Loading weights:  18%|█▊        | 73/398 [00:40<01:04,  5.07it/s, Materializing param=model.layers.6.self_attn.k_norm.weight]        
Loading weights:  18%|█▊        | 73/398 [00:40<01:04,  5.07it/s, Materializing param=model.layers.6.self_attn.k_norm.weight]
Loading weights:  19%|█▊        | 74/398 [00:40<00:46,  6.98it/s, Materializing param=model.layers.6.self_attn.k_norm.weight]
Loading weights:  19%|█▊        | 74/398 [00:40<00:46,  6.98it/s, Materializing param=model.layers.6.self_attn.k_proj.weight]
Loading weights:  19%|█▊        | 74/398 [00:40<00:46,  6.98it/s, Materializing param=model.layers.6.self_attn.k_proj.weight]
Loading weights:  19%|█▉        | 75/398 [00:40<00:46,  6.98it/s, Materializing param=model.layers.6.self_attn.o_proj.weight]
Loading weights:  19%|█▉        | 75/398 [00:40<00:46,  6.98it/s, Materializing param=model.layers.6.self_attn.o_proj.weight]
Loading weights:  19%|█▉        | 76/398 [00:40<00:46,  6.98it/s, Materializing param=model.layers.6.self_attn.q_norm.weight]
Loading weights:  19%|█▉        | 76/398 [00:40<00:46,  6.98it/s, Materializing param=model.layers.6.self_attn.q_norm.weight]
Loading weights:  19%|█▉        | 77/398 [00:40<00:46,  6.98it/s, Materializing param=model.layers.6.self_attn.q_proj.weight]
Loading weights:  19%|█▉        | 77/398 [00:40<00:46,  6.98it/s, Materializing param=model.layers.6.self_attn.q_proj.weight]
Loading weights:  20%|█▉        | 78/398 [00:41<00:35,  8.92it/s, Materializing param=model.layers.6.self_attn.q_proj.weight]
Loading weights:  20%|█▉        | 78/398 [00:41<00:35,  8.92it/s, Materializing param=model.layers.6.self_attn.v_proj.weight]
Loading weights:  20%|█▉        | 78/398 [00:41<00:35,  8.92it/s, Materializing param=model.layers.6.self_attn.v_proj.weight]
Loading weights:  20%|█▉        | 79/398 [00:41<00:35,  8.92it/s, Materializing param=model.layers.7.input_layernorm.weight] 
Loading weights:  20%|█▉        | 79/398 [00:41<00:35,  8.92it/s, Materializing param=model.layers.7.input_layernorm.weight]
Loading weights:  20%|██        | 80/398 [00:41<00:37,  8.49it/s, Materializing param=model.layers.7.input_layernorm.weight]
Loading weights:  20%|██        | 80/398 [00:41<00:37,  8.49it/s, Materializing param=model.layers.7.mlp.down_proj.weight]  
Loading weights:  20%|██        | 80/398 [00:41<00:37,  8.49it/s, Materializing param=model.layers.7.mlp.down_proj.weight]
Loading weights:  20%|██        | 81/398 [00:41<00:37,  8.49it/s, Materializing param=model.layers.7.mlp.gate_proj.weight]
Loading weights:  20%|██        | 81/398 [00:41<00:37,  8.49it/s, Materializing param=model.layers.7.mlp.gate_proj.weight]
Loading weights:  21%|██        | 82/398 [00:42<01:08,  4.59it/s, Materializing param=model.layers.7.mlp.gate_proj.weight]
Loading weights:  21%|██        | 82/398 [00:42<01:08,  4.59it/s, Materializing param=model.layers.7.mlp.up_proj.weight]  
Loading weights:  21%|██        | 82/398 [00:42<01:08,  4.59it/s, Materializing param=model.layers.7.mlp.up_proj.weight]
Loading weights:  21%|██        | 83/398 [00:42<01:08,  4.59it/s, Materializing param=model.layers.7.post_attention_layernorm.weight]
Loading weights:  21%|██        | 83/398 [00:42<01:08,  4.59it/s, Materializing param=model.layers.7.post_attention_layernorm.weight]
Loading weights:  21%|██        | 84/398 [00:42<01:08,  4.59it/s, Materializing param=model.layers.7.self_attn.k_norm.weight]        
Loading weights:  21%|██        | 84/398 [00:42<01:08,  4.59it/s, Materializing param=model.layers.7.self_attn.k_norm.weight]
Loading weights:  21%|██▏       | 85/398 [00:42<01:08,  4.59it/s, Materializing param=model.layers.7.self_attn.k_proj.weight]
Loading weights:  21%|██▏       | 85/398 [00:42<01:08,  4.59it/s, Materializing param=model.layers.7.self_attn.k_proj.weight]
Loading weights:  22%|██▏       | 86/398 [00:42<00:43,  7.23it/s, Materializing param=model.layers.7.self_attn.k_proj.weight]
Loading weights:  22%|██▏       | 86/398 [00:42<00:43,  7.23it/s, Materializing param=model.layers.7.self_attn.o_proj.weight]
Loading weights:  22%|██▏       | 86/398 [00:42<00:43,  7.23it/s, Materializing param=model.layers.7.self_attn.o_proj.weight]
Loading weights:  22%|██▏       | 87/398 [00:42<00:43,  7.23it/s, Materializing param=model.layers.7.self_attn.q_norm.weight]
Loading weights:  22%|██▏       | 87/398 [00:42<00:43,  7.23it/s, Materializing param=model.layers.7.self_attn.q_norm.weight]
Loading weights:  22%|██▏       | 88/398 [00:42<00:38,  8.09it/s, Materializing param=model.layers.7.self_attn.q_norm.weight]
Loading weights:  22%|██▏       | 88/398 [00:42<00:38,  8.09it/s, Materializing param=model.layers.7.self_attn.q_proj.weight]
Loading weights:  22%|██▏       | 88/398 [00:42<00:38,  8.09it/s, Materializing param=model.layers.7.self_attn.q_proj.weight]
Loading weights:  22%|██▏       | 89/398 [00:42<00:38,  8.09it/s, Materializing param=model.layers.7.self_attn.v_proj.weight]
Loading weights:  22%|██▏       | 89/398 [00:42<00:38,  8.09it/s, Materializing param=model.layers.7.self_attn.v_proj.weight]
Loading weights:  23%|██▎       | 90/398 [00:43<00:42,  7.28it/s, Materializing param=model.layers.7.self_attn.v_proj.weight]
Loading weights:  23%|██▎       | 90/398 [00:43<00:42,  7.28it/s, Materializing param=model.layers.8.input_layernorm.weight] 
Loading weights:  23%|██▎       | 90/398 [00:43<00:42,  7.28it/s, Materializing param=model.layers.8.input_layernorm.weight]
Loading weights:  23%|██▎       | 91/398 [00:43<00:42,  7.28it/s, Materializing param=model.layers.8.mlp.down_proj.weight]  
Loading weights:  23%|██▎       | 91/398 [00:43<00:42,  7.28it/s, Materializing param=model.layers.8.mlp.down_proj.weight]
Loading weights:  23%|██▎       | 92/398 [00:43<00:42,  7.28it/s, Materializing param=model.layers.8.mlp.gate_proj.weight]
Loading weights:  23%|██▎       | 92/398 [00:43<00:42,  7.28it/s, Materializing param=model.layers.8.mlp.gate_proj.weight]
Loading weights:  23%|██▎       | 93/398 [00:43<00:33,  9.03it/s, Materializing param=model.layers.8.mlp.gate_proj.weight]
Loading weights:  23%|██▎       | 93/398 [00:43<00:33,  9.03it/s, Materializing param=model.layers.8.mlp.up_proj.weight]  
Loading weights:  23%|██▎       | 93/398 [00:43<00:33,  9.03it/s, Materializing param=model.layers.8.mlp.up_proj.weight]
Loading weights:  24%|██▎       | 94/398 [00:43<00:33,  9.03it/s, Materializing param=model.layers.8.post_attention_layernorm.weight]
Loading weights:  24%|██▎       | 94/398 [00:43<00:33,  9.03it/s, Materializing param=model.layers.8.post_attention_layernorm.weight]
Loading weights:  24%|██▍       | 95/398 [00:43<00:37,  8.15it/s, Materializing param=model.layers.8.post_attention_layernorm.weight]
Loading weights:  24%|██▍       | 95/398 [00:43<00:37,  8.15it/s, Materializing param=model.layers.8.self_attn.k_norm.weight]        
Loading weights:  24%|██▍       | 95/398 [00:43<00:37,  8.15it/s, Materializing param=model.layers.8.self_attn.k_norm.weight]
Loading weights:  24%|██▍       | 96/398 [00:43<00:37,  8.15it/s, Materializing param=model.layers.8.self_attn.k_proj.weight]
Loading weights:  24%|██▍       | 96/398 [00:43<00:37,  8.15it/s, Materializing param=model.layers.8.self_attn.k_proj.weight]
Loading weights:  24%|██▍       | 97/398 [00:43<00:31,  9.49it/s, Materializing param=model.layers.8.self_attn.k_proj.weight]
Loading weights:  24%|██▍       | 97/398 [00:43<00:31,  9.49it/s, Materializing param=model.layers.8.self_attn.o_proj.weight]
Loading weights:  24%|██▍       | 97/398 [00:43<00:31,  9.49it/s, Materializing param=model.layers.8.self_attn.o_proj.weight]
Loading weights:  25%|██▍       | 98/398 [00:44<00:31,  9.49it/s, Materializing param=model.layers.8.self_attn.q_norm.weight]
Loading weights:  25%|██▍       | 98/398 [00:44<00:31,  9.49it/s, Materializing param=model.layers.8.self_attn.q_norm.weight]
Loading weights:  25%|██▍       | 99/398 [00:44<00:44,  6.74it/s, Materializing param=model.layers.8.self_attn.q_norm.weight]
Loading weights:  25%|██▍       | 99/398 [00:44<00:44,  6.74it/s, Materializing param=model.layers.8.self_attn.q_proj.weight]
Loading weights:  25%|██▍       | 99/398 [00:44<00:44,  6.74it/s, Materializing param=model.layers.8.self_attn.q_proj.weight]
Loading weights:  25%|██▌       | 100/398 [00:44<00:44,  6.74it/s, Materializing param=model.layers.8.self_attn.v_proj.weight]
Loading weights:  25%|██▌       | 100/398 [00:44<00:44,  6.74it/s, Materializing param=model.layers.8.self_attn.v_proj.weight]
Loading weights:  25%|██▌       | 101/398 [00:44<00:44,  6.74it/s, Materializing param=model.layers.9.input_layernorm.weight] 
Loading weights:  25%|██▌       | 101/398 [00:44<00:44,  6.74it/s, Materializing param=model.layers.9.input_layernorm.weight]
Loading weights:  26%|██▌       | 102/398 [00:44<00:43,  6.74it/s, Materializing param=model.layers.9.mlp.down_proj.weight]  
Loading weights:  26%|██▌       | 102/398 [00:44<00:43,  6.74it/s, Materializing param=model.layers.9.mlp.down_proj.weight]
Loading weights:  26%|██▌       | 103/398 [00:44<00:28, 10.21it/s, Materializing param=model.layers.9.mlp.down_proj.weight]
Loading weights:  26%|██▌       | 103/398 [00:44<00:28, 10.21it/s, Materializing param=model.layers.9.mlp.gate_proj.weight]
Loading weights:  26%|██▌       | 103/398 [00:44<00:28, 10.21it/s, Materializing param=model.layers.9.mlp.gate_proj.weight]
Loading weights:  26%|██▌       | 104/398 [00:44<00:28, 10.21it/s, Materializing param=model.layers.9.mlp.up_proj.weight]  
Loading weights:  26%|██▌       | 104/398 [00:44<00:28, 10.21it/s, Materializing param=model.layers.9.mlp.up_proj.weight]
Loading weights:  26%|██▋       | 105/398 [00:45<00:58,  5.02it/s, Materializing param=model.layers.9.mlp.up_proj.weight]
Loading weights:  26%|██▋       | 105/398 [00:45<00:58,  5.02it/s, Materializing param=model.layers.9.post_attention_layernorm.weight]
Loading weights:  26%|██▋       | 105/398 [00:45<00:58,  5.02it/s, Materializing param=model.layers.9.post_attention_layernorm.weight]
Loading weights:  27%|██▋       | 106/398 [00:45<00:58,  5.02it/s, Materializing param=model.layers.9.self_attn.k_norm.weight]        
Loading weights:  27%|██▋       | 106/398 [00:45<00:58,  5.02it/s, Materializing param=model.layers.9.self_attn.k_norm.weight]
Loading weights:  27%|██▋       | 107/398 [00:45<00:57,  5.02it/s, Materializing param=model.layers.9.self_attn.k_proj.weight]
Loading weights:  27%|██▋       | 107/398 [00:45<00:57,  5.02it/s, Materializing param=model.layers.9.self_attn.k_proj.weight]
Loading weights:  27%|██▋       | 108/398 [00:45<00:57,  5.02it/s, Materializing param=model.layers.9.self_attn.o_proj.weight]
Loading weights:  27%|██▋       | 108/398 [00:45<00:57,  5.02it/s, Materializing param=model.layers.9.self_attn.o_proj.weight]
Loading weights:  27%|██▋       | 109/398 [00:45<00:43,  6.59it/s, Materializing param=model.layers.9.self_attn.o_proj.weight]
Loading weights:  27%|██▋       | 109/398 [00:45<00:43,  6.59it/s, Materializing param=model.layers.9.self_attn.q_norm.weight]
Loading weights:  27%|██▋       | 109/398 [00:45<00:43,  6.59it/s, Materializing param=model.layers.9.self_attn.q_norm.weight]
Loading weights:  28%|██▊       | 110/398 [00:45<00:43,  6.59it/s, Materializing param=model.layers.9.self_attn.q_proj.weight]
Loading weights:  28%|██▊       | 110/398 [00:45<00:43,  6.59it/s, Materializing param=model.layers.9.self_attn.q_proj.weight]
Loading weights:  28%|██▊       | 111/398 [00:45<00:40,  7.15it/s, Materializing param=model.layers.9.self_attn.q_proj.weight]
Loading weights:  28%|██▊       | 111/398 [00:45<00:40,  7.15it/s, Materializing param=model.layers.9.self_attn.v_proj.weight]
Loading weights:  28%|██▊       | 111/398 [00:45<00:40,  7.15it/s, Materializing param=model.layers.9.self_attn.v_proj.weight]
Loading weights:  28%|██▊       | 112/398 [00:45<00:39,  7.15it/s, Materializing param=model.layers.10.input_layernorm.weight]
Loading weights:  28%|██▊       | 112/398 [00:45<00:39,  7.15it/s, Materializing param=model.layers.10.input_layernorm.weight]
Loading weights:  28%|██▊       | 113/398 [00:45<00:39,  7.15it/s, Materializing param=model.layers.10.mlp.down_proj.weight]  
Loading weights:  28%|██▊       | 113/398 [00:45<00:39,  7.15it/s, Materializing param=model.layers.10.mlp.down_proj.weight]
Loading weights:  29%|██▊       | 114/398 [00:46<00:36,  7.86it/s, Materializing param=model.layers.10.mlp.down_proj.weight]
Loading weights:  29%|██▊       | 114/398 [00:46<00:36,  7.86it/s, Materializing param=model.layers.10.mlp.gate_proj.weight]
Loading weights:  29%|██▊       | 114/398 [00:46<00:36,  7.86it/s, Materializing param=model.layers.10.mlp.gate_proj.weight]
Loading weights:  29%|██▉       | 115/398 [00:46<00:36,  7.86it/s, Materializing param=model.layers.10.mlp.up_proj.weight]  
Loading weights:  29%|██▉       | 115/398 [00:46<00:36,  7.86it/s, Materializing param=model.layers.10.mlp.up_proj.weight]
Loading weights:  29%|██▉       | 116/398 [00:47<00:55,  5.13it/s, Materializing param=model.layers.10.mlp.up_proj.weight]
Loading weights:  29%|██▉       | 116/398 [00:47<00:55,  5.13it/s, Materializing param=model.layers.10.post_attention_layernorm.weight]
Loading weights:  29%|██▉       | 116/398 [00:47<00:55,  5.13it/s, Materializing param=model.layers.10.post_attention_layernorm.weight]
Loading weights:  29%|██▉       | 117/398 [00:47<00:54,  5.11it/s, Materializing param=model.layers.10.post_attention_layernorm.weight]
Loading weights:  29%|██▉       | 117/398 [00:47<00:54,  5.11it/s, Materializing param=model.layers.10.self_attn.k_norm.weight]        
Loading weights:  29%|██▉       | 117/398 [00:47<00:54,  5.11it/s, Materializing param=model.layers.10.self_attn.k_norm.weight]
Loading weights:  30%|██▉       | 118/398 [00:47<00:54,  5.11it/s, Materializing param=model.layers.10.self_attn.k_proj.weight]
Loading weights:  30%|██▉       | 118/398 [00:47<00:54,  5.11it/s, Materializing param=model.layers.10.self_attn.k_proj.weight]
Loading weights:  30%|██▉       | 119/398 [00:47<00:54,  5.11it/s, Materializing param=model.layers.10.self_attn.o_proj.weight]
Loading weights:  30%|██▉       | 119/398 [00:47<00:54,  5.11it/s, Materializing param=model.layers.10.self_attn.o_proj.weight]
Loading weights:  30%|███       | 120/398 [00:47<00:54,  5.11it/s, Materializing param=model.layers.10.self_attn.q_norm.weight]
Loading weights:  30%|███       | 120/398 [00:47<00:54,  5.11it/s, Materializing param=model.layers.10.self_attn.q_norm.weight]
Loading weights:  30%|███       | 121/398 [00:47<00:54,  5.11it/s, Materializing param=model.layers.10.self_attn.q_proj.weight]
Loading weights:  30%|███       | 121/398 [00:47<00:54,  5.11it/s, Materializing param=model.layers.10.self_attn.q_proj.weight]
Loading weights:  31%|███       | 122/398 [00:47<00:41,  6.67it/s, Materializing param=model.layers.10.self_attn.q_proj.weight]
Loading weights:  31%|███       | 122/398 [00:47<00:41,  6.67it/s, Materializing param=model.layers.10.self_attn.v_proj.weight]
Loading weights:  31%|███       | 122/398 [00:47<00:41,  6.67it/s, Materializing param=model.layers.10.self_attn.v_proj.weight]
Loading weights:  31%|███       | 123/398 [00:48<00:55,  4.96it/s, Materializing param=model.layers.10.self_attn.v_proj.weight]
Loading weights:  31%|███       | 123/398 [00:48<00:55,  4.96it/s, Materializing param=model.layers.11.input_layernorm.weight] 
Loading weights:  31%|███       | 123/398 [00:48<00:55,  4.96it/s, Materializing param=model.layers.11.input_layernorm.weight]
Loading weights:  31%|███       | 124/398 [00:48<00:55,  4.96it/s, Materializing param=model.layers.11.mlp.down_proj.weight]  
Loading weights:  31%|███       | 124/398 [00:48<00:55,  4.96it/s, Materializing param=model.layers.11.mlp.down_proj.weight]
Loading weights:  31%|███▏      | 125/398 [00:48<00:55,  4.96it/s, Materializing param=model.layers.11.mlp.gate_proj.weight]
Loading weights:  31%|███▏      | 125/398 [00:48<00:55,  4.96it/s, Materializing param=model.layers.11.mlp.gate_proj.weight]
Loading weights:  32%|███▏      | 126/398 [00:48<00:56,  4.79it/s, Materializing param=model.layers.11.mlp.gate_proj.weight]
Loading weights:  32%|███▏      | 126/398 [00:48<00:56,  4.79it/s, Materializing param=model.layers.11.mlp.up_proj.weight]  
Loading weights:  32%|███▏      | 126/398 [00:48<00:56,  4.79it/s, Materializing param=model.layers.11.mlp.up_proj.weight]
Loading weights:  32%|███▏      | 127/398 [00:49<00:54,  4.98it/s, Materializing param=model.layers.11.mlp.up_proj.weight]
Loading weights:  32%|███▏      | 127/398 [00:49<00:54,  4.98it/s, Materializing param=model.layers.11.post_attention_layernorm.weight]
Loading weights:  32%|███▏      | 127/398 [00:49<00:54,  4.98it/s, Materializing param=model.layers.11.post_attention_layernorm.weight]
Loading weights:  32%|███▏      | 128/398 [00:49<00:54,  4.98it/s, Materializing param=model.layers.11.self_attn.k_norm.weight]        
Loading weights:  32%|███▏      | 128/398 [00:49<00:54,  4.98it/s, Materializing param=model.layers.11.self_attn.k_norm.weight]
Loading weights:  32%|███▏      | 129/398 [00:49<00:53,  4.98it/s, Materializing param=model.layers.11.self_attn.k_proj.weight]
Loading weights:  32%|███▏      | 129/398 [00:49<00:53,  4.98it/s, Materializing param=model.layers.11.self_attn.k_proj.weight]
Loading weights:  33%|███▎      | 130/398 [00:49<00:53,  4.98it/s, Materializing param=model.layers.11.self_attn.o_proj.weight]
Loading weights:  33%|███▎      | 130/398 [00:49<00:53,  4.98it/s, Materializing param=model.layers.11.self_attn.o_proj.weight]
Loading weights:  33%|███▎      | 131/398 [00:49<00:53,  4.98it/s, Materializing param=model.layers.11.self_attn.q_norm.weight]
Loading weights:  33%|███▎      | 131/398 [00:49<00:53,  4.98it/s, Materializing param=model.layers.11.self_attn.q_norm.weight]
Loading weights:  33%|███▎      | 132/398 [00:49<00:33,  8.06it/s, Materializing param=model.layers.11.self_attn.q_norm.weight]
Loading weights:  33%|███▎      | 132/398 [00:49<00:33,  8.06it/s, Materializing param=model.layers.11.self_attn.q_proj.weight]
Loading weights:  33%|███▎      | 132/398 [00:49<00:33,  8.06it/s, Materializing param=model.layers.11.self_attn.q_proj.weight]
Loading weights:  33%|███▎      | 133/398 [00:49<00:32,  8.06it/s, Materializing param=model.layers.11.self_attn.v_proj.weight]
Loading weights:  33%|███▎      | 133/398 [00:49<00:32,  8.06it/s, Materializing param=model.layers.11.self_attn.v_proj.weight]
Loading weights:  34%|███▎      | 134/398 [00:49<00:32,  8.21it/s, Materializing param=model.layers.11.self_attn.v_proj.weight]
Loading weights:  34%|███▎      | 134/398 [00:49<00:32,  8.21it/s, Materializing param=model.layers.12.input_layernorm.weight] 
Loading weights:  34%|███▎      | 134/398 [00:49<00:32,  8.21it/s, Materializing param=model.layers.12.input_layernorm.weight]
Loading weights:  34%|███▍      | 135/398 [00:49<00:32,  8.21it/s, Materializing param=model.layers.12.mlp.down_proj.weight]  
Loading weights:  34%|███▍      | 135/398 [00:49<00:32,  8.21it/s, Materializing param=model.layers.12.mlp.down_proj.weight]
Loading weights:  34%|███▍      | 136/398 [00:49<00:31,  8.21it/s, Materializing param=model.layers.12.mlp.gate_proj.weight]
Loading weights:  34%|███▍      | 136/398 [00:49<00:31,  8.21it/s, Materializing param=model.layers.12.mlp.gate_proj.weight]
Loading weights:  34%|███▍      | 137/398 [00:50<00:37,  6.98it/s, Materializing param=model.layers.12.mlp.gate_proj.weight]
Loading weights:  34%|███▍      | 137/398 [00:50<00:37,  6.98it/s, Materializing param=model.layers.12.mlp.up_proj.weight]  
Loading weights:  34%|███▍      | 137/398 [00:50<00:37,  6.98it/s, Materializing param=model.layers.12.mlp.up_proj.weight]
Loading weights:  35%|███▍      | 138/398 [00:50<00:37,  6.98it/s, Materializing param=model.layers.12.post_attention_layernorm.weight]
Loading weights:  35%|███▍      | 138/398 [00:50<00:37,  6.98it/s, Materializing param=model.layers.12.post_attention_layernorm.weight]
Loading weights:  35%|███▍      | 139/398 [00:50<00:37,  6.98it/s, Materializing param=model.layers.12.self_attn.k_norm.weight]        
Loading weights:  35%|███▍      | 139/398 [00:50<00:37,  6.98it/s, Materializing param=model.layers.12.self_attn.k_norm.weight]
Loading weights:  35%|███▌      | 140/398 [00:50<00:36,  6.98it/s, Materializing param=model.layers.12.self_attn.k_proj.weight]
Loading weights:  35%|███▌      | 140/398 [00:50<00:36,  6.98it/s, Materializing param=model.layers.12.self_attn.k_proj.weight]
Loading weights:  35%|███▌      | 141/398 [00:50<00:37,  6.78it/s, Materializing param=model.layers.12.self_attn.k_proj.weight]
Loading weights:  35%|███▌      | 141/398 [00:50<00:37,  6.78it/s, Materializing param=model.layers.12.self_attn.o_proj.weight]
Loading weights:  35%|███▌      | 141/398 [00:50<00:37,  6.78it/s, Materializing param=model.layers.12.self_attn.o_proj.weight]
Loading weights:  36%|███▌      | 142/398 [00:50<00:37,  6.78it/s, Materializing param=model.layers.12.self_attn.q_norm.weight]
Loading weights:  36%|███▌      | 142/398 [00:50<00:37,  6.78it/s, Materializing param=model.layers.12.self_attn.q_norm.weight]
Loading weights:  36%|███▌      | 143/398 [00:51<00:36,  7.07it/s, Materializing param=model.layers.12.self_attn.q_norm.weight]
Loading weights:  36%|███▌      | 143/398 [00:51<00:36,  7.07it/s, Materializing param=model.layers.12.self_attn.q_proj.weight]
Loading weights:  36%|███▌      | 143/398 [00:51<00:36,  7.07it/s, Materializing param=model.layers.12.self_attn.q_proj.weight]
Loading weights:  36%|███▌      | 144/398 [00:51<00:35,  7.07it/s, Materializing param=model.layers.12.self_attn.v_proj.weight]
Loading weights:  36%|███▌      | 144/398 [00:51<00:35,  7.07it/s, Materializing param=model.layers.12.self_attn.v_proj.weight]
Loading weights:  36%|███▋      | 145/398 [00:51<00:35,  7.07it/s, Materializing param=model.layers.13.input_layernorm.weight] 
Loading weights:  36%|███▋      | 145/398 [00:51<00:35,  7.07it/s, Materializing param=model.layers.13.input_layernorm.weight]
Loading weights:  37%|███▋      | 146/398 [00:51<00:35,  7.07it/s, Materializing param=model.layers.13.mlp.down_proj.weight]  
Loading weights:  37%|███▋      | 146/398 [00:51<00:35,  7.07it/s, Materializing param=model.layers.13.mlp.down_proj.weight]
Loading weights:  37%|███▋      | 147/398 [00:51<00:30,  8.34it/s, Materializing param=model.layers.13.mlp.down_proj.weight]
Loading weights:  37%|███▋      | 147/398 [00:51<00:30,  8.34it/s, Materializing param=model.layers.13.mlp.gate_proj.weight]
Loading weights:  37%|███▋      | 147/398 [00:51<00:30,  8.34it/s, Materializing param=model.layers.13.mlp.gate_proj.weight]
Loading weights:  37%|███▋      | 148/398 [00:52<01:01,  4.09it/s, Materializing param=model.layers.13.mlp.gate_proj.weight]
Loading weights:  37%|███▋      | 148/398 [00:52<01:01,  4.09it/s, Materializing param=model.layers.13.mlp.up_proj.weight]  
Loading weights:  37%|███▋      | 148/398 [00:52<01:01,  4.09it/s, Materializing param=model.layers.13.mlp.up_proj.weight]
Loading weights:  37%|███▋      | 149/398 [00:52<01:00,  4.09it/s, Materializing param=model.layers.13.post_attention_layernorm.weight]
Loading weights:  37%|███▋      | 149/398 [00:52<01:00,  4.09it/s, Materializing param=model.layers.13.post_attention_layernorm.weight]
Loading weights:  38%|███▊      | 150/398 [00:52<01:00,  4.09it/s, Materializing param=model.layers.13.self_attn.k_norm.weight]        
Loading weights:  38%|███▊      | 150/398 [00:52<01:00,  4.09it/s, Materializing param=model.layers.13.self_attn.k_norm.weight]
Loading weights:  38%|███▊      | 151/398 [00:52<01:00,  4.09it/s, Materializing param=model.layers.13.self_attn.k_proj.weight]
Loading weights:  38%|███▊      | 151/398 [00:52<01:00,  4.09it/s, Materializing param=model.layers.13.self_attn.k_proj.weight]
Loading weights:  38%|███▊      | 152/398 [00:52<01:00,  4.09it/s, Materializing param=model.layers.13.self_attn.o_proj.weight]
Loading weights:  38%|███▊      | 152/398 [00:52<01:00,  4.09it/s, Materializing param=model.layers.13.self_attn.o_proj.weight]
Loading weights:  38%|███▊      | 153/398 [00:52<00:35,  6.95it/s, Materializing param=model.layers.13.self_attn.o_proj.weight]
Loading weights:  38%|███▊      | 153/398 [00:52<00:35,  6.95it/s, Materializing param=model.layers.13.self_attn.q_norm.weight]
Loading weights:  38%|███▊      | 153/398 [00:52<00:35,  6.95it/s, Materializing param=model.layers.13.self_attn.q_norm.weight]
Loading weights:  39%|███▊      | 154/398 [00:52<00:35,  6.95it/s, Materializing param=model.layers.13.self_attn.q_proj.weight]
Loading weights:  39%|███▊      | 154/398 [00:52<00:35,  6.95it/s, Materializing param=model.layers.13.self_attn.q_proj.weight]
Loading weights:  39%|███▉      | 155/398 [00:53<00:46,  5.18it/s, Materializing param=model.layers.13.self_attn.q_proj.weight]
Loading weights:  39%|███▉      | 155/398 [00:53<00:46,  5.18it/s, Materializing param=model.layers.13.self_attn.v_proj.weight]
Loading weights:  39%|███▉      | 155/398 [00:53<00:46,  5.18it/s, Materializing param=model.layers.13.self_attn.v_proj.weight]
Loading weights:  39%|███▉      | 156/398 [00:53<00:46,  5.18it/s, Materializing param=model.layers.14.input_layernorm.weight] 
Loading weights:  39%|███▉      | 156/398 [00:53<00:46,  5.18it/s, Materializing param=model.layers.14.input_layernorm.weight]
Loading weights:  39%|███▉      | 157/398 [00:53<00:42,  5.70it/s, Materializing param=model.layers.14.input_layernorm.weight]
Loading weights:  39%|███▉      | 157/398 [00:53<00:42,  5.70it/s, Materializing param=model.layers.14.mlp.down_proj.weight]  
Loading weights:  39%|███▉      | 157/398 [00:53<00:42,  5.70it/s, Materializing param=model.layers.14.mlp.down_proj.weight]
Loading weights:  40%|███▉      | 158/398 [00:53<00:42,  5.70it/s, Materializing param=model.layers.14.mlp.gate_proj.weight]
Loading weights:  40%|███▉      | 158/398 [00:53<00:42,  5.70it/s, Materializing param=model.layers.14.mlp.gate_proj.weight]
Loading weights:  40%|███▉      | 159/398 [00:54<01:01,  3.88it/s, Materializing param=model.layers.14.mlp.gate_proj.weight]
Loading weights:  40%|███▉      | 159/398 [00:54<01:01,  3.88it/s, Materializing param=model.layers.14.mlp.up_proj.weight]  
Loading weights:  40%|███▉      | 159/398 [00:54<01:01,  3.88it/s, Materializing param=model.layers.14.mlp.up_proj.weight]
Loading weights:  40%|████      | 160/398 [00:54<01:01,  3.88it/s, Materializing param=model.layers.14.post_attention_layernorm.weight]
Loading weights:  40%|████      | 160/398 [00:54<01:01,  3.88it/s, Materializing param=model.layers.14.post_attention_layernorm.weight]
Loading weights:  40%|████      | 161/398 [00:54<01:01,  3.88it/s, Materializing param=model.layers.14.self_attn.k_norm.weight]        
Loading weights:  40%|████      | 161/398 [00:54<01:01,  3.88it/s, Materializing param=model.layers.14.self_attn.k_norm.weight]
Loading weights:  41%|████      | 162/398 [00:54<01:00,  3.88it/s, Materializing param=model.layers.14.self_attn.k_proj.weight]
Loading weights:  41%|████      | 162/398 [00:54<01:00,  3.88it/s, Materializing param=model.layers.14.self_attn.k_proj.weight]
Loading weights:  41%|████      | 163/398 [00:54<01:00,  3.88it/s, Materializing param=model.layers.14.self_attn.o_proj.weight]
Loading weights:  41%|████      | 163/398 [00:54<01:00,  3.88it/s, Materializing param=model.layers.14.self_attn.o_proj.weight]
Loading weights:  41%|████      | 164/398 [00:54<01:00,  3.88it/s, Materializing param=model.layers.14.self_attn.q_norm.weight]
Loading weights:  41%|████      | 164/398 [00:54<01:00,  3.88it/s, Materializing param=model.layers.14.self_attn.q_norm.weight]
Loading weights:  41%|████▏     | 165/398 [00:55<00:37,  6.29it/s, Materializing param=model.layers.14.self_attn.q_norm.weight]
Loading weights:  41%|████▏     | 165/398 [00:55<00:37,  6.29it/s, Materializing param=model.layers.14.self_attn.q_proj.weight]
Loading weights:  41%|████▏     | 165/398 [00:55<00:37,  6.29it/s, Materializing param=model.layers.14.self_attn.q_proj.weight]
Loading weights:  42%|████▏     | 166/398 [00:55<00:36,  6.29it/s, Materializing param=model.layers.14.self_attn.v_proj.weight]
Loading weights:  42%|████▏     | 166/398 [00:55<00:36,  6.29it/s, Materializing param=model.layers.14.self_attn.v_proj.weight]
Loading weights:  42%|████▏     | 167/398 [00:55<00:33,  6.91it/s, Materializing param=model.layers.14.self_attn.v_proj.weight]
Loading weights:  42%|████▏     | 167/398 [00:55<00:33,  6.91it/s, Materializing param=model.layers.15.input_layernorm.weight] 
Loading weights:  42%|████▏     | 167/398 [00:55<00:33,  6.91it/s, Materializing param=model.layers.15.input_layernorm.weight]
Loading weights:  42%|████▏     | 168/398 [00:55<00:33,  6.91it/s, Materializing param=model.layers.15.mlp.down_proj.weight]  
Loading weights:  42%|████▏     | 168/398 [00:55<00:33,  6.91it/s, Materializing param=model.layers.15.mlp.down_proj.weight]
Loading weights:  42%|████▏     | 169/398 [00:56<01:00,  3.77it/s, Materializing param=model.layers.15.mlp.down_proj.weight]
Loading weights:  42%|████▏     | 169/398 [00:56<01:00,  3.77it/s, Materializing param=model.layers.15.mlp.gate_proj.weight]
Loading weights:  42%|████▏     | 169/398 [00:56<01:00,  3.77it/s, Materializing param=model.layers.15.mlp.gate_proj.weight]
Loading weights:  43%|████▎     | 170/398 [00:57<01:06,  3.45it/s, Materializing param=model.layers.15.mlp.gate_proj.weight]
Loading weights:  43%|████▎     | 170/398 [00:57<01:06,  3.45it/s, Materializing param=model.layers.15.mlp.up_proj.weight]  
Loading weights:  43%|████▎     | 170/398 [00:57<01:06,  3.45it/s, Materializing param=model.layers.15.mlp.up_proj.weight]
Loading weights:  43%|████▎     | 171/398 [00:57<01:05,  3.45it/s, Materializing param=model.layers.15.post_attention_layernorm.weight]
Loading weights:  43%|████▎     | 171/398 [00:57<01:05,  3.45it/s, Materializing param=model.layers.15.post_attention_layernorm.weight]
Loading weights:  43%|████▎     | 172/398 [00:57<01:05,  3.45it/s, Materializing param=model.layers.15.self_attn.k_norm.weight]        
Loading weights:  43%|████▎     | 172/398 [00:57<01:05,  3.45it/s, Materializing param=model.layers.15.self_attn.k_norm.weight]
Loading weights:  43%|████▎     | 173/398 [00:57<01:05,  3.45it/s, Materializing param=model.layers.15.self_attn.k_proj.weight]
Loading weights:  43%|████▎     | 173/398 [00:57<01:05,  3.45it/s, Materializing param=model.layers.15.self_attn.k_proj.weight]
Loading weights:  44%|████▎     | 174/398 [00:57<01:05,  3.45it/s, Materializing param=model.layers.15.self_attn.o_proj.weight]
Loading weights:  44%|████▎     | 174/398 [00:57<01:05,  3.45it/s, Materializing param=model.layers.15.self_attn.o_proj.weight]
Loading weights:  44%|████▍     | 175/398 [00:58<00:56,  3.95it/s, Materializing param=model.layers.15.self_attn.o_proj.weight]
Loading weights:  44%|████▍     | 175/398 [00:58<00:56,  3.95it/s, Materializing param=model.layers.15.self_attn.q_norm.weight]
Loading weights:  44%|████▍     | 175/398 [00:58<00:56,  3.95it/s, Materializing param=model.layers.15.self_attn.q_norm.weight]
Loading weights:  44%|████▍     | 176/398 [00:58<00:56,  3.95it/s, Materializing param=model.layers.15.self_attn.q_proj.weight]
Loading weights:  44%|████▍     | 176/398 [00:58<00:56,  3.95it/s, Materializing param=model.layers.15.self_attn.q_proj.weight]
Loading weights:  44%|████▍     | 177/398 [00:58<00:56,  3.95it/s, Materializing param=model.layers.15.self_attn.v_proj.weight]
Loading weights:  44%|████▍     | 177/398 [00:58<00:56,  3.95it/s, Materializing param=model.layers.15.self_attn.v_proj.weight]
Loading weights:  45%|████▍     | 178/398 [00:58<00:45,  4.82it/s, Materializing param=model.layers.15.self_attn.v_proj.weight]
Loading weights:  45%|████▍     | 178/398 [00:58<00:45,  4.82it/s, Materializing param=model.layers.16.input_layernorm.weight] 
Loading weights:  45%|████▍     | 178/398 [00:58<00:45,  4.82it/s, Materializing param=model.layers.16.input_layernorm.weight]
Loading weights:  45%|████▍     | 179/398 [00:58<00:45,  4.82it/s, Materializing param=model.layers.16.mlp.down_proj.weight]  
Loading weights:  45%|████▍     | 179/398 [00:58<00:45,  4.82it/s, Materializing param=model.layers.16.mlp.down_proj.weight]
Loading weights:  45%|████▌     | 180/398 [00:58<00:45,  4.82it/s, Materializing param=model.layers.16.mlp.gate_proj.weight]
Loading weights:  45%|████▌     | 180/398 [00:58<00:45,  4.82it/s, Materializing param=model.layers.16.mlp.gate_proj.weight]
Loading weights:  45%|████▌     | 181/398 [00:58<00:40,  5.42it/s, Materializing param=model.layers.16.mlp.gate_proj.weight]
Loading weights:  45%|████▌     | 181/398 [00:58<00:40,  5.42it/s, Materializing param=model.layers.16.mlp.up_proj.weight]  
Loading weights:  45%|████▌     | 181/398 [00:58<00:40,  5.42it/s, Materializing param=model.layers.16.mlp.up_proj.weight]
Loading weights:  46%|████▌     | 182/398 [00:59<00:50,  4.27it/s, Materializing param=model.layers.16.mlp.up_proj.weight]
Loading weights:  46%|████▌     | 182/398 [00:59<00:50,  4.27it/s, Materializing param=model.layers.16.post_attention_layernorm.weight]
Loading weights:  46%|████▌     | 182/398 [00:59<00:50,  4.27it/s, Materializing param=model.layers.16.post_attention_layernorm.weight]
Loading weights:  46%|████▌     | 183/398 [00:59<00:46,  4.64it/s, Materializing param=model.layers.16.post_attention_layernorm.weight]
Loading weights:  46%|████▌     | 183/398 [00:59<00:46,  4.64it/s, Materializing param=model.layers.16.self_attn.k_norm.weight]        
Loading weights:  46%|████▌     | 183/398 [00:59<00:46,  4.64it/s, Materializing param=model.layers.16.self_attn.k_norm.weight]
Loading weights:  46%|████▌     | 184/398 [00:59<00:46,  4.64it/s, Materializing param=model.layers.16.self_attn.k_proj.weight]
Loading weights:  46%|████▌     | 184/398 [00:59<00:46,  4.64it/s, Materializing param=model.layers.16.self_attn.k_proj.weight]
Loading weights:  46%|████▋     | 185/398 [00:59<00:45,  4.64it/s, Materializing param=model.layers.16.self_attn.o_proj.weight]
Loading weights:  46%|████▋     | 185/398 [00:59<00:45,  4.64it/s, Materializing param=model.layers.16.self_attn.o_proj.weight]
Loading weights:  47%|████▋     | 186/398 [00:59<00:35,  5.99it/s, Materializing param=model.layers.16.self_attn.o_proj.weight]
Loading weights:  47%|████▋     | 186/398 [00:59<00:35,  5.99it/s, Materializing param=model.layers.16.self_attn.q_norm.weight]
Loading weights:  47%|████▋     | 186/398 [00:59<00:35,  5.99it/s, Materializing param=model.layers.16.self_attn.q_norm.weight]
Loading weights:  47%|████▋     | 187/398 [00:59<00:34,  6.05it/s, Materializing param=model.layers.16.self_attn.q_norm.weight]
Loading weights:  47%|████▋     | 187/398 [00:59<00:34,  6.05it/s, Materializing param=model.layers.16.self_attn.q_proj.weight]
Loading weights:  47%|████▋     | 187/398 [00:59<00:34,  6.05it/s, Materializing param=model.layers.16.self_attn.q_proj.weight]
Loading weights:  47%|████▋     | 188/398 [00:59<00:34,  6.05it/s, Materializing param=model.layers.16.self_attn.v_proj.weight]
Loading weights:  47%|████▋     | 188/398 [01:00<00:34,  6.05it/s, Materializing param=model.layers.16.self_attn.v_proj.weight]
Loading weights:  47%|████▋     | 189/398 [01:00<00:30,  6.80it/s, Materializing param=model.layers.16.self_attn.v_proj.weight]
Loading weights:  47%|████▋     | 189/398 [01:00<00:30,  6.80it/s, Materializing param=model.layers.17.input_layernorm.weight] 
Loading weights:  47%|████▋     | 189/398 [01:00<00:30,  6.80it/s, Materializing param=model.layers.17.input_layernorm.weight]
Loading weights:  48%|████▊     | 190/398 [01:00<00:30,  6.80it/s, Materializing param=model.layers.17.mlp.down_proj.weight]  
Loading weights:  48%|████▊     | 190/398 [01:00<00:30,  6.80it/s, Materializing param=model.layers.17.mlp.down_proj.weight]
Loading weights:  48%|████▊     | 191/398 [01:00<00:29,  7.12it/s, Materializing param=model.layers.17.mlp.down_proj.weight]
Loading weights:  48%|████▊     | 191/398 [01:00<00:29,  7.12it/s, Materializing param=model.layers.17.mlp.gate_proj.weight]
Loading weights:  48%|████▊     | 191/398 [01:00<00:29,  7.12it/s, Materializing param=model.layers.17.mlp.gate_proj.weight]
Loading weights:  48%|████▊     | 192/398 [01:01<00:49,  4.13it/s, Materializing param=model.layers.17.mlp.gate_proj.weight]
Loading weights:  48%|████▊     | 192/398 [01:01<00:49,  4.13it/s, Materializing param=model.layers.17.mlp.up_proj.weight]  
Loading weights:  48%|████▊     | 192/398 [01:01<00:49,  4.13it/s, Materializing param=model.layers.17.mlp.up_proj.weight]
Loading weights:  48%|████▊     | 193/398 [01:01<01:11,  2.85it/s, Materializing param=model.layers.17.mlp.up_proj.weight]
Loading weights:  48%|████▊     | 193/398 [01:01<01:11,  2.85it/s, Materializing param=model.layers.17.post_attention_layernorm.weight]
Loading weights:  48%|████▊     | 193/398 [01:01<01:11,  2.85it/s, Materializing param=model.layers.17.post_attention_layernorm.weight]
Loading weights:  49%|████▊     | 194/398 [01:01<01:11,  2.85it/s, Materializing param=model.layers.17.self_attn.k_norm.weight]        
Loading weights:  49%|████▊     | 194/398 [01:01<01:11,  2.85it/s, Materializing param=model.layers.17.self_attn.k_norm.weight]
Loading weights:  49%|████▉     | 195/398 [01:01<01:11,  2.85it/s, Materializing param=model.layers.17.self_attn.k_proj.weight]
Loading weights:  49%|████▉     | 195/398 [01:01<01:11,  2.85it/s, Materializing param=model.layers.17.self_attn.k_proj.weight]
Loading weights:  49%|████▉     | 196/398 [01:01<01:10,  2.85it/s, Materializing param=model.layers.17.self_attn.o_proj.weight]
Loading weights:  49%|████▉     | 196/398 [01:01<01:10,  2.85it/s, Materializing param=model.layers.17.self_attn.o_proj.weight]
Loading weights:  49%|████▉     | 197/398 [01:02<00:43,  4.60it/s, Materializing param=model.layers.17.self_attn.o_proj.weight]
Loading weights:  49%|████▉     | 197/398 [01:02<00:43,  4.60it/s, Materializing param=model.layers.17.self_attn.q_norm.weight]
Loading weights:  49%|████▉     | 197/398 [01:02<00:43,  4.60it/s, Materializing param=model.layers.17.self_attn.q_norm.weight]
Loading weights:  50%|████▉     | 198/398 [01:02<00:43,  4.60it/s, Materializing param=model.layers.17.self_attn.q_proj.weight]
Loading weights:  50%|████▉     | 198/398 [01:02<00:43,  4.60it/s, Materializing param=model.layers.17.self_attn.q_proj.weight]
Loading weights:  50%|█████     | 199/398 [01:02<00:48,  4.12it/s, Materializing param=model.layers.17.self_attn.q_proj.weight]
Loading weights:  50%|█████     | 199/398 [01:03<00:48,  4.12it/s, Materializing param=model.layers.17.self_attn.v_proj.weight]
Loading weights:  50%|█████     | 199/398 [01:03<00:48,  4.12it/s, Materializing param=model.layers.17.self_attn.v_proj.weight]
Loading weights:  50%|█████     | 200/398 [01:03<00:44,  4.48it/s, Materializing param=model.layers.17.self_attn.v_proj.weight]
Loading weights:  50%|█████     | 200/398 [01:03<00:44,  4.48it/s, Materializing param=model.layers.18.input_layernorm.weight] 
Loading weights:  50%|█████     | 200/398 [01:03<00:44,  4.48it/s, Materializing param=model.layers.18.input_layernorm.weight]
Loading weights:  51%|█████     | 201/398 [01:03<00:43,  4.48it/s, Materializing param=model.layers.18.mlp.down_proj.weight]  
Loading weights:  51%|█████     | 201/398 [01:03<00:43,  4.48it/s, Materializing param=model.layers.18.mlp.down_proj.weight]
Loading weights:  51%|█████     | 202/398 [01:03<00:41,  4.73it/s, Materializing param=model.layers.18.mlp.down_proj.weight]
Loading weights:  51%|█████     | 202/398 [01:03<00:41,  4.73it/s, Materializing param=model.layers.18.mlp.gate_proj.weight]
Loading weights:  51%|█████     | 202/398 [01:03<00:41,  4.73it/s, Materializing param=model.layers.18.mlp.gate_proj.weight]
Loading weights:  51%|█████     | 203/398 [01:03<00:43,  4.53it/s, Materializing param=model.layers.18.mlp.gate_proj.weight]
Loading weights:  51%|█████     | 203/398 [01:03<00:43,  4.53it/s, Materializing param=model.layers.18.mlp.up_proj.weight]  
Loading weights:  51%|█████     | 203/398 [01:03<00:43,  4.53it/s, Materializing param=model.layers.18.mlp.up_proj.weight]
Loading weights:  51%|█████▏    | 204/398 [01:04<01:08,  2.83it/s, Materializing param=model.layers.18.mlp.up_proj.weight]
Loading weights:  51%|█████▏    | 204/398 [01:04<01:08,  2.83it/s, Materializing param=model.layers.18.post_attention_layernorm.weight]
Loading weights:  51%|█████▏    | 204/398 [01:04<01:08,  2.83it/s, Materializing param=model.layers.18.post_attention_layernorm.weight]
Loading weights:  52%|█████▏    | 205/398 [01:04<01:08,  2.83it/s, Materializing param=model.layers.18.self_attn.k_norm.weight]        
Loading weights:  52%|█████▏    | 205/398 [01:04<01:08,  2.83it/s, Materializing param=model.layers.18.self_attn.k_norm.weight]
Loading weights:  52%|█████▏    | 206/398 [01:04<00:46,  4.15it/s, Materializing param=model.layers.18.self_attn.k_norm.weight]
Loading weights:  52%|█████▏    | 206/398 [01:04<00:46,  4.15it/s, Materializing param=model.layers.18.self_attn.k_proj.weight]
Loading weights:  52%|█████▏    | 206/398 [01:04<00:46,  4.15it/s, Materializing param=model.layers.18.self_attn.k_proj.weight]
Loading weights:  52%|█████▏    | 207/398 [01:04<00:45,  4.15it/s, Materializing param=model.layers.18.self_attn.o_proj.weight]
Loading weights:  52%|█████▏    | 207/398 [01:04<00:45,  4.15it/s, Materializing param=model.layers.18.self_attn.o_proj.weight]
Loading weights:  52%|█████▏    | 208/398 [01:04<00:38,  4.96it/s, Materializing param=model.layers.18.self_attn.o_proj.weight]
Loading weights:  52%|█████▏    | 208/398 [01:04<00:38,  4.96it/s, Materializing param=model.layers.18.self_attn.q_norm.weight]
Loading weights:  52%|█████▏    | 208/398 [01:04<00:38,  4.96it/s, Materializing param=model.layers.18.self_attn.q_norm.weight]
Loading weights:  53%|█████▎    | 209/398 [01:04<00:38,  4.96it/s, Materializing param=model.layers.18.self_attn.q_proj.weight]
Loading weights:  53%|█████▎    | 209/398 [01:04<00:38,  4.96it/s, Materializing param=model.layers.18.self_attn.q_proj.weight]
Loading weights:  53%|█████▎    | 210/398 [01:05<00:34,  5.50it/s, Materializing param=model.layers.18.self_attn.q_proj.weight]
Loading weights:  53%|█████▎    | 210/398 [01:05<00:34,  5.50it/s, Materializing param=model.layers.18.self_attn.v_proj.weight]
Loading weights:  53%|█████▎    | 210/398 [01:05<00:34,  5.50it/s, Materializing param=model.layers.18.self_attn.v_proj.weight]
Loading weights:  53%|█████▎    | 211/398 [01:05<00:34,  5.50it/s, Materializing param=model.layers.19.input_layernorm.weight] 
Loading weights:  53%|█████▎    | 211/398 [01:05<00:34,  5.50it/s, Materializing param=model.layers.19.input_layernorm.weight]
Loading weights:  53%|█████▎    | 212/398 [01:05<00:37,  4.96it/s, Materializing param=model.layers.19.input_layernorm.weight]
Loading weights:  53%|█████▎    | 212/398 [01:05<00:37,  4.96it/s, Materializing param=model.layers.19.mlp.down_proj.weight]  
Loading weights:  53%|█████▎    | 212/398 [01:05<00:37,  4.96it/s, Materializing param=model.layers.19.mlp.down_proj.weight]
Loading weights:  54%|█████▎    | 213/398 [01:05<00:37,  4.96it/s, Materializing param=model.layers.19.mlp.gate_proj.weight]
Loading weights:  54%|█████▎    | 213/398 [01:05<00:37,  4.96it/s, Materializing param=model.layers.19.mlp.gate_proj.weight]
Loading weights:  54%|█████▍    | 214/398 [01:05<00:32,  5.72it/s, Materializing param=model.layers.19.mlp.gate_proj.weight]
Loading weights:  54%|█████▍    | 214/398 [01:05<00:32,  5.72it/s, Materializing param=model.layers.19.mlp.up_proj.weight]  
Loading weights:  54%|█████▍    | 214/398 [01:05<00:32,  5.72it/s, Materializing param=model.layers.19.mlp.up_proj.weight]
Loading weights:  54%|█████▍    | 215/398 [01:06<00:49,  3.71it/s, Materializing param=model.layers.19.mlp.up_proj.weight]
Loading weights:  54%|█████▍    | 215/398 [01:06<00:49,  3.71it/s, Materializing param=model.layers.19.post_attention_layernorm.weight]
Loading weights:  54%|█████▍    | 215/398 [01:06<00:49,  3.71it/s, Materializing param=model.layers.19.post_attention_layernorm.weight]
Loading weights:  54%|█████▍    | 216/398 [01:07<00:53,  3.40it/s, Materializing param=model.layers.19.post_attention_layernorm.weight]
Loading weights:  54%|█████▍    | 216/398 [01:07<00:53,  3.40it/s, Materializing param=model.layers.19.self_attn.k_norm.weight]        
Loading weights:  54%|█████▍    | 216/398 [01:07<00:53,  3.40it/s, Materializing param=model.layers.19.self_attn.k_norm.weight]
Loading weights:  55%|█████▍    | 217/398 [01:07<00:53,  3.40it/s, Materializing param=model.layers.19.self_attn.k_proj.weight]
Loading weights:  55%|█████▍    | 217/398 [01:07<00:53,  3.40it/s, Materializing param=model.layers.19.self_attn.k_proj.weight]
Loading weights:  55%|█████▍    | 218/398 [01:07<00:52,  3.40it/s, Materializing param=model.layers.19.self_attn.o_proj.weight]
Loading weights:  55%|█████▍    | 218/398 [01:07<00:52,  3.40it/s, Materializing param=model.layers.19.self_attn.o_proj.weight]
Loading weights:  55%|█████▌    | 219/398 [01:07<00:34,  5.15it/s, Materializing param=model.layers.19.self_attn.o_proj.weight]
Loading weights:  55%|█████▌    | 219/398 [01:07<00:34,  5.15it/s, Materializing param=model.layers.19.self_attn.q_norm.weight]
Loading weights:  55%|██��██▌    | 219/398 [01:07<00:34,  5.15it/s, Materializing param=model.layers.19.self_attn.q_norm.weight]
Loading weights:  55%|█████▌    | 220/398 [01:07<00:38,  4.67it/s, Materializing param=model.layers.19.self_attn.q_norm.weight]
Loading weights:  55%|█████▌    | 220/398 [01:07<00:38,  4.67it/s, Materializing param=model.layers.19.self_attn.q_proj.weight]
Loading weights:  55%|█████▌    | 220/398 [01:07<00:38,  4.67it/s, Materializing param=model.layers.19.self_attn.q_proj.weight]
Loading weights:  56%|█████▌    | 221/398 [01:07<00:37,  4.67it/s, Materializing param=model.layers.19.self_attn.v_proj.weight]
Loading weights:  56%|█████▌    | 221/398 [01:07<00:37,  4.67it/s, Materializing param=model.layers.19.self_attn.v_proj.weight]
Loading weights:  56%|█████▌    | 222/398 [01:07<00:31,  5.56it/s, Materializing param=model.layers.19.self_attn.v_proj.weight]
Loading weights:  56%|█████▌    | 222/398 [01:07<00:31,  5.56it/s, Materializing param=model.layers.20.input_layernorm.weight] 
Loading weights:  56%|█████▌    | 222/398 [01:07<00:31,  5.56it/s, Materializing param=model.layers.20.input_layernorm.weight]
Loading weights:  56%|█████▌    | 223/398 [01:09<01:10,  2.48it/s, Materializing param=model.layers.20.input_layernorm.weight]
Loading weights:  56%|█████▌    | 223/398 [01:09<01:10,  2.48it/s, Materializing param=model.layers.20.mlp.down_proj.weight]  
Loading weights:  56%|█████▌    | 223/398 [01:09<01:10,  2.48it/s, Materializing param=model.layers.20.mlp.down_proj.weight]
Loading weights:  56%|█████▋    | 224/398 [01:09<01:10,  2.48it/s, Materializing param=model.layers.20.mlp.gate_proj.weight]
Loading weights:  56%|█████▋    | 224/398 [01:09<01:10,  2.48it/s, Materializing param=model.layers.20.mlp.gate_proj.weight]
Loading weights:  57%|█████▋    | 225/398 [01:09<01:09,  2.48it/s, Materializing param=model.layers.20.mlp.up_proj.weight]  
Loading weights:  57%|█████▋    | 225/398 [01:09<01:09,  2.48it/s, Materializing param=model.layers.20.mlp.up_proj.weight]
Loading weights:  57%|█████▋    | 226/398 [01:09<00:54,  3.15it/s, Materializing param=model.layers.20.mlp.up_proj.weight]
Loading weights:  57%|█████▋    | 226/398 [01:09<00:54,  3.15it/s, Materializing param=model.layers.20.post_attention_layernorm.weight]
Loading weights:  57%|█████▋    | 226/398 [01:09<00:54,  3.15it/s, Materializing param=model.layers.20.post_attention_layernorm.weight]
Loading weights:  57%|█████▋    | 227/398 [01:09<00:48,  3.53it/s, Materializing param=model.layers.20.post_attention_layernorm.weight]
Loading weights:  57%|█████▋    | 227/398 [01:09<00:48,  3.53it/s, Materializing param=model.layers.20.self_attn.k_norm.weight]        
Loading weights:  57%|█████▋    | 227/398 [01:09<00:48,  3.53it/s, Materializing param=model.layers.20.self_attn.k_norm.weight]
Loading weights:  57%|█████▋    | 228/398 [01:09<00:48,  3.53it/s, Materializing param=model.layers.20.self_attn.k_proj.weight]
Loading weights:  57%|█████▋    | 228/398 [01:09<00:48,  3.53it/s, Materializing param=model.layers.20.self_attn.k_proj.weight]
Loading weights:  58%|█████▊    | 229/398 [01:09<00:34,  4.87it/s, Materializing param=model.layers.20.self_attn.k_proj.weight]
Loading weights:  58%|█████▊    | 229/398 [01:09<00:34,  4.87it/s, Materializing param=model.layers.20.self_attn.o_proj.weight]
Loading weights:  58%|█████▊    | 229/398 [01:09<00:34,  4.87it/s, Materializing param=model.layers.20.self_attn.o_proj.weight]
Loading weights:  58%|█████▊    | 230/398 [01:10<00:54,  3.08it/s, Materializing param=model.layers.20.self_attn.o_proj.weight]
Loading weights:  58%|█████▊    | 230/398 [01:10<00:54,  3.08it/s, Materializing param=model.layers.20.self_attn.q_norm.weight]
Loading weights:  58%|█████▊    | 230/398 [01:10<00:54,  3.08it/s, Materializing param=model.layers.20.self_attn.q_norm.weight]
Loading weights:  58%|█████▊    | 231/398 [01:10<00:54,  3.08it/s, Materializing param=model.layers.20.self_attn.q_proj.weight]
Loading weights:  58%|█████▊    | 231/398 [01:10<00:54,  3.08it/s, Materializing param=model.layers.20.self_attn.q_proj.weight]
Loading weights:  58%|█████▊    | 232/398 [01:10<00:53,  3.08it/s, Materializing param=model.layers.20.self_attn.v_proj.weight]
Loading weights:  58%|█████▊    | 232/398 [01:10<00:53,  3.08it/s, Materializing param=model.layers.20.self_attn.v_proj.weight]
Loading weights:  59%|█████▊    | 233/398 [01:10<00:53,  3.08it/s, Materializing param=model.layers.21.input_layernorm.weight] 
Loading weights:  59%|█████▊    | 233/398 [01:10<00:53,  3.08it/s, Materializing param=model.layers.21.input_layernorm.weight]
Loading weights:  59%|█████▉    | 234/398 [01:11<00:43,  3.74it/s, Materializing param=model.layers.21.input_layernorm.weight]
Loading weights:  59%|█████▉    | 234/398 [01:11<00:43,  3.74it/s, Materializing param=model.layers.21.mlp.down_proj.weight]  
Loading weights:  59%|█████▉    | 234/398 [01:11<00:43,  3.74it/s, Materializing param=model.layers.21.mlp.down_proj.weight]
Loading weights:  59%|█████▉    | 235/398 [01:11<00:43,  3.74it/s, Materializing param=model.layers.21.mlp.gate_proj.weight]
Loading weights:  59%|█████▉    | 235/398 [01:11<00:43,  3.74it/s, Materializing param=model.layers.21.mlp.gate_proj.weight]
Loading weights:  59%|█████▉    | 236/398 [01:11<00:43,  3.74it/s, Materializing param=model.layers.21.mlp.up_proj.weight]  
Loading weights:  59%|█████▉    | 236/398 [01:11<00:43,  3.74it/s, Materializing param=model.layers.21.mlp.up_proj.weight]
Loading weights:  60%|█████▉    | 237/398 [01:11<00:32,  4.88it/s, Materializing param=model.layers.21.mlp.up_proj.weight]
Loading weights:  60%|█████▉    | 237/398 [01:11<00:32,  4.88it/s, Materializing param=model.layers.21.post_attention_layernorm.weight]
Loading weights:  60%|█████▉    | 237/398 [01:12<00:32,  4.88it/s, Materializing param=model.layers.21.post_attention_layernorm.weight]
Loading weights:  60%|█████▉    | 238/398 [01:12<00:30,  5.23it/s, Materializing param=model.layers.21.post_attention_layernorm.weight]
Loading weights:  60%|█████▉    | 238/398 [01:12<00:30,  5.23it/s, Materializing param=model.layers.21.self_attn.k_norm.weight]        
Loading weights:  60%|█████▉    | 238/398 [01:12<00:30,  5.23it/s, Materializing param=model.layers.21.self_attn.k_norm.weight]
Loading weights:  60%|██████    | 239/398 [01:12<00:30,  5.23it/s, Materializing param=model.layers.21.self_attn.k_proj.weight]
Loading weights:  60%|██████    | 239/398 [01:12<00:30,  5.23it/s, Materializing param=model.layers.21.self_attn.k_proj.weight]
Loading weights:  60%|██████    | 240/398 [01:12<00:23,  6.66it/s, Materializing param=model.layers.21.self_attn.k_proj.weight]
Loading weights:  60%|██████    | 240/398 [01:12<00:23,  6.66it/s, Materializing param=model.layers.21.self_attn.o_proj.weight]
Loading weights:  60%|██████    | 240/398 [01:12<00:23,  6.66it/s, Materializing param=model.layers.21.self_attn.o_proj.weight]
Loading weights:  61%|██████    | 241/398 [01:12<00:23,  6.66it/s, Materializing param=model.layers.21.self_attn.q_norm.weight]
Loading weights:  61%|██████    | 241/398 [01:12<00:23,  6.66it/s, Materializing param=model.layers.21.self_attn.q_norm.weight]
Loading weights:  61%|██████    | 242/398 [01:12<00:34,  4.47it/s, Materializing param=model.layers.21.self_attn.q_norm.weight]
Loading weights:  61%|██████    | 242/398 [01:12<00:34,  4.47it/s, Materializing param=model.layers.21.self_attn.q_proj.weight]
Loading weights:  61%|██████    | 242/398 [01:12<00:34,  4.47it/s, Materializing param=model.layers.21.self_attn.q_proj.weight]
Loading weights:  61%|██████    | 243/398 [01:12<00:34,  4.47it/s, Materializing param=model.layers.21.self_attn.v_proj.weight]
Loading weights:  61%|██████    | 243/398 [01:12<00:34,  4.47it/s, Materializing param=model.layers.21.self_attn.v_proj.weight]
Loading weights:  61%|██████▏   | 244/398 [01:13<00:26,  5.71it/s, Materializing param=model.layers.21.self_attn.v_proj.weight]
Loading weights:  61%|██████▏   | 244/398 [01:13<00:26,  5.71it/s, Materializing param=model.layers.22.input_layernorm.weight] 
Loading weights:  61%|██████▏   | 244/398 [01:13<00:26,  5.71it/s, Materializing param=model.layers.22.input_layernorm.weight]
Loading weights:  62%|██████▏   | 245/398 [01:13<00:26,  5.71it/s, Materializing param=model.layers.22.mlp.down_proj.weight]  
Loading weights:  62%|██████▏   | 245/398 [01:13<00:26,  5.71it/s, Materializing param=model.layers.22.mlp.down_proj.weight]
Loading weights:  62%|██████▏   | 246/398 [01:13<00:26,  5.71it/s, Materializing param=model.layers.22.mlp.gate_proj.weight]
Loading weights:  62%|██████▏   | 246/398 [01:13<00:26,  5.71it/s, Materializing param=model.layers.22.mlp.gate_proj.weight]
Loading weights:  62%|██████▏   | 247/398 [01:13<00:29,  5.09it/s, Materializing param=model.layers.22.mlp.gate_proj.weight]
Loading weights:  62%|██████▏   | 247/398 [01:13<00:29,  5.09it/s, Materializing param=model.layers.22.mlp.up_proj.weight]  
Loading weights:  62%|██████▏   | 247/398 [01:13<00:29,  5.09it/s, Materializing param=model.layers.22.mlp.up_proj.weight]
Loading weights:  62%|██████▏   | 248/398 [01:13<00:27,  5.50it/s, Materializing param=model.layers.22.mlp.up_proj.weight]
Loading weights:  62%|██████▏   | 248/398 [01:13<00:27,  5.50it/s, Materializing param=model.layers.22.post_attention_layernorm.weight]
Loading weights:  62%|██████▏   | 248/398 [01:13<00:27,  5.50it/s, Materializing param=model.layers.22.post_attention_layernorm.weight]
Loading weights:  63%|██████▎   | 249/398 [01:13<00:27,  5.50it/s, Materializing param=model.layers.22.self_attn.k_norm.weight]        
Loading weights:  63%|██████▎   | 249/398 [01:13<00:27,  5.50it/s, Materializing param=model.layers.22.self_attn.k_norm.weight]
Loading weights:  63%|██████▎   | 250/398 [01:13<00:26,  5.50it/s, Materializing param=model.layers.22.self_attn.k_proj.weight]
Loading weights:  63%|██████▎   | 250/398 [01:13<00:26,  5.50it/s, Materializing param=model.layers.22.self_attn.k_proj.weight]
Loading weights:  63%|██████▎   | 251/398 [01:14<00:21,  6.77it/s, Materializing param=model.layers.22.self_attn.k_proj.weight]
Loading weights:  63%|██████▎   | 251/398 [01:14<00:21,  6.77it/s, Materializing param=model.layers.22.self_attn.o_proj.weight]
Loading weights:  63%|██████▎   | 251/398 [01:14<00:21,  6.77it/s, Materializing param=model.layers.22.self_attn.o_proj.weight]
Loading weights:  63%|██████▎   | 252/398 [01:14<00:21,  6.77it/s, Materializing param=model.layers.22.self_attn.q_norm.weight]
Loading weights:  63%|██████▎   | 252/398 [01:14<00:21,  6.77it/s, Materializing param=model.layers.22.self_attn.q_norm.weight]
Loading weights:  64%|██████▎   | 253/398 [01:14<00:23,  6.27it/s, Materializing param=model.layers.22.self_attn.q_norm.weight]
Loading weights:  64%|██████▎   | 253/398 [01:14<00:23,  6.27it/s, Materializing param=model.layers.22.self_attn.q_proj.weight]
Loading weights:  64%|██████▎   | 253/398 [01:14<00:23,  6.27it/s, Materializing param=model.layers.22.self_attn.q_proj.weight]
Loading weights:  64%|██████▍   | 254/398 [01:14<00:22,  6.27it/s, Materializing param=model.layers.22.self_attn.v_proj.weight]
Loading weights:  64%|██████▍   | 254/398 [01:14<00:22,  6.27it/s, Materializing param=model.layers.22.self_attn.v_proj.weight]
Loading weights:  64%|██████▍   | 255/398 [01:14<00:19,  7.36it/s, Materializing param=model.layers.22.self_attn.v_proj.weight]
Loading weights:  64%|██████▍   | 255/398 [01:14<00:19,  7.36it/s, Materializing param=model.layers.23.input_layernorm.weight] 
Loading weights:  64%|██████▍   | 255/398 [01:14<00:19,  7.36it/s, Materializing param=model.layers.23.input_layernorm.weight]
Loading weights:  64%|██████▍   | 256/398 [01:14<00:19,  7.17it/s, Materializing param=model.layers.23.input_layernorm.weight]
Loading weights:  64%|██████▍   | 256/398 [01:14<00:19,  7.17it/s, Materializing param=model.layers.23.mlp.down_proj.weight]  
Loading weights:  64%|██████▍   | 256/398 [01:14<00:19,  7.17it/s, Materializing param=model.layers.23.mlp.down_proj.weight]
Loading weights:  65%|██████▍   | 257/398 [01:15<00:24,  5.66it/s, Materializing param=model.layers.23.mlp.down_proj.weight]
Loading weights:  65%|██████▍   | 257/398 [01:15<00:24,  5.66it/s, Materializing param=model.layers.23.mlp.gate_proj.weight]
Loading weights:  65%|██████▍   | 257/398 [01:15<00:24,  5.66it/s, Materializing param=model.layers.23.mlp.gate_proj.weight]
Loading weights:  65%|██████▍   | 258/398 [01:16<00:47,  2.94it/s, Materializing param=model.layers.23.mlp.gate_proj.weight]
Loading weights:  65%|██████▍   | 258/398 [01:16<00:47,  2.94it/s, Materializing param=model.layers.23.mlp.up_proj.weight]  
Loading weights:  65%|██████▍   | 258/398 [01:16<00:47,  2.94it/s, Materializing param=model.layers.23.mlp.up_proj.weight]
Loading weights:  65%|██████▌   | 259/398 [01:16<00:47,  2.94it/s, Materializing param=model.layers.23.post_attention_layernorm.weight]
Loading weights:  65%|██████▌   | 259/398 [01:16<00:47,  2.94it/s, Materializing param=model.layers.23.post_attention_layernorm.weight]
Loading weights:  65%|██████▌   | 260/398 [01:16<00:31,  4.34it/s, Materializing param=model.layers.23.post_attention_layernorm.weight]
Loading weights:  65%|██████▌   | 260/398 [01:16<00:31,  4.34it/s, Materializing param=model.layers.23.self_attn.k_norm.weight]        
Loading weights:  65%|██████▌   | 260/398 [01:16<00:31,  4.34it/s, Materializing param=model.layers.23.self_attn.k_norm.weight]
Loading weights:  66%|██████▌   | 261/398 [01:16<00:31,  4.34it/s, Materializing param=model.layers.23.self_attn.k_proj.weight]
Loading weights:  66%|██████▌   | 261/398 [01:16<00:31,  4.34it/s, Materializing param=model.layers.23.self_attn.k_proj.weight]
Loading weights:  66%|██████▌   | 262/398 [01:16<00:31,  4.34it/s, Materializing param=model.layers.23.self_attn.o_proj.weight]
Loading weights:  66%|██████▌   | 262/398 [01:16<00:31,  4.34it/s, Materializing param=model.layers.23.self_attn.o_proj.weight]
Loading weights:  66%|██████▌   | 263/398 [01:16<00:21,  6.32it/s, Materializing param=model.layers.23.self_attn.o_proj.weight]
Loading weights:  66%|██████▌   | 263/398 [01:16<00:21,  6.32it/s, Materializing param=model.layers.23.self_attn.q_norm.weight]
Loading weights:  66%|██████▌   | 263/398 [01:16<00:21,  6.32it/s, Materializing param=model.layers.23.self_attn.q_norm.weight]
Loading weights:  66%|██████▋   | 264/398 [01:17<00:21,  6.32it/s, Materializing param=model.layers.23.self_attn.q_proj.weight]
Loading weights:  66%|██████▋   | 264/398 [01:17<00:21,  6.32it/s, Materializing param=model.layers.23.self_attn.q_proj.weight]
Loading weights:  67%|██████▋   | 265/398 [01:17<00:34,  3.81it/s, Materializing param=model.layers.23.self_attn.q_proj.weight]
Loading weights:  67%|██████▋   | 265/398 [01:17<00:34,  3.81it/s, Materializing param=model.layers.23.self_attn.v_proj.weight]
Loading weights:  67%|██████▋   | 265/398 [01:17<00:34,  3.81it/s, Materializing param=model.layers.23.self_attn.v_proj.weight]
Loading weights:  67%|██████▋   | 266/398 [01:17<00:34,  3.81it/s, Materializing param=model.layers.24.input_layernorm.weight] 
Loading weights:  67%|██████▋   | 266/398 [01:17<00:34,  3.81it/s, Materializing param=model.layers.24.input_layernorm.weight]
Loading weights:  67%|██████▋   | 267/398 [01:17<00:34,  3.81it/s, Materializing param=model.layers.24.mlp.down_proj.weight]  
Loading weights:  67%|██████▋   | 267/398 [01:17<00:34,  3.81it/s, Materializing param=model.layers.24.mlp.down_proj.weight]
Loading weights:  67%|██████▋   | 268/398 [01:17<00:34,  3.81it/s, Materializing param=model.layers.24.mlp.gate_proj.weight]
Loading weights:  67%|██████▋   | 268/398 [01:17<00:34,  3.81it/s, Materializing param=model.layers.24.mlp.gate_proj.weight]
Loading weights:  68%|██████▊   | 269/398 [01:17<00:21,  6.06it/s, Materializing param=model.layers.24.mlp.gate_proj.weight]
Loading weights:  68%|██████▊   | 269/398 [01:17<00:21,  6.06it/s, Materializing param=model.layers.24.mlp.up_proj.weight]  
Loading weights:  68%|██████▊   | 269/398 [01:17<00:21,  6.06it/s, Materializing param=model.layers.24.mlp.up_proj.weight]
Loading weights:  68%|██████▊   | 270/398 [01:18<00:21,  6.06it/s, Materializing param=model.layers.24.post_attention_layernorm.weight]
Loading weights:  68%|██████▊   | 270/398 [01:18<00:21,  6.06it/s, Materializing param=model.layers.24.post_attention_layernorm.weight]
Loading weights:  68%|██████▊   | 271/398 [01:18<00:25,  4.95it/s, Materializing param=model.layers.24.post_attention_layernorm.weight]
Loading weights:  68%|██████▊   | 271/398 [01:18<00:25,  4.95it/s, Materializing param=model.layers.24.self_attn.k_norm.weight]        
Loading weights:  68%|██████▊   | 271/398 [01:18<00:25,  4.95it/s, Materializing param=model.layers.24.self_attn.k_norm.weight]
Loading weights:  68%|██████▊   | 272/398 [01:18<00:25,  4.95it/s, Materializing param=model.layers.24.self_attn.k_proj.weight]
Loading weights:  68%|██████▊   | 272/398 [01:18<00:25,  4.95it/s, Materializing param=model.layers.24.self_attn.k_proj.weight]
Loading weights:  69%|██████▊   | 273/398 [01:18<00:25,  4.95it/s, Materializing param=model.layers.24.self_attn.o_proj.weight]
Loading weights:  69%|██████▊   | 273/398 [01:18<00:25,  4.95it/s, Materializing param=model.layers.24.self_attn.o_proj.weight]
Loading weights:  69%|██████▉   | 274/398 [01:18<00:23,  5.35it/s, Materializing param=model.layers.24.self_attn.o_proj.weight]
Loading weights:  69%|██████▉   | 274/398 [01:18<00:23,  5.35it/s, Materializing param=model.layers.24.self_attn.q_norm.weight]
Loading weights:  69%|██████▉   | 274/398 [01:18<00:23,  5.35it/s, Materializing param=model.layers.24.self_attn.q_norm.weight]
Loading weights:  69%|██████▉   | 275/398 [01:19<00:31,  3.86it/s, Materializing param=model.layers.24.self_attn.q_norm.weight]
Loading weights:  69%|██████▉   | 275/398 [01:19<00:31,  3.86it/s, Materializing param=model.layers.24.self_attn.q_proj.weight]
Loading weights:  69%|██████▉   | 275/398 [01:19<00:31,  3.86it/s, Materializing param=model.layers.24.self_attn.q_proj.weight]
Loading weights:  69%|██████▉   | 276/398 [01:19<00:31,  3.86it/s, Materializing param=model.layers.24.self_attn.v_proj.weight]
Loading weights:  69%|██████▉   | 276/398 [01:19<00:31,  3.86it/s, Materializing param=model.layers.24.self_attn.v_proj.weight]
Loading weights:  70%|██████▉   | 277/398 [01:19<00:31,  3.86it/s, Materializing param=model.layers.25.input_layernorm.weight] 
Loading weights:  70%|██████▉   | 277/398 [01:19<00:31,  3.86it/s, Materializing param=model.layers.25.input_layernorm.weight]
Loading weights:  70%|██████▉   | 278/398 [01:19<00:22,  5.37it/s, Materializing param=model.layers.25.input_layernorm.weight]
Loading weights:  70%|██████▉   | 278/398 [01:19<00:22,  5.37it/s, Materializing param=model.layers.25.mlp.down_proj.weight]  
Loading weights:  70%|██████▉   | 278/398 [01:19<00:22,  5.37it/s, Materializing param=model.layers.25.mlp.down_proj.weight]
Loading weights:  70%|███████   | 279/398 [01:19<00:24,  4.93it/s, Materializing param=model.layers.25.mlp.down_proj.weight]
Loading weights:  70%|███████   | 279/398 [01:19<00:24,  4.93it/s, Materializing param=model.layers.25.mlp.gate_proj.weight]
Loading weights:  70%|███████   | 279/398 [01:19<00:24,  4.93it/s, Materializing param=model.layers.25.mlp.gate_proj.weight]
Loading weights:  70%|███████   | 280/398 [01:20<00:26,  4.51it/s, Materializing param=model.layers.25.mlp.gate_proj.weight]
Loading weights:  70%|███████   | 280/398 [01:20<00:26,  4.51it/s, Materializing param=model.layers.25.mlp.up_proj.weight]  
Loading weights:  70%|███████   | 280/398 [01:20<00:26,  4.51it/s, Materializing param=model.layers.25.mlp.up_proj.weight]
Loading weights:  71%|███████   | 281/398 [01:20<00:26,  4.41it/s, Materializing param=model.layers.25.mlp.up_proj.weight]
Loading weights:  71%|███████   | 281/398 [01:20<00:26,  4.41it/s, Materializing param=model.layers.25.post_attention_layernorm.weight]
Loading weights:  71%|███████   | 281/398 [01:20<00:26,  4.41it/s, Materializing param=model.layers.25.post_attention_layernorm.weight]
Loading weights:  71%|███████   | 282/398 [01:20<00:23,  4.85it/s, Materializing param=model.layers.25.post_attention_layernorm.weight]
Loading weights:  71%|███████   | 282/398 [01:20<00:23,  4.85it/s, Materializing param=model.layers.25.self_attn.k_norm.weight]        
Loading weights:  71%|███████   | 282/398 [01:20<00:23,  4.85it/s, Materializing param=model.layers.25.self_attn.k_norm.weight]
Loading weights:  71%|███████   | 283/398 [01:20<00:23,  4.85it/s, Materializing param=model.layers.25.self_attn.k_proj.weight]
Loading weights:  71%|███████   | 283/398 [01:20<00:23,  4.85it/s, Materializing param=model.layers.25.self_attn.k_proj.weight]
Loading weights:  71%|███████▏  | 284/398 [01:20<00:23,  4.85it/s, Materializing param=model.layers.25.self_attn.o_proj.weight]
Loading weights:  71%|███████▏  | 284/398 [01:20<00:23,  4.85it/s, Materializing param=model.layers.25.self_attn.o_proj.weight]
Loading weights:  72%|███████▏  | 285/398 [01:20<00:23,  4.85it/s, Materializing param=model.layers.25.self_attn.q_norm.weight]
Loading weights:  72%|███████▏  | 285/398 [01:20<00:23,  4.85it/s, Materializing param=model.layers.25.self_attn.q_norm.weight]
Loading weights:  72%|███████▏  | 286/398 [01:21<00:20,  5.38it/s, Materializing param=model.layers.25.self_attn.q_norm.weight]
Loading weights:  72%|███████▏  | 286/398 [01:21<00:20,  5.38it/s, Materializing param=model.layers.25.self_attn.q_proj.weight]
Loading weights:  72%|███████▏  | 286/398 [01:21<00:20,  5.38it/s, Materializing param=model.layers.25.self_attn.q_proj.weight]
Loading weights:  72%|███████▏  | 287/398 [01:21<00:20,  5.38it/s, Materializing param=model.layers.25.self_attn.v_proj.weight]
Loading weights:  72%|███████▏  | 287/398 [01:21<00:20,  5.38it/s, Materializing param=model.layers.25.self_attn.v_proj.weight]
Loading weights:  72%|███████▏  | 288/398 [01:21<00:20,  5.38it/s, Materializing param=model.layers.26.input_layernorm.weight] 
Loading weights:  72%|███████▏  | 288/398 [01:21<00:20,  5.38it/s, Materializing param=model.layers.26.input_layernorm.weight]
Loading weights:  73%|███████▎  | 289/398 [01:21<00:20,  5.38it/s, Materializing param=model.layers.26.mlp.down_proj.weight]  
Loading weights:  73%|███████▎  | 289/398 [01:21<00:20,  5.38it/s, Materializing param=model.layers.26.mlp.down_proj.weight]
Loading weights:  73%|███████▎  | 290/398 [01:21<00:14,  7.47it/s, Materializing param=model.layers.26.mlp.down_proj.weight]
Loading weights:  73%|███████▎  | 290/398 [01:21<00:14,  7.47it/s, Materializing param=model.layers.26.mlp.gate_proj.weight]
Loading weights:  73%|███████▎  | 290/398 [01:21<00:14,  7.47it/s, Materializing param=model.layers.26.mlp.gate_proj.weight]
Loading weights:  73%|███████▎  | 291/398 [01:21<00:16,  6.64it/s, Materializing param=model.layers.26.mlp.gate_proj.weight]
Loading weights:  73%|███████▎  | 291/398 [01:21<00:16,  6.64it/s, Materializing param=model.layers.26.mlp.up_proj.weight]  
Loading weights:  73%|███████▎  | 291/398 [01:21<00:16,  6.64it/s, Materializing param=model.layers.26.mlp.up_proj.weight]
Loading weights:  73%|███████▎  | 292/398 [01:22<00:20,  5.12it/s, Materializing param=model.layers.26.mlp.up_proj.weight]
Loading weights:  73%|███████▎  | 292/398 [01:22<00:20,  5.12it/s, Materializing param=model.layers.26.post_attention_layernorm.weight]
Loading weights:  73%|███████▎  | 292/398 [01:22<00:20,  5.12it/s, Materializing param=model.layers.26.post_attention_layernorm.weight]
Loading weights:  74%|███████▎  | 293/398 [01:22<00:20,  5.12it/s, Materializing param=model.layers.26.self_attn.k_norm.weight]        
Loading weights:  74%|███████▎  | 293/398 [01:22<00:20,  5.12it/s, Materializing param=model.layers.26.self_attn.k_norm.weight]
Loading weights:  74%|███████▍  | 294/398 [01:22<00:15,  6.68it/s, Materializing param=model.layers.26.self_attn.k_norm.weight]
Loading weights:  74%|███████▍  | 294/398 [01:22<00:15,  6.68it/s, Materializing param=model.layers.26.self_attn.k_proj.weight]
Loading weights:  74%|███████▍  | 294/398 [01:22<00:15,  6.68it/s, Materializing param=model.layers.26.self_attn.k_proj.weight]
Loading weights:  74%|███████▍  | 295/398 [01:22<00:15,  6.68it/s, Materializing param=model.layers.26.self_attn.o_proj.weight]
Loading weights:  74%|███████▍  | 295/398 [01:22<00:15,  6.68it/s, Materializing param=model.layers.26.self_attn.o_proj.weight]
Loading weights:  74%|███████▍  | 296/398 [01:23<00:21,  4.77it/s, Materializing param=model.layers.26.self_attn.o_proj.weight]
Loading weights:  74%|███████▍  | 296/398 [01:23<00:21,  4.77it/s, Materializing param=model.layers.26.self_attn.q_norm.weight]
Loading weights:  74%|███████▍  | 296/398 [01:23<00:21,  4.77it/s, Materializing param=model.layers.26.self_attn.q_norm.weight]
Loading weights:  75%|███████▍  | 297/398 [01:23<00:21,  4.77it/s, Materializing param=model.layers.26.self_attn.q_proj.weight]
Loading weights:  75%|███████▍  | 297/398 [01:23<00:21,  4.77it/s, Materializing param=model.layers.26.self_attn.q_proj.weight]
Loading weights:  75%|███████▍  | 298/398 [01:23<00:17,  5.74it/s, Materializing param=model.layers.26.self_attn.q_proj.weight]
Loading weights:  75%|███████▍  | 298/398 [01:23<00:17,  5.74it/s, Materializing param=model.layers.26.self_attn.v_proj.weight]
Loading weights:  75%|███████▍  | 298/398 [01:23<00:17,  5.74it/s, Materializing param=model.layers.26.self_attn.v_proj.weight]
Loading weights:  75%|███████▌  | 299/398 [01:23<00:17,  5.74it/s, Materializing param=model.layers.27.input_layernorm.weight] 
Loading weights:  75%|███████▌  | 299/398 [01:23<00:17,  5.74it/s, Materializing param=model.layers.27.input_layernorm.weight]
Loading weights:  75%|███████▌  | 300/398 [01:23<00:21,  4.67it/s, Materializing param=model.layers.27.input_layernorm.weight]
Loading weights:  75%|███████▌  | 300/398 [01:23<00:21,  4.67it/s, Materializing param=model.layers.27.mlp.down_proj.weight]  
Loading weights:  75%|███████▌  | 300/398 [01:23<00:21,  4.67it/s, Materializing param=model.layers.27.mlp.down_proj.weight]
Loading weights:  76%|███████▌  | 301/398 [01:23<00:20,  4.67it/s, Materializing param=model.layers.27.mlp.gate_proj.weight]
Loading weights:  76%|███████▌  | 301/398 [01:23<00:20,  4.67it/s, Materializing param=model.layers.27.mlp.gate_proj.weight]
Loading weights:  76%|███████▌  | 302/398 [01:23<00:20,  4.67it/s, Materializing param=model.layers.27.mlp.up_proj.weight]  
Loading weights:  76%|███████▌  | 302/398 [01:23<00:20,  4.67it/s, Materializing param=model.layers.27.mlp.up_proj.weight]
Loading weights:  76%|███████▌  | 303/398 [01:23<00:20,  4.67it/s, Materializing param=model.layers.27.post_attention_layernorm.weight]
Loading weights:  76%|███████▌  | 303/398 [01:23<00:20,  4.67it/s, Materializing param=model.layers.27.post_attention_layernorm.weight]
Loading weights:  76%|███████▋  | 304/398 [01:23<00:20,  4.67it/s, Materializing param=model.layers.27.self_attn.k_norm.weight]        
Loading weights:  76%|███████▋  | 304/398 [01:23<00:20,  4.67it/s, Materializing param=model.layers.27.self_attn.k_norm.weight]
Loading weights:  77%|███████▋  | 305/398 [01:23<00:19,  4.67it/s, Materializing param=model.layers.27.self_attn.k_proj.weight]
Loading weights:  77%|███████▋  | 305/398 [01:23<00:19,  4.67it/s, Materializing param=model.layers.27.self_attn.k_proj.weight]
Loading weights:  77%|███████▋  | 306/398 [01:24<00:10,  8.65it/s, Materializing param=model.layers.27.self_attn.k_proj.weight]
Loading weights:  77%|███████▋  | 306/398 [01:24<00:10,  8.65it/s, Materializing param=model.layers.27.self_attn.o_proj.weight]
Loading weights:  77%|███████▋  | 306/398 [01:24<00:10,  8.65it/s, Materializing param=model.layers.27.self_attn.o_proj.weight]
Loading weights:  77%|███████▋  | 307/398 [01:24<00:10,  8.65it/s, Materializing param=model.layers.27.self_attn.q_norm.weight]
Loading weights:  77%|███████▋  | 307/398 [01:24<00:10,  8.65it/s, Materializing param=model.layers.27.self_attn.q_norm.weight]
Loading weights:  77%|███████▋  | 308/398 [01:24<00:12,  7.13it/s, Materializing param=model.layers.27.self_attn.q_norm.weight]
Loading weights:  77%|███████▋  | 308/398 [01:24<00:12,  7.13it/s, Materializing param=model.layers.27.self_attn.q_proj.weight]
Loading weights:  77%|███████▋  | 308/398 [01:24<00:12,  7.13it/s, Materializing param=model.layers.27.self_attn.q_proj.weight]
Loading weights:  78%|███████▊  | 309/398 [01:24<00:12,  7.13it/s, Materializing param=model.layers.27.self_attn.v_proj.weight]
Loading weights:  78%|███████▊  | 309/398 [01:24<00:12,  7.13it/s, Materializing param=model.layers.27.self_attn.v_proj.weight]
Loading weights:  78%|███████▊  | 310/398 [01:24<00:11,  7.72it/s, Materializing param=model.layers.27.self_attn.v_proj.weight]
Loading weights:  78%|███████▊  | 310/398 [01:24<00:11,  7.72it/s, Materializing param=model.layers.28.input_layernorm.weight] 
Loading weights:  78%|███████▊  | 310/398 [01:24<00:11,  7.72it/s, Materializing param=model.layers.28.input_layernorm.weight]
Loading weights:  78%|███████▊  | 311/398 [01:24<00:11,  7.72it/s, Materializing param=model.layers.28.mlp.down_proj.weight]  
Loading weights:  78%|███████▊  | 311/398 [01:24<00:11,  7.72it/s, Materializing param=model.layers.28.mlp.down_proj.weight]
Loading weights:  78%|███████▊  | 312/398 [01:25<00:12,  7.12it/s, Materializing param=model.layers.28.mlp.down_proj.weight]
Loading weights:  78%|███████▊  | 312/398 [01:25<00:12,  7.12it/s, Materializing param=model.layers.28.mlp.gate_proj.weight]
Loading weights:  78%|███████▊  | 312/398 [01:25<00:12,  7.12it/s, Materializing param=model.layers.28.mlp.gate_proj.weight]
Loading weights:  79%|███████▊  | 313/398 [01:25<00:11,  7.12it/s, Materializing param=model.layers.28.mlp.up_proj.weight]  
Loading weights:  79%|███████▊  | 313/398 [01:25<00:11,  7.12it/s, Materializing param=model.layers.28.mlp.up_proj.weight]
Loading weights:  79%|███████▉  | 314/398 [01:25<00:11,  7.12it/s, Materializing param=model.layers.28.post_attention_layernorm.weight]
Loading weights:  79%|███████▉  | 314/398 [01:25<00:11,  7.12it/s, Materializing param=model.layers.28.post_attention_layernorm.weight]
Loading weights:  79%|███████▉  | 315/398 [01:25<00:11,  7.12it/s, Materializing param=model.layers.28.self_attn.k_norm.weight]        
Loading weights:  79%|███████▉  | 315/398 [01:25<00:11,  7.12it/s, Materializing param=model.layers.28.self_attn.k_norm.weight]
Loading weights:  79%|███████▉  | 316/398 [01:25<00:11,  7.12it/s, Materializing param=model.layers.28.self_attn.k_proj.weight]
Loading weights:  79%|███████▉  | 316/398 [01:25<00:11,  7.12it/s, Materializing param=model.layers.28.self_attn.k_proj.weight]
Loading weights:  80%|███████▉  | 317/398 [01:25<00:07, 11.01it/s, Materializing param=model.layers.28.self_attn.k_proj.weight]
Loading weights:  80%|███████▉  | 317/398 [01:25<00:07, 11.01it/s, Materializing param=model.layers.28.self_attn.o_proj.weight]
Loading weights:  80%|███████▉  | 317/398 [01:25<00:07, 11.01it/s, Materializing param=model.layers.28.self_attn.o_proj.weight]
Loading weights:  80%|███████▉  | 318/398 [01:25<00:07, 11.01it/s, Materializing param=model.layers.28.self_attn.q_norm.weight]
Loading weights:  80%|███████▉  | 318/398 [01:25<00:07, 11.01it/s, Materializing param=model.layers.28.self_attn.q_norm.weight]
Loading weights:  80%|████████  | 319/398 [01:25<00:08,  9.75it/s, Materializing param=model.layers.28.self_attn.q_norm.weight]
Loading weights:  80%|████████  | 319/398 [01:25<00:08,  9.75it/s, Materializing param=model.layers.28.self_attn.q_proj.weight]
Loading weights:  80%|████████  | 319/398 [01:25<00:08,  9.75it/s, Materializing param=model.layers.28.self_attn.q_proj.weight]
Loading weights:  80%|████████  | 320/398 [01:25<00:08,  9.75it/s, Materializing param=model.layers.28.self_attn.v_proj.weight]
Loading weights:  80%|████████  | 320/398 [01:25<00:08,  9.75it/s, Materializing param=model.layers.28.self_attn.v_proj.weight]
Loading weights:  81%|████████  | 321/398 [01:25<00:07, 10.55it/s, Materializing param=model.layers.28.self_attn.v_proj.weight]
Loading weights:  81%|████████  | 321/398 [01:25<00:07, 10.55it/s, Materializing param=model.layers.29.input_layernorm.weight] 
Loading weights:  81%|████████  | 321/398 [01:25<00:07, 10.55it/s, Materializing param=model.layers.29.input_layernorm.weight]
Loading weights:  81%|████████  | 322/398 [01:26<00:07, 10.55it/s, Materializing param=model.layers.29.mlp.down_proj.weight]  
Loading weights:  81%|████████  | 322/398 [01:26<00:07, 10.55it/s, Materializing param=model.layers.29.mlp.down_proj.weight]
Loading weights:  81%|████████  | 323/398 [01:26<00:10,  7.49it/s, Materializing param=model.layers.29.mlp.down_proj.weight]
Loading weights:  81%|████████  | 323/398 [01:26<00:10,  7.49it/s, Materializing param=model.layers.29.mlp.gate_proj.weight]
Loading weights:  81%|████████  | 323/398 [01:26<00:10,  7.49it/s, Materializing param=model.layers.29.mlp.gate_proj.weight]
Loading weights:  81%|████████▏ | 324/398 [01:26<00:09,  7.49it/s, Materializing param=model.layers.29.mlp.up_proj.weight]  
Loading weights:  81%|████████▏ | 324/398 [01:26<00:09,  7.49it/s, Materializing param=model.layers.29.mlp.up_proj.weight]
Loading weights:  82%|████████▏ | 325/398 [01:26<00:13,  5.29it/s, Materializing param=model.layers.29.mlp.up_proj.weight]
Loading weights:  82%|████████▏ | 325/398 [01:26<00:13,  5.29it/s, Materializing param=model.layers.29.post_attention_layernorm.weight]
Loading weights:  82%|████████▏ | 325/398 [01:26<00:13,  5.29it/s, Materializing param=model.layers.29.post_attention_layernorm.weight]
Loading weights:  82%|████████▏ | 326/398 [01:26<00:13,  5.29it/s, Materializing param=model.layers.29.self_attn.k_norm.weight]        
Loading weights:  82%|████████▏ | 326/398 [01:26<00:13,  5.29it/s, Materializing param=model.layers.29.self_attn.k_norm.weight]
Loading weights:  82%|████████▏ | 327/398 [01:26<00:13,  5.29it/s, Materializing param=model.layers.29.self_attn.k_proj.weight]
Loading weights:  82%|████████▏ | 327/398 [01:26<00:13,  5.29it/s, Materializing param=model.layers.29.self_attn.k_proj.weight]
Loading weights:  82%|████████▏ | 328/398 [01:27<00:09,  7.20it/s, Materializing param=model.layers.29.self_attn.k_proj.weight]
Loading weights:  82%|████████▏ | 328/398 [01:27<00:09,  7.20it/s, Materializing param=model.layers.29.self_attn.o_proj.weight]
Loading weights:  82%|████████▏ | 328/398 [01:27<00:09,  7.20it/s, Materializing param=model.layers.29.self_attn.o_proj.weight]
Loading weights:  83%|████████▎ | 329/398 [01:27<00:09,  7.20it/s, Materializing param=model.layers.29.self_attn.q_norm.weight]
Loading weights:  83%|████████▎ | 329/398 [01:27<00:09,  7.20it/s, Materializing param=model.layers.29.self_attn.q_norm.weight]
Loading weights:  83%|████████▎ | 330/398 [01:27<00:10,  6.35it/s, Materializing param=model.layers.29.self_attn.q_norm.weight]
Loading weights:  83%|████████▎ | 330/398 [01:27<00:10,  6.35it/s, Materializing param=model.layers.29.self_attn.q_proj.weight]
Loading weights:  83%|████████▎ | 330/398 [01:27<00:10,  6.35it/s, Materializing param=model.layers.29.self_attn.q_proj.weight]
Loading weights:  83%|████████▎ | 331/398 [01:27<00:10,  6.35it/s, Materializing param=model.layers.29.self_attn.v_proj.weight]
Loading weights:  83%|████████▎ | 331/398 [01:27<00:10,  6.35it/s, Materializing param=model.layers.29.self_attn.v_proj.weight]
Loading weights:  83%|████████▎ | 332/398 [01:27<00:09,  6.61it/s, Materializing param=model.layers.29.self_attn.v_proj.weight]
Loading weights:  83%|████████▎ | 332/398 [01:27<00:09,  6.61it/s, Materializing param=model.layers.30.input_layernorm.weight] 
Loading weights:  83%|████████▎ | 332/398 [01:27<00:09,  6.61it/s, Materializing param=model.layers.30.input_layernorm.weight]
Loading weights:  84%|████████▎ | 333/398 [01:27<00:09,  6.61it/s, Materializing param=model.layers.30.mlp.down_proj.weight]  
Loading weights:  84%|████████▎ | 333/398 [01:27<00:09,  6.61it/s, Materializing param=model.layers.30.mlp.down_proj.weight]
Loading weights:  84%|████████▍ | 334/398 [01:29<00:20,  3.11it/s, Materializing param=model.layers.30.mlp.down_proj.weight]
Loading weights:  84%|████████▍ | 334/398 [01:29<00:20,  3.11it/s, Materializing param=model.layers.30.mlp.gate_proj.weight]
Loading weights:  84%|████████▍ | 334/398 [01:29<00:20,  3.11it/s, Materializing param=model.layers.30.mlp.gate_proj.weight]
Loading weights:  84%|████████▍ | 335/398 [01:29<00:20,  3.11it/s, Materializing param=model.layers.30.mlp.up_proj.weight]  
Loading weights:  84%|████████▍ | 335/398 [01:29<00:20,  3.11it/s, Materializing param=model.layers.30.mlp.up_proj.weight]
Loading weights:  84%|████████��� | 336/398 [01:29<00:19,  3.11it/s, Materializing param=model.layers.30.post_attention_layernorm.weight]
Loading weights:  84%|████████▍ | 336/398 [01:29<00:19,  3.11it/s, Materializing param=model.layers.30.post_attention_layernorm.weight]
Loading weights:  85%|████████▍ | 337/398 [01:29<00:19,  3.11it/s, Materializing param=model.layers.30.self_attn.k_norm.weight]        
Loading weights:  85%|████████▍ | 337/398 [01:29<00:19,  3.11it/s, Materializing param=model.layers.30.self_attn.k_norm.weight]
Loading weights:  85%|████████▍ | 338/398 [01:29<00:19,  3.11it/s, Materializing param=model.layers.30.self_attn.k_proj.weight]
Loading weights:  85%|████████▍ | 338/398 [01:29<00:19,  3.11it/s, Materializing param=model.layers.30.self_attn.k_proj.weight]
Loading weights:  85%|████████▌ | 339/398 [01:29<00:10,  5.66it/s, Materializing param=model.layers.30.self_attn.k_proj.weight]
Loading weights:  85%|████████▌ | 339/398 [01:29<00:10,  5.66it/s, Materializing param=model.layers.30.self_attn.o_proj.weight]
Loading weights:  85%|████████▌ | 339/398 [01:29<00:10,  5.66it/s, Materializing param=model.layers.30.self_attn.o_proj.weight]
Loading weights:  85%|████████▌ | 340/398 [01:29<00:10,  5.66it/s, Materializing param=model.layers.30.self_attn.q_norm.weight]
Loading weights:  85%|████████▌ | 340/398 [01:29<00:10,  5.66it/s, Materializing param=model.layers.30.self_attn.q_norm.weight]
Loading weights:  86%|████████▌ | 341/398 [01:29<00:11,  5.07it/s, Materializing param=model.layers.30.self_attn.q_norm.weight]
Loading weights:  86%|████████▌ | 341/398 [01:29<00:11,  5.07it/s, Materializing param=model.layers.30.self_attn.q_proj.weight]
Loading weights:  86%|████████▌ | 341/398 [01:29<00:11,  5.07it/s, Materializing param=model.layers.30.self_attn.q_proj.weight]
Loading weights:  86%|████████▌ | 342/398 [01:29<00:11,  5.07it/s, Materializing param=model.layers.30.self_attn.v_proj.weight]
Loading weights:  86%|████████▌ | 342/398 [01:29<00:11,  5.07it/s, Materializing param=model.layers.30.self_attn.v_proj.weight]
Loading weights:  86%|████████▌ | 343/398 [01:30<00:09,  5.80it/s, Materializing param=model.layers.30.self_attn.v_proj.weight]
Loading weights:  86%|████████▌ | 343/398 [01:30<00:09,  5.80it/s, Materializing param=model.layers.31.input_layernorm.weight] 
Loading weights:  86%|████████▌ | 343/398 [01:30<00:09,  5.80it/s, Materializing param=model.layers.31.input_layernorm.weight]
Loading weights:  86%|████████▋ | 344/398 [01:30<00:09,  5.80it/s, Materializing param=model.layers.31.mlp.down_proj.weight]  
Loading weights:  86%|████████▋ | 344/398 [01:30<00:09,  5.80it/s, Materializing param=model.layers.31.mlp.down_proj.weight]
Loading weights:  87%|████████▋ | 345/398 [01:30<00:07,  6.74it/s, Materializing param=model.layers.31.mlp.down_proj.weight]
Loading weights:  87%|████████▋ | 345/398 [01:30<00:07,  6.74it/s, Materializing param=model.layers.31.mlp.gate_proj.weight]
Loading weights:  87%|████████▋ | 345/398 [01:30<00:07,  6.74it/s, Materializing param=model.layers.31.mlp.gate_proj.weight]
Loading weights:  87%|████████▋ | 346/398 [01:31<00:07,  6.74it/s, Materializing param=model.layers.31.mlp.up_proj.weight]  
Loading weights:  87%|████████▋ | 346/398 [01:31<00:07,  6.74it/s, Materializing param=model.layers.31.mlp.up_proj.weight]
Loading weights:  87%|████████▋ | 347/398 [01:31<00:15,  3.24it/s, Materializing param=model.layers.31.mlp.up_proj.weight]
Loading weights:  87%|████████▋ | 347/398 [01:31<00:15,  3.24it/s, Materializing param=model.layers.31.post_attention_layernorm.weight]
Loading weights:  87%|████████▋ | 347/398 [01:31<00:15,  3.24it/s, Materializing param=model.layers.31.post_attention_layernorm.weight]
Loading weights:  87%|████████▋ | 348/398 [01:31<00:15,  3.24it/s, Materializing param=model.layers.31.self_attn.k_norm.weight]        
Loading weights:  87%|████████▋ | 348/398 [01:31<00:15,  3.24it/s, Materializing param=model.layers.31.self_attn.k_norm.weight]
Loading weights:  88%|████████▊ | 349/398 [01:32<00:12,  3.94it/s, Materializing param=model.layers.31.self_attn.k_norm.weight]
Loading weights:  88%|████████▊ | 349/398 [01:32<00:12,  3.94it/s, Materializing param=model.layers.31.self_attn.k_proj.weight]
Loading weights:  88%|████████▊ | 349/398 [01:32<00:12,  3.94it/s, Materializing param=model.layers.31.self_attn.k_proj.weight]
Loading weights:  88%|████████▊ | 350/398 [01:32<00:12,  3.94it/s, Materializing param=model.layers.31.self_attn.o_proj.weight]
Loading weights:  88%|████��███▊ | 350/398 [01:32<00:12,  3.94it/s, Materializing param=model.layers.31.self_attn.o_proj.weight]
Loading weights:  88%|████████▊ | 351/398 [01:32<00:11,  3.94it/s, Materializing param=model.layers.31.self_attn.q_norm.weight]
Loading weights:  88%|████████▊ | 351/398 [01:32<00:11,  3.94it/s, Materializing param=model.layers.31.self_attn.q_norm.weight]
Loading weights:  88%|████████▊ | 352/398 [01:32<00:11,  3.94it/s, Materializing param=model.layers.31.self_attn.q_proj.weight]
Loading weights:  88%|████████▊ | 352/398 [01:32<00:11,  3.94it/s, Materializing param=model.layers.31.self_attn.q_proj.weight]
Loading weights:  89%|████████▊ | 353/398 [01:32<00:09,  4.89it/s, Materializing param=model.layers.31.self_attn.q_proj.weight]
Loading weights:  89%|████████▊ | 353/398 [01:32<00:09,  4.89it/s, Materializing param=model.layers.31.self_attn.v_proj.weight]
Loading weights:  89%|████████▊ | 353/398 [01:32<00:09,  4.89it/s, Materializing param=model.layers.31.self_attn.v_proj.weight]
Loading weights:  89%|████████▉ | 354/398 [01:32<00:08,  5.10it/s, Materializing param=model.layers.31.self_attn.v_proj.weight]
Loading weights:  89%|████████▉ | 354/398 [01:32<00:08,  5.10it/s, Materializing param=model.layers.32.input_layernorm.weight] 
Loading weights:  89%|████████▉ | 354/398 [01:32<00:08,  5.10it/s, Materializing param=model.layers.32.input_layernorm.weight]
Loading weights:  89%|████████▉ | 355/398 [01:32<00:07,  5.45it/s, Materializing param=model.layers.32.input_layernorm.weight]
Loading weights:  89%|████████▉ | 355/398 [01:32<00:07,  5.45it/s, Materializing param=model.layers.32.mlp.down_proj.weight]  
Loading weights:  89%|████████▉ | 355/398 [01:32<00:07,  5.45it/s, Materializing param=model.layers.32.mlp.down_proj.weight]
Loading weights:  89%|████████▉ | 356/398 [01:33<00:08,  5.05it/s, Materializing param=model.layers.32.mlp.down_proj.weight]
Loading weights:  89%|████████▉ | 356/398 [01:33<00:08,  5.05it/s, Materializing param=model.layers.32.mlp.gate_proj.weight]
Loading weights:  89%|████████▉ | 356/398 [01:33<00:08,  5.05it/s, Materializing param=model.layers.32.mlp.gate_proj.weight]
Loading weights:  90%|████████▉ | 357/398 [01:33<00:10,  3.76it/s, Materializing param=model.layers.32.mlp.gate_proj.weight]
Loading weights:  90%|████████▉ | 357/398 [01:33<00:10,  3.76it/s, Materializing param=model.layers.32.mlp.up_proj.weight]  
Loading weights:  90%|████████▉ | 357/398 [01:33<00:10,  3.76it/s, Materializing param=model.layers.32.mlp.up_proj.weight]
Loading weights:  90%|████████▉ | 358/398 [01:34<00:14,  2.71it/s, Materializing param=model.layers.32.mlp.up_proj.weight]
Loading weights:  90%|████████▉ | 358/398 [01:34<00:14,  2.71it/s, Materializing param=model.layers.32.post_attention_layernorm.weight]
Loading weights:  90%|████████▉ | 358/398 [01:34<00:14,  2.71it/s, Materializing param=model.layers.32.post_attention_layernorm.weight]
Loading weights:  90%|█████████ | 359/398 [01:34<00:14,  2.71it/s, Materializing param=model.layers.32.self_attn.k_norm.weight]        
Loading weights:  90%|█████████ | 359/398 [01:34<00:14,  2.71it/s, Materializing param=model.layers.32.self_attn.k_norm.weight]
Loading weights:  90%|█████████ | 360/398 [01:34<00:14,  2.71it/s, Materializing param=model.layers.32.self_attn.k_proj.weight]
Loading weights:  90%|█████████ | 360/398 [01:34<00:14,  2.71it/s, Materializing param=model.layers.32.self_attn.k_proj.weight]
Loading weights:  91%|█████████ | 361/398 [01:34<00:07,  4.91it/s, Materializing param=model.layers.32.self_attn.k_proj.weight]
Loading weights:  91%|█████████ | 361/398 [01:34<00:07,  4.91it/s, Materializing param=model.layers.32.self_attn.o_proj.weight]
Loading weights:  91%|█████████ | 361/398 [01:34<00:07,  4.91it/s, Materializing param=model.layers.32.self_attn.o_proj.weight]
Loading weights:  91%|█████████ | 362/398 [01:35<00:07,  4.91it/s, Materializing param=model.layers.32.self_attn.q_norm.weight]
Loading weights:  91%|█████████ | 362/398 [01:35<00:07,  4.91it/s, Materializing param=model.layers.32.self_attn.q_norm.weight]
Loading weights:  91%|█████████ | 363/398 [01:35<00:11,  3.12it/s, Materializing param=model.layers.32.self_attn.q_norm.weight]
Loading weights:  91%|█████████ | 363/398 [01:35<00:11,  3.12it/s, Materializing param=model.layers.32.self_attn.q_proj.weight]
Loading weights:  91%|█████████ | 363/398 [01:35<00:11,  3.12it/s, Materializing param=model.layers.32.self_attn.q_proj.weight]
Loading weights:  91%|█████████▏| 364/398 [01:35<00:10,  3.12it/s, Materializing param=model.layers.32.self_attn.v_proj.weight]
Loading weights:  91%|█████████▏| 364/398 [01:35<00:10,  3.12it/s, Materializing param=model.layers.32.self_attn.v_proj.weight]
Loading weights:  92%|█████████▏| 365/398 [01:35<00:10,  3.12it/s, Materializing param=model.layers.33.input_layernorm.weight] 
Loading weights:  92%|█████████▏| 365/398 [01:35<00:10,  3.12it/s, Materializing param=model.layers.33.input_layernorm.weight]
Loading weights:  92%|█████████▏| 366/398 [01:35<00:10,  3.12it/s, Materializing param=model.layers.33.mlp.down_proj.weight]  
Loading weights:  92%|█████████▏| 366/398 [01:35<00:10,  3.12it/s, Materializing param=model.layers.33.mlp.down_proj.weight]
Loading weights:  92%|█████████▏| 367/398 [01:36<00:07,  4.22it/s, Materializing param=model.layers.33.mlp.down_proj.weight]
Loading weights:  92%|█████████▏| 367/398 [01:36<00:07,  4.22it/s, Materializing param=model.layers.33.mlp.gate_proj.weight]
Loading weights:  92%|█████████▏| 367/398 [01:36<00:07,  4.22it/s, Materializing param=model.layers.33.mlp.gate_proj.weight]
Loading weights:  92%|█████████▏| 368/398 [01:36<00:08,  3.66it/s, Materializing param=model.layers.33.mlp.gate_proj.weight]
Loading weights:  92%|█████████▏| 368/398 [01:36<00:08,  3.66it/s, Materializing param=model.layers.33.mlp.up_proj.weight]  
Loading weights:  92%|█████████▏| 368/398 [01:36<00:08,  3.66it/s, Materializing param=model.layers.33.mlp.up_proj.weight]
Loading weights:  93%|█████████▎| 369/398 [01:37<00:08,  3.47it/s, Materializing param=model.layers.33.mlp.up_proj.weight]
Loading weights:  93%|█████████▎| 369/398 [01:37<00:08,  3.47it/s, Materializing param=model.layers.33.post_attention_layernorm.weight]
Loading weights:  93%|█████████▎| 369/398 [01:37<00:08,  3.47it/s, Materializing param=model.layers.33.post_attention_layernorm.weight]
Loading weights:  93%|█████████▎| 370/398 [01:37<00:08,  3.47it/s, Materializing param=model.layers.33.self_attn.k_norm.weight]        
Loading weights:  93%|█████████▎| 370/398 [01:37<00:08,  3.47it/s, Materializing param=model.layers.33.self_attn.k_norm.weight]
Loading weights:  93%|█████████▎| 371/398 [01:37<00:07,  3.47it/s, Materializing param=model.layers.33.self_attn.k_proj.weight]
Loading weights:  93%|█████████▎| 371/398 [01:37<00:07,  3.47it/s, Materializing param=model.layers.33.self_attn.k_proj.weight]
Loading weights:  93%|█████████▎| 372/398 [01:37<00:07,  3.47it/s, Materializing param=model.layers.33.self_attn.o_proj.weight]
Loading weights:  93%|█████████▎| 372/398 [01:37<00:07,  3.47it/s, Materializing param=model.layers.33.self_attn.o_proj.weight]
Loading weights:  94%|█████████▎| 373/398 [01:37<00:05,  4.94it/s, Materializing param=model.layers.33.self_attn.o_proj.weight]
Loading weights:  94%|█████████▎| 373/398 [01:37<00:05,  4.94it/s, Materializing param=model.layers.33.self_attn.q_norm.weight]
Loading weights:  94%|█████████▎| 373/398 [01:37<00:05,  4.94it/s, Materializing param=model.layers.33.self_attn.q_norm.weight]
Loading weights:  94%|█████████▍| 374/398 [01:37<00:05,  4.21it/s, Materializing param=model.layers.33.self_attn.q_norm.weight]
Loading weights:  94%|█████████▍| 374/398 [01:37<00:05,  4.21it/s, Materializing param=model.layers.33.self_attn.q_proj.weight]
Loading weights:  94%|█████████▍| 374/398 [01:37<00:05,  4.21it/s, Materializing param=model.layers.33.self_attn.q_proj.weight]
Loading weights:  94%|█████████▍| 375/398 [01:37<00:05,  4.21it/s, Materializing param=model.layers.33.self_attn.v_proj.weight]
Loading weights:  94%|█████████▍| 375/398 [01:37<00:05,  4.21it/s, Materializing param=model.layers.33.self_attn.v_proj.weight]
Loading weights:  94%|█████████▍| 376/398 [01:38<00:04,  4.75it/s, Materializing param=model.layers.33.self_attn.v_proj.weight]
Loading weights:  94%|█████████▍| 376/398 [01:38<00:04,  4.75it/s, Materializing param=model.layers.34.input_layernorm.weight] 
Loading weights:  94%|█████████▍| 376/398 [01:38<00:04,  4.75it/s, Materializing param=model.layers.34.input_layernorm.weight]
Loading weights:  95%|█████████▍| 377/398 [01:38<00:04,  4.75it/s, Materializing param=model.layers.34.mlp.down_proj.weight]  
Loading weights:  95%|█████████▍| 377/398 [01:38<00:04,  4.75it/s, Materializing param=model.layers.34.mlp.down_proj.weight]
Loading weights:  95%|█████████▍| 378/398 [01:38<00:03,  6.10it/s, Materializing param=model.layers.34.mlp.down_proj.weight]
Loading weights:  95%|█████████▍| 378/398 [01:38<00:03,  6.10it/s, Materializing param=model.layers.34.mlp.gate_proj.weight]
Loading weights:  95%|█████████▍| 378/398 [01:38<00:03,  6.10it/s, Materializing param=model.layers.34.mlp.gate_proj.weight]
Loading weights:  95%|█████████▌| 379/398 [01:39<00:05,  3.30it/s, Materializing param=model.layers.34.mlp.gate_proj.weight]
Loading weights:  95%|█████████▌| 379/398 [01:39<00:05,  3.30it/s, Materializing param=model.layers.34.mlp.up_proj.weight]  
Loading weights:  95%|█████████▌| 379/398 [01:39<00:05,  3.30it/s, Materializing param=model.layers.34.mlp.up_proj.weight]
Loading weights:  95%|█████████▌| 380/398 [01:39<00:05,  3.30it/s, Materializing param=model.layers.34.post_attention_layernorm.weight]
Loading weights:  95%|█████████▌| 380/398 [01:39<00:05,  3.30it/s, Materializing param=model.layers.34.post_attention_layernorm.weight]
Loading weights:  96%|█████████▌| 381/398 [01:39<00:05,  3.30it/s, Materializing param=model.layers.34.self_attn.k_norm.weight]        
Loading weights:  96%|█████████▌| 381/398 [01:39<00:05,  3.30it/s, Materializing param=model.layers.34.self_attn.k_norm.weight]
Loading weights:  96%|█████████▌| 382/398 [01:39<00:04,  3.30it/s, Materializing param=model.layers.34.self_attn.k_proj.weight]
Loading weights:  96%|█████████▌| 382/398 [01:39<00:04,  3.30it/s, Materializing param=model.layers.34.self_attn.k_proj.weight]
Loading weights:  96%|█████████▌| 383/398 [01:39<00:04,  3.30it/s, Materializing param=model.layers.34.self_attn.o_proj.weight]
Loading weights:  96%|█████████▌| 383/398 [01:39<00:04,  3.30it/s, Materializing param=model.layers.34.self_attn.o_proj.weight]
Loading weights:  96%|█████████▋| 384/398 [01:39<00:02,  5.19it/s, Materializing param=model.layers.34.self_attn.o_proj.weight]
Loading weights:  96%|█████████▋| 384/398 [01:39<00:02,  5.19it/s, Materializing param=model.layers.34.self_attn.q_norm.weight]
Loading weights:  96%|█████████▋| 384/398 [01:39<00:02,  5.19it/s, Materializing param=model.layers.34.self_attn.q_norm.weight]
Loading weights:  97%|█████████▋| 385/398 [01:39<00:02,  5.19it/s, Materializing param=model.layers.34.self_attn.q_proj.weight]
Loading weights:  97%|█████████▋| 385/398 [01:39<00:02,  5.19it/s, Materializing param=model.layers.34.self_attn.q_proj.weight]
Loading weights:  97%|█████████▋| 386/398 [01:40<00:02,  4.47it/s, Materializing param=model.layers.34.self_attn.q_proj.weight]
Loading weights:  97%|█████████▋| 386/398 [01:40<00:02,  4.47it/s, Materializing param=model.layers.34.self_attn.v_proj.weight]
Loading weights:  97%|█████████▋| 386/398 [01:40<00:02,  4.47it/s, Materializing param=model.layers.34.self_attn.v_proj.weight]
Loading weights:  97%|█████████▋| 387/398 [01:40<00:02,  4.22it/s, Materializing param=model.layers.34.self_attn.v_proj.weight]
Loading weights:  97%|█████████▋| 387/398 [01:40<00:02,  4.22it/s, Materializing param=model.layers.35.input_layernorm.weight] 
Loading weights:  97%|█████████▋| 387/398 [01:40<00:02,  4.22it/s, Materializing param=model.layers.35.input_layernorm.weight]
Loading weights:  97%|█████████▋| 388/398 [01:40<00:02,  4.22it/s, Materializing param=model.layers.35.mlp.down_proj.weight]  
Loading weights:  97%|█████████▋| 388/398 [01:40<00:02,  4.22it/s, Materializing param=model.layers.35.mlp.down_proj.weight]
Loading weights:  98%|█████████▊| 389/398 [01:44<00:06,  1.46it/s, Materializing param=model.layers.35.mlp.down_proj.weight]
Loading weights:  98%|█████████▊| 389/398 [01:44<00:06,  1.46it/s, Materializing param=model.layers.35.mlp.gate_proj.weight]
Loading weights:  98%|█████████▊| 389/398 [01:44<00:06,  1.46it/s, Materializing param=model.layers.35.mlp.gate_proj.weight]
Loading weights:  98%|█████████▊| 390/398 [01:45<00:06,  1.20it/s, Materializing param=model.layers.35.mlp.gate_proj.weight]
Loading weights:  98%|█████████▊| 390/398 [01:45<00:06,  1.20it/s, Materializing param=model.layers.35.mlp.up_proj.weight]  
Loading weights:  98%|█████████▊| 390/398 [01:45<00:06,  1.20it/s, Materializing param=model.layers.35.mlp.up_proj.weight]
Loading weights:  98%|█████████▊| 391/398 [01:45<00:05,  1.20it/s, Materializing param=model.layers.35.post_attention_layernorm.weight]
Loading weights:  98%|█████████▊| 391/398 [01:45<00:05,  1.20it/s, Materializing param=model.layers.35.post_attention_layernorm.weight]
Loading weights:  98%|█████████▊| 392/398 [01:45<00:04,  1.20it/s, Materializing param=model.layers.35.self_attn.k_norm.weight]        
Loading weights:  98%|█████████▊| 392/398 [01:45<00:04,  1.20it/s, Materializing param=model.layers.35.self_attn.k_norm.weight]
Loading weights:  99%|█████████▊| 393/398 [01:45<00:04,  1.20it/s, Materializing param=model.layers.35.self_attn.k_proj.weight]
Loading weights:  99%|█████████▊| 393/398 [01:45<00:04,  1.20it/s, Materializing param=model.layers.35.self_attn.k_proj.weight]
Loading weights:  99%|█████████▉| 394/398 [01:45<00:03,  1.20it/s, Materializing param=model.layers.35.self_attn.o_proj.weight]
Loading weights:  99%|█████████▉| 394/398 [01:45<00:03,  1.20it/s, Materializing param=model.layers.35.self_attn.o_proj.weight]
Loading weights:  99%|█████████▉| 395/398 [01:45<00:02,  1.20it/s, Materializing param=model.layers.35.self_attn.q_norm.weight]
Loading weights:  99%|█████████▉| 395/398 [01:45<00:02,  1.20it/s, Materializing param=model.layers.35.self_attn.q_norm.weight]
Loading weights:  99%|█████████▉| 396/398 [01:46<00:00,  2.78it/s, Materializing param=model.layers.35.self_attn.q_norm.weight]
Loading weights:  99%|█████████▉| 396/398 [01:46<00:00,  2.78it/s, Materializing param=model.layers.35.self_attn.q_proj.weight]
Loading weights:  99%|█████████▉| 396/398 [01:46<00:00,  2.78it/s, Materializing param=model.layers.35.self_attn.q_proj.weight]
Loading weights: 100%|█████████▉| 397/398 [01:46<00:00,  2.78it/s, Materializing param=model.layers.35.self_attn.v_proj.weight]
Loading weights: 100%|█████████▉| 397/398 [01:46<00:00,  2.78it/s, Materializing param=model.layers.35.self_attn.v_proj.weight]
Loading weights: 100%|██████████| 398/398 [01:46<00:00,  2.78it/s, Materializing param=model.norm.weight]                      
Loading weights: 100%|██████████| 398/398 [01:46<00:00,  2.78it/s, Materializing param=model.norm.weight]
Loading weights: 100%|██████████| 398/398 [01:46<00:00,  3.75it/s, Materializing param=model.norm.weight]
+[2026-03-08 15:56:07,195] [INFO] [axolotl.loaders.model._configure_embedding_dtypes:356] [PID:1659682] Converting modules to torch.bfloat16
+[2026-03-08 15:56:07,200] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:1659682] Memory usage after model load 8.942GB (+8.942GB allocated, +8.943GB reserved)
+[2026-03-08 15:56:53,452] [INFO] [axolotl.train.save_initial_configs:406] [PID:1659682] Pre-saving tokenizer to /home/jiaruil5/math_rl/mix_teachers/r3lit_rl/mix_teachers/checkpoints/math_operations/primitive_atomic_full_sft_50k_lr1e4_t20260308...
+[2026-03-08 15:56:53,543] [INFO] [axolotl.train.save_initial_configs:411] [PID:1659682] Pre-saving model config to /home/jiaruil5/math_rl/mix_teachers/r3lit_rl/mix_teachers/checkpoints/math_operations/primitive_atomic_full_sft_50k_lr1e4_t20260308...
+[2026-03-08 15:56:53,546] [INFO] [axolotl.train.execute_training:207] [PID:1659682] Starting trainer...
+[2026-03-08 15:56:55,512] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:1659682] generate_batches time: 0.5768194198608398
+[2026-03-08 15:56:56,114] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:1659682] generate_batches time: 0.6016864776611328
+[2026-03-08 15:56:56,687] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:1659682] generate_batches time: 0.5716288089752197
+[2026-03-08 15:56:57,255] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:1659682] generate_batches time: 0.5663418769836426
+[2026-03-08 15:56:57,255] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:1659682] gather_len_batches: [8334]
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: jiaruil5 (mathrl-jiarui) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: setting up run axzg81xq
+wandb: Tracking run with wandb version 0.25.0
+wandb: Run data is saved locally in /home/jiaruil5/math_rl/.wandb/run-20260308_155659-axzg81xq
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run primitive-atomic-full-sft-50k-lr1e4-t20260308
+wandb: ⭐️ View project at https://wandb.ai/mathrl-jiarui/math_operations_sft
+wandb: 🚀 View run at https://wandb.ai/mathrl-jiarui/math_operations_sft/runs/axzg81xq
+wandb: Detected [huggingface_hub.inference] in use.
+wandb: Use W&B Weave for improved LLM call tracing. Install Weave with `pip install weave` then add `import weave` to the top of your script.
+wandb: For more information, check out the docs at: https://weave-docs.wandb.ai/
+wandb: WARNING Saving files without folders. If you want to preserve subdirectories pass base_path to wandb.save, i.e. wandb.save("/mnt/folder/file.h5", base_path="/mnt")
+wandb: WARNING Symlinked 1 file into the W&B run directory; call wandb.save again to sync new files.
+[2026-03-08 15:57:13,948] [INFO] [axolotl.utils.callbacks.on_train_begin:757] [PID:1659682] The Axolotl config has been saved to the WandB run under files.
+
  0%|          | 0/3123 [00:00<?, ?it/s][2026-03-08 15:57:13,952] [INFO] [axolotl.core.trainers.base.evaluate:400] [PID:1659682] Running evaluation step...
+[2026-03-08 15:57:14,810] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:1659682] generate_batches time: 0.43496251106262207
+[2026-03-08 15:57:15,236] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:1659682] generate_batches time: 0.4260694980621338
+[2026-03-08 15:57:15,678] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:1659682] generate_batches time: 0.4417140483856201
+[2026-03-08 15:57:16,122] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:1659682] generate_batches time: 0.4431419372558594
+[2026-03-08 15:57:16,122] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:1659682] gather_len_batches: [34]
+
+
  0%|          | 0/34 [00:00<?, ?it/s]
+
  6%|▌         | 2/34 [00:00<00:03,  9.73it/s]
+
  9%|▉         | 3/34 [00:00<00:04,  6.34it/s]
+
 12%|█▏        | 4/34 [00:00<00:05,  5.37it/s]
+
 15%|█▍        | 5/34 [00:00<00:05,  4.93it/s]
+
 18%|█▊        | 6/34 [00:01<00:05,  4.68it/s]
+
 21%|██        | 7/34 [00:01<00:05,  4.53it/s]
+
 24%|██▎       | 8/34 [00:01<00:05,  4.45it/s]
+
 26%|██▋       | 9/34 [00:01<00:06,  4.13it/s]
+
 29%|██▉       | 10/34 [00:02<00:05,  4.18it/s]
+
 32%|███▏      | 11/34 [00:02<00:05,  4.21it/s]
+
 35%|███▌      | 12/34 [00:02<00:05,  4.22it/s]
+
 38%|███▊      | 13/34 [00:02<00:04,  4.23it/s]
+
 41%|████      | 14/34 [00:03<00:04,  4.24it/s]
+
 44%|████▍     | 15/34 [00:03<00:04,  4.25it/s]
+
 47%|████▋     | 16/34 [00:03<00:04,  4.27it/s]
+
 50%|█████     | 17/34 [00:03<00:04,  3.59it/s]
+
 53%|█████▎    | 18/34 [00:04<00:04,  3.77it/s]
+
 56%|█████▌    | 19/34 [00:04<00:03,  3.91it/s]
+
 59%|█████▉    | 20/34 [00:04<00:03,  4.00it/s]
+
 62%|██████▏   | 21/34 [00:04<00:03,  4.08it/s]
+
 65%|██████▍   | 22/34 [00:05<00:02,  4.12it/s]
+
 68%|██████▊   | 23/34 [00:05<00:02,  4.17it/s]
+
 71%|███████   | 24/34 [00:05<00:02,  4.20it/s]
+
 74%|███████▎  | 25/34 [00:06<00:03,  2.94it/s]
+
 76%|███████▋  | 26/34 [00:06<00:02,  3.30it/s]
+
 79%|███████▉  | 27/34 [00:06<00:01,  3.54it/s]
+
 82%|████████▏ | 28/34 [00:06<00:01,  3.73it/s]
+
 85%|████████▌ | 29/34 [00:07<00:01,  3.87it/s]
+
 88%|████████▊ | 30/34 [00:07<00:01,  3.97it/s]
+
 91%|█████████ | 31/34 [00:07<00:00,  4.05it/s]
+
 94%|█████████▍| 32/34 [00:07<00:00,  4.11it/s]
+
 97%|█████████▋| 33/34 [00:08<00:00,  3.43it/s]Traceback (most recent call last):
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/multiprocessing/util.py", line 300, in _run_finalizers
+    finalizer()
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/multiprocessing/util.py", line 224, in __call__
+    res = self._callback(*self._args, **self._kwargs)
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/multiprocessing/util.py", line 133, in _remove_temp_dir
+    rmtree(tempdir)
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/shutil.py", line 725, in rmtree
+    _rmtree_safe_fd(fd, path, onerror)
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/shutil.py", line 681, in _rmtree_safe_fd
+    onerror(os.unlink, fullname, sys.exc_info())
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/shutil.py", line 679, in _rmtree_safe_fd
+    os.unlink(entry.name, dir_fd=topfd)
+OSError: [Errno 16] Device or resource busy: '.nfs00000000000daa0800012505'
+
+
100%|██████████| 34/34 [00:08<00:00,  3.51it/s]
                                        
+
                                               
{'eval_loss': '0.892', 'eval_runtime': '17.36', 'eval_samples_per_second': '11.52', 'eval_steps_per_second': '11.52', 'eval_ppl': '2.44', 'memory/max_active (GiB)': '10.41', 'memory/max_allocated (GiB)': '10.41', 'memory/device_reserved (GiB)': '10.64', 'epoch': 0}
+
  0%|          | 0/3123 [00:19<?, ?it/s]
+
100%|██████████| 34/34 [00:08<00:00,  3.51it/s]
+
                                               
  0%|          | 1/3123 [00:34<29:53:48, 34.47s/it]
  0%|          | 2/3123 [00:42<16:11:34, 18.68s/it]
  0%|          | 3/3123 [00:49<11:49:52, 13.65s/it]
  0%|          | 4/3123 [00:57<9:55:09, 11.45s/it] 
  0%|          | 5/3123 [01:05<8:44:32, 10.09s/it]
  0%|          | 6/3123 [01:13<8:02:16,  9.28s/it]
  0%|          | 7/3123 [01:21<7:40:39,  8.87s/it]
  0%|          | 8/3123 [01:28<7:21:41,  8.51s/it]
  0%|          | 9/3123 [01:36<7:09:03,  8.27s/it]
  0%|          | 10/3123 [01:44<7:00:40,  8.11s/it]
                                                   
{'loss': '0.8231', 'grad_norm': '19.62', 'learning_rate': '2.885e-06', 'ppl': '2.277', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '85.28', 'tokens/total': 163840, 'tokens/trainable': 51937, 'epoch': '0.009599'}
+
  0%|          | 10/3123 [01:44<7:00:40,  8.11s/it]
  0%|          | 11/3123 [01:52<6:59:55,  8.10s/it]
  0%|          | 12/3123 [02:00<6:54:26,  7.99s/it]
  0%|          | 13/3123 [02:08<6:50:32,  7.92s/it]
  0%|          | 14/3123 [02:16<6:52:52,  7.97s/it]
  0%|          | 15/3123 [02:23<6:49:42,  7.91s/it]
  1%|          | 16/3123 [02:31<6:47:24,  7.87s/it]
  1%|          | 17/3123 [02:39<6:50:33,  7.93s/it]
  1%|          | 18/3123 [02:47<6:48:06,  7.89s/it]
  1%|          | 19/3123 [02:55<6:46:19,  7.85s/it]
  1%|          | 20/3123 [03:03<6:45:03,  7.83s/it]
                                                   
{'loss': '0.4511', 'grad_norm': '6.062', 'learning_rate': '6.09e-06', 'ppl': '1.57', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '85.76', 'tokens/total': 327680, 'tokens/trainable': 103351, 'epoch': '0.0192'}
+
  1%|          | 20/3123 [03:03<6:45:03,  7.83s/it]
  1%|          | 21/3123 [03:11<6:48:24,  7.90s/it]
  1%|          | 22/3123 [03:18<6:46:33,  7.87s/it]
  1%|          | 23/3123 [03:26<6:45:06,  7.84s/it]
  1%|          | 24/3123 [03:34<6:48:35,  7.91s/it]
  1%|          | 25/3123 [03:42<6:46:33,  7.87s/it]
  1%|          | 26/3123 [03:50<6:45:04,  7.85s/it]
  1%|          | 27/3123 [03:58<6:43:55,  7.83s/it]
  1%|          | 28/3123 [04:06<6:47:39,  7.90s/it]
  1%|          | 29/3123 [04:14<6:45:39,  7.87s/it]
  1%|          | 30/3123 [04:21<6:44:12,  7.84s/it]
                                                   
{'loss': '0.1399', 'grad_norm': '3.797', 'learning_rate': '9.295e-06', 'ppl': '1.15', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '83.34', 'tokens/total': 491520, 'tokens/trainable': 154482, 'epoch': '0.0288'}
+
  1%|          | 30/3123 [04:22<6:44:12,  7.84s/it]
  1%|          | 31/3123 [04:29<6:47:40,  7.91s/it]
  1%|          | 32/3123 [04:37<6:45:37,  7.87s/it]
  1%|          | 33/3123 [04:45<6:44:08,  7.85s/it]
  1%|          | 34/3123 [04:53<6:47:35,  7.92s/it]
  1%|          | 35/3123 [05:01<6:45:22,  7.88s/it]
  1%|          | 36/3123 [05:09<6:43:47,  7.85s/it]
  1%|          | 37/3123 [05:16<6:42:37,  7.83s/it]
  1%|          | 38/3123 [05:24<6:46:06,  7.90s/it]
  1%|          | 39/3123 [05:32<6:44:06,  7.86s/it]
  1%|▏         | 40/3123 [05:40<6:42:39,  7.84s/it]
                                                   
{'loss': '0.01782', 'grad_norm': '1.617', 'learning_rate': '1.25e-05', 'ppl': '1.018', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '77.78', 'tokens/total': 655360, 'tokens/trainable': 205247, 'epoch': '0.0384'}
+
  1%|▏         | 40/3123 [05:40<6:42:39,  7.84s/it]
  1%|▏         | 41/3123 [05:48<6:46:23,  7.91s/it]
  1%|▏         | 42/3123 [05:56<6:44:07,  7.87s/it]
  1%|▏         | 43/3123 [06:04<6:42:27,  7.84s/it]
  1%|▏         | 44/3123 [06:11<6:41:19,  7.82s/it]
  1%|▏         | 45/3123 [06:19<6:44:44,  7.89s/it]
  1%|▏         | 46/3123 [06:27<6:42:46,  7.85s/it]
  2%|▏         | 47/3123 [06:35<6:41:26,  7.83s/it]
  2%|▏         | 48/3123 [06:43<6:45:07,  7.90s/it]
  2%|▏         | 49/3123 [06:51<6:43:03,  7.87s/it]
  2%|▏         | 50/3123 [06:59<6:41:42,  7.84s/it]
                                                   
{'loss': '0.005615', 'grad_norm': '1.055', 'learning_rate': '1.571e-05', 'ppl': '1.006', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '97.31', 'tokens/total': 819200, 'tokens/trainable': 257032, 'epoch': '0.048'}
+
  2%|▏         | 50/3123 [06:59<6:41:42,  7.84s/it]
  2%|▏         | 51/3123 [07:07<6:45:08,  7.91s/it]
  2%|▏         | 52/3123 [07:15<6:43:06,  7.88s/it]
  2%|▏         | 53/3123 [07:22<6:41:35,  7.85s/it]
  2%|▏         | 54/3123 [07:30<6:40:26,  7.83s/it]
  2%|▏         | 55/3123 [07:38<6:43:59,  7.90s/it]
  2%|▏         | 56/3123 [07:46<6:41:55,  7.86s/it]
  2%|▏         | 57/3123 [07:54<6:40:21,  7.83s/it]
  2%|▏         | 58/3123 [08:02<6:43:29,  7.90s/it]
  2%|▏         | 59/3123 [08:09<6:41:21,  7.86s/it]
  2%|▏         | 60/3123 [08:17<6:39:53,  7.83s/it]
                                                   
{'loss': '0.003247', 'grad_norm': '0.7227', 'learning_rate': '1.891e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '77.57', 'tokens/total': 983040, 'tokens/trainable': 309407, 'epoch': '0.0576'}
+
  2%|▏         | 60/3123 [08:18<6:39:53,  7.83s/it]
  2%|▏         | 61/3123 [08:25<6:38:49,  7.81s/it]
  2%|▏         | 62/3123 [08:33<6:42:32,  7.89s/it]
  2%|▏         | 63/3123 [08:41<6:40:36,  7.85s/it]
  2%|▏         | 64/3123 [08:49<6:39:13,  7.83s/it]
  2%|▏         | 65/3123 [08:57<6:42:33,  7.90s/it]
  2%|▏         | 66/3123 [09:04<6:40:29,  7.86s/it]
  2%|▏         | 67/3123 [09:12<6:39:00,  7.83s/it]
  2%|▏         | 68/3123 [09:20<6:42:24,  7.90s/it]
  2%|▏         | 69/3123 [09:28<6:40:12,  7.86s/it]
  2%|▏         | 70/3123 [09:36<6:38:39,  7.83s/it]
                                                   
{'loss': '0.002711', 'grad_norm': '0.6094', 'learning_rate': '2.212e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '79.2', 'tokens/total': 1146880, 'tokens/trainable': 361385, 'epoch': '0.06719'}
+
  2%|▏         | 70/3123 [09:36<6:38:39,  7.83s/it]
  2%|▏         | 71/3123 [09:44<6:37:36,  7.82s/it]
  2%|▏         | 72/3123 [09:52<6:41:04,  7.89s/it]
  2%|▏         | 73/3123 [09:59<6:39:09,  7.85s/it]
  2%|▏         | 74/3123 [10:07<6:37:45,  7.83s/it]
  2%|▏         | 75/3123 [10:15<6:41:25,  7.90s/it]
  2%|▏         | 76/3123 [10:23<6:39:13,  7.86s/it]
  2%|▏         | 77/3123 [10:31<6:37:43,  7.83s/it]
  2%|▏         | 78/3123 [10:39<6:36:37,  7.82s/it]
  3%|▎         | 79/3123 [10:47<6:40:10,  7.89s/it]
  3%|▎         | 80/3123 [10:54<6:38:21,  7.85s/it]
                                                   
{'loss': '0.002677', 'grad_norm': '0.3594', 'learning_rate': '2.532e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '78.99', 'tokens/total': 1310720, 'tokens/trainable': 413011, 'epoch': '0.07679'}
+
  3%|▎         | 80/3123 [10:55<6:38:21,  7.85s/it]
  3%|▎         | 81/3123 [11:02<6:36:59,  7.83s/it]
  3%|▎         | 82/3123 [11:10<6:40:30,  7.90s/it]
  3%|▎         | 83/3123 [11:18<6:38:24,  7.86s/it]
  3%|▎         | 84/3123 [11:26<6:36:47,  7.83s/it]
  3%|▎         | 85/3123 [11:34<6:35:42,  7.82s/it]
  3%|▎         | 86/3123 [11:42<6:38:57,  7.88s/it]
  3%|▎         | 87/3123 [11:49<6:37:08,  7.85s/it]
  3%|▎         | 88/3123 [11:57<6:35:45,  7.82s/it]
  3%|▎         | 89/3123 [12:05<6:39:27,  7.90s/it]
  3%|▎         | 90/3123 [12:13<6:37:22,  7.86s/it]
                                                   
{'loss': '0.002554', 'grad_norm': '1.336', 'learning_rate': '2.853e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '78.37', 'tokens/total': 1474560, 'tokens/trainable': 464413, 'epoch': '0.08639'}
+
  3%|▎         | 90/3123 [12:13<6:37:22,  7.86s/it]
  3%|▎         | 91/3123 [12:21<6:35:53,  7.83s/it]
  3%|▎         | 92/3123 [12:29<6:39:08,  7.90s/it]
  3%|▎         | 93/3123 [12:37<6:36:57,  7.86s/it]
  3%|▎         | 94/3123 [12:44<6:35:31,  7.83s/it]
  3%|▎         | 95/3123 [12:52<6:34:29,  7.82s/it]
  3%|▎         | 96/3123 [13:00<6:38:31,  7.90s/it]
  3%|▎         | 97/3123 [13:08<6:36:24,  7.86s/it]
  3%|▎         | 98/3123 [13:16<6:34:57,  7.83s/it]
  3%|▎         | 99/3123 [13:24<6:38:14,  7.90s/it]
  3%|▎         | 100/3123 [13:32<6:36:14,  7.86s/it]
                                                    
{'loss': '0.003568', 'grad_norm': '0.1641', 'learning_rate': '3.173e-05', 'ppl': '1.004', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '75.5', 'tokens/total': 1638400, 'tokens/trainable': 516315, 'epoch': '0.09599'}
+
  3%|▎         | 100/3123 [13:32<6:36:14,  7.86s/it]
  3%|▎         | 101/3123 [13:39<6:34:40,  7.84s/it]
  3%|▎         | 102/3123 [13:47<6:33:38,  7.82s/it]
  3%|▎         | 103/3123 [13:55<6:37:14,  7.89s/it]
  3%|▎         | 104/3123 [14:03<6:35:19,  7.86s/it]
  3%|▎         | 105/3123 [14:11<6:33:54,  7.83s/it]
  3%|▎         | 106/3123 [14:19<6:37:29,  7.90s/it]
  3%|▎         | 107/3123 [14:27<6:35:24,  7.87s/it]
  3%|▎         | 108/3123 [14:34<6:33:57,  7.84s/it]
  3%|▎         | 109/3123 [14:42<6:32:49,  7.82s/it]
  4%|▎         | 110/3123 [14:50<6:36:32,  7.90s/it]
                                                    
{'loss': '0.002936', 'grad_norm': '0.7734', 'learning_rate': '3.494e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '83.41', 'tokens/total': 1802240, 'tokens/trainable': 568044, 'epoch': '0.1056'}
+
  4%|▎         | 110/3123 [14:51<6:36:32,  7.90s/it]
  4%|▎         | 111/3123 [14:58<6:34:37,  7.86s/it]
  4%|▎         | 112/3123 [15:06<6:33:14,  7.84s/it]
  4%|▎         | 113/3123 [15:14<6:36:24,  7.90s/it]
  4%|▎         | 114/3123 [15:22<6:34:20,  7.86s/it]
  4%|▎         | 115/3123 [15:29<6:32:56,  7.84s/it]
  4%|▎         | 116/3123 [15:38<6:36:23,  7.91s/it]
  4%|▎         | 117/3123 [15:45<6:34:15,  7.87s/it]
  4%|▍         | 118/3123 [15:53<6:32:47,  7.84s/it]
  4%|▍         | 119/3123 [16:01<6:31:35,  7.82s/it]
  4%|▍         | 120/3123 [16:09<6:35:05,  7.89s/it]
                                                    
{'loss': '0.002772', 'grad_norm': '0.334', 'learning_rate': '3.814e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '90.64', 'tokens/total': 1966080, 'tokens/trainable': 620018, 'epoch': '0.1152'}
+
  4%|▍         | 120/3123 [16:09<6:35:05,  7.89s/it]
  4%|▍         | 121/3123 [16:17<6:33:06,  7.86s/it]
  4%|▍         | 122/3123 [16:24<6:31:49,  7.83s/it]
  4%|▍         | 123/3123 [16:33<6:35:22,  7.91s/it]
  4%|▍         | 124/3123 [16:40<6:33:09,  7.87s/it]
  4%|▍         | 125/3123 [16:48<6:31:33,  7.84s/it]
  4%|▍         | 126/3123 [16:56<6:30:20,  7.81s/it]
  4%|▍         | 127/3123 [17:04<6:33:50,  7.89s/it]
  4%|▍         | 128/3123 [17:12<6:31:50,  7.85s/it]
  4%|▍         | 129/3123 [17:19<6:30:33,  7.83s/it]
  4%|▍         | 130/3123 [17:27<6:33:51,  7.90s/it]
                                                    
{'loss': '0.002472', 'grad_norm': '0.2021', 'learning_rate': '4.135e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '70.85', 'tokens/total': 2129920, 'tokens/trainable': 670706, 'epoch': '0.1248'}
+
  4%|▍         | 130/3123 [17:28<6:33:51,  7.90s/it]
  4%|▍         | 131/3123 [17:35<6:31:56,  7.86s/it]
  4%|▍         | 132/3123 [17:43<6:30:22,  7.83s/it]
  4%|▍         | 133/3123 [17:51<6:33:35,  7.90s/it]
  4%|▍         | 134/3123 [17:59<6:31:25,  7.86s/it]
  4%|▍         | 135/3123 [18:07<6:30:01,  7.83s/it]
  4%|▍         | 136/3123 [18:14<6:29:00,  7.81s/it]
  4%|▍         | 137/3123 [18:22<6:32:39,  7.89s/it]
  4%|▍         | 138/3123 [18:30<6:30:38,  7.85s/it]
  4%|▍         | 139/3123 [18:38<6:29:18,  7.83s/it]
  4%|▍         | 140/3123 [18:46<6:32:20,  7.89s/it]
                                                    
{'loss': '0.003623', 'grad_norm': '0.3379', 'learning_rate': '4.455e-05', 'ppl': '1.004', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '81.86', 'tokens/total': 2293760, 'tokens/trainable': 722137, 'epoch': '0.1344'}
+
  4%|▍         | 140/3123 [18:46<6:32:20,  7.89s/it]
  5%|▍         | 141/3123 [18:54<6:30:24,  7.86s/it]
  5%|▍         | 142/3123 [19:02<6:29:00,  7.83s/it]
  5%|▍         | 143/3123 [19:09<6:28:00,  7.81s/it]
  5%|▍         | 144/3123 [19:17<6:31:38,  7.89s/it]
  5%|▍         | 145/3123 [19:25<6:29:49,  7.85s/it]
  5%|▍         | 146/3123 [19:33<6:28:21,  7.83s/it]
  5%|▍         | 147/3123 [19:41<6:31:23,  7.89s/it]
  5%|▍         | 148/3123 [19:49<6:29:22,  7.85s/it]
  5%|▍         | 149/3123 [19:57<6:27:57,  7.83s/it]
  5%|▍         | 150/3123 [20:05<6:31:13,  7.90s/it]
                                                    
{'loss': '0.002823', 'grad_norm': '0.2988', 'learning_rate': '4.776e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '74.85', 'tokens/total': 2457600, 'tokens/trainable': 773351, 'epoch': '0.144'}
+
  5%|▍         | 150/3123 [20:05<6:31:13,  7.90s/it]
  5%|▍         | 151/3123 [20:12<6:29:08,  7.86s/it]
  5%|▍         | 152/3123 [20:20<6:27:43,  7.83s/it]
  5%|▍         | 153/3123 [20:28<6:26:43,  7.81s/it]
  5%|▍         | 154/3123 [20:36<6:30:15,  7.89s/it]
  5%|▍         | 155/3123 [20:44<6:28:14,  7.85s/it]
  5%|▍         | 156/3123 [20:51<6:27:04,  7.83s/it]
  5%|▌         | 157/3123 [21:00<6:30:25,  7.90s/it]
  5%|▌         | 158/3123 [21:07<6:28:20,  7.86s/it]
  5%|▌         | 159/3123 [21:15<6:26:38,  7.83s/it]
  5%|▌         | 160/3123 [21:23<6:25:41,  7.81s/it]
                                                    
{'loss': '0.001758', 'grad_norm': '0.2256', 'learning_rate': '5.096e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '97.67', 'tokens/total': 2621440, 'tokens/trainable': 824611, 'epoch': '0.1536'}
+
  5%|▌         | 160/3123 [21:23<6:25:41,  7.81s/it]
  5%|▌         | 161/3123 [21:31<6:29:03,  7.88s/it]
  5%|▌         | 162/3123 [21:39<6:27:18,  7.85s/it]
  5%|▌         | 163/3123 [21:46<6:25:58,  7.82s/it]
  5%|▌         | 164/3123 [21:54<6:29:15,  7.89s/it]
  5%|▌         | 165/3123 [22:02<6:27:14,  7.85s/it]
  5%|▌         | 166/3123 [22:10<6:25:49,  7.83s/it]
  5%|▌         | 167/3123 [22:18<6:28:47,  7.89s/it]
  5%|▌         | 168/3123 [22:26<6:26:50,  7.85s/it]
  5%|▌         | 169/3123 [22:34<6:25:21,  7.83s/it]
  5%|▌         | 170/3123 [22:41<6:24:19,  7.81s/it]
                                                    
{'loss': '0.002541', 'grad_norm': '0.2168', 'learning_rate': '5.417e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '87.14', 'tokens/total': 2785280, 'tokens/trainable': 875746, 'epoch': '0.1632'}
+
  5%|▌         | 170/3123 [22:42<6:24:19,  7.81s/it]
  5%|▌         | 171/3123 [22:49<6:27:54,  7.88s/it]
  6%|▌         | 172/3123 [22:57<6:26:00,  7.85s/it]
  6%|▌         | 173/3123 [23:05<6:24:40,  7.82s/it]
  6%|▌         | 174/3123 [23:13<6:27:45,  7.89s/it]
  6%|▌         | 175/3123 [23:21<6:25:53,  7.85s/it]
  6%|▌         | 176/3123 [23:28<6:24:24,  7.83s/it]
  6%|▌         | 177/3123 [23:36<6:23:25,  7.81s/it]
  6%|▌         | 178/3123 [23:44<6:26:32,  7.88s/it]
  6%|▌         | 179/3123 [23:52<6:24:45,  7.84s/it]
  6%|▌         | 180/3123 [24:00<6:23:24,  7.82s/it]
                                                    
{'loss': '0.003716', 'grad_norm': '0.3105', 'learning_rate': '5.737e-05', 'ppl': '1.004', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '78.77', 'tokens/total': 2949120, 'tokens/trainable': 927000, 'epoch': '0.1728'}
+
  6%|▌         | 180/3123 [24:00<6:23:24,  7.82s/it]
  6%|▌         | 181/3123 [24:08<6:27:00,  7.89s/it]
  6%|▌         | 182/3123 [24:16<6:24:59,  7.85s/it]
  6%|▌         | 183/3123 [24:23<6:23:37,  7.83s/it]
  6%|▌         | 184/3123 [24:31<6:26:38,  7.89s/it]
  6%|▌         | 185/3123 [24:39<6:24:48,  7.86s/it]
  6%|▌         | 186/3123 [24:47<6:23:18,  7.83s/it]
  6%|▌         | 187/3123 [24:55<6:22:12,  7.81s/it]
  6%|▌         | 188/3123 [25:03<6:25:27,  7.88s/it]
  6%|▌         | 189/3123 [25:11<6:23:39,  7.85s/it]
  6%|▌         | 190/3123 [25:18<6:22:23,  7.82s/it]
                                                    
{'loss': '0.002808', 'grad_norm': '0.3125', 'learning_rate': '6.058e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '90.6', 'tokens/total': 3112960, 'tokens/trainable': 978689, 'epoch': '0.1824'}
+
  6%|▌         | 190/3123 [25:19<6:22:23,  7.82s/it]
  6%|▌         | 191/3123 [25:26<6:25:48,  7.90s/it]
  6%|▌         | 192/3123 [25:34<6:23:44,  7.86s/it]
  6%|▌         | 193/3123 [25:42<6:22:19,  7.83s/it]
  6%|▌         | 194/3123 [25:50<6:21:17,  7.81s/it]
  6%|▌         | 195/3123 [25:58<6:24:32,  7.88s/it]
  6%|▋         | 196/3123 [26:06<6:22:40,  7.84s/it]
  6%|▋         | 197/3123 [26:13<6:21:25,  7.82s/it]
  6%|▋         | 198/3123 [26:21<6:24:32,  7.89s/it]
  6%|▋         | 199/3123 [26:29<6:22:41,  7.85s/it]
  6%|▋         | 200/3123 [26:37<6:21:23,  7.83s/it]
                                                    
{'loss': '0.05687', 'grad_norm': '2.766', 'learning_rate': '6.378e-05', 'ppl': '1.059', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '91.21', 'tokens/total': 3276800, 'tokens/trainable': 1029912, 'epoch': '0.192'}
+
  6%|▋         | 200/3123 [26:37<6:21:23,  7.83s/it]
  6%|▋         | 201/3123 [26:45<6:24:53,  7.90s/it]
  6%|▋         | 202/3123 [26:53<6:22:37,  7.86s/it]
  7%|▋         | 203/3123 [27:00<6:21:13,  7.83s/it]
  7%|▋         | 204/3123 [27:08<6:20:03,  7.81s/it]
  7%|▋         | 205/3123 [27:16<6:23:05,  7.88s/it]
  7%|▋         | 206/3123 [27:24<6:21:23,  7.84s/it]
  7%|▋         | 207/3123 [27:32<6:20:01,  7.82s/it]
  7%|▋         | 208/3123 [27:40<6:23:33,  7.89s/it]
  7%|▋         | 209/3123 [27:48<6:21:28,  7.85s/it]
  7%|▋         | 210/3123 [27:55<6:20:08,  7.83s/it]
                                                    
{'loss': '0.02166', 'grad_norm': '2.047', 'learning_rate': '6.699e-05', 'ppl': '1.022', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '86', 'tokens/total': 3440640, 'tokens/trainable': 1081023, 'epoch': '0.2016'}
+
  7%|▋         | 210/3123 [27:56<6:20:08,  7.83s/it]
  7%|▋         | 211/3123 [28:03<6:19:02,  7.81s/it]
  7%|▋         | 212/3123 [28:11<6:22:30,  7.88s/it]
  7%|▋         | 213/3123 [28:19<6:20:38,  7.85s/it]
  7%|▋         | 214/3123 [28:27<6:19:22,  7.82s/it]
  7%|▋         | 215/3123 [28:35<6:22:46,  7.90s/it]
  7%|▋         | 216/3123 [28:43<6:20:47,  7.86s/it]
  7%|▋         | 217/3123 [28:50<6:19:09,  7.83s/it]
  7%|▋         | 218/3123 [28:58<6:18:12,  7.81s/it]
  7%|▋         | 219/3123 [29:06<6:21:35,  7.88s/it]
  7%|▋         | 220/3123 [29:14<6:19:57,  7.85s/it]
                                                    
{'loss': '0.0128', 'grad_norm': '2.219', 'learning_rate': '7.019e-05', 'ppl': '1.013', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '92.72', 'tokens/total': 3604480, 'tokens/trainable': 1132719, 'epoch': '0.2112'}
+
  7%|▋         | 220/3123 [29:14<6:19:57,  7.85s/it]
  7%|▋         | 221/3123 [29:22<6:18:39,  7.83s/it]
  7%|▋         | 222/3123 [29:30<6:22:04,  7.90s/it]
  7%|▋         | 223/3123 [29:38<6:19:58,  7.86s/it]
  7%|▋         | 224/3123 [29:45<6:18:32,  7.83s/it]
  7%|▋         | 225/3123 [29:53<6:21:26,  7.90s/it]
  7%|▋         | 226/3123 [30:01<6:19:32,  7.86s/it]
  7%|▋         | 227/3123 [30:09<6:18:01,  7.83s/it]
  7%|▋         | 228/3123 [30:17<6:17:05,  7.82s/it]
  7%|▋         | 229/3123 [30:25<6:20:54,  7.90s/it]
  7%|▋         | 230/3123 [30:33<6:18:54,  7.86s/it]
                                                    
{'loss': '0.01219', 'grad_norm': '0.5312', 'learning_rate': '7.34e-05', 'ppl': '1.012', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '83.89', 'tokens/total': 3768320, 'tokens/trainable': 1184217, 'epoch': '0.2208'}
+
  7%|▋         | 230/3123 [30:33<6:18:54,  7.86s/it]
  7%|▋         | 231/3123 [30:40<6:17:32,  7.83s/it]
  7%|▋         | 232/3123 [30:48<6:20:44,  7.90s/it]
  7%|▋         | 233/3123 [30:56<6:18:46,  7.86s/it]
  7%|▋         | 234/3123 [31:04<6:17:21,  7.84s/it]
  8%|▊         | 235/3123 [31:12<6:16:24,  7.82s/it]
  8%|▊         | 236/3123 [31:20<6:19:45,  7.89s/it]
  8%|▊         | 237/3123 [31:28<6:17:54,  7.86s/it]
  8%|▊         | 238/3123 [31:35<6:16:25,  7.83s/it]
  8%|▊         | 239/3123 [31:43<6:19:46,  7.90s/it]
  8%|▊         | 240/3123 [31:51<6:17:48,  7.86s/it]
                                                    
{'loss': '0.01126', 'grad_norm': '4.062', 'learning_rate': '7.66e-05', 'ppl': '1.011', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '75.15', 'tokens/total': 3932160, 'tokens/trainable': 1235669, 'epoch': '0.2304'}
+
  8%|▊         | 240/3123 [31:51<6:17:48,  7.86s/it]
  8%|▊         | 241/3123 [31:59<6:16:27,  7.84s/it]
  8%|▊         | 242/3123 [32:07<6:19:55,  7.91s/it]
  8%|▊         | 243/3123 [32:15<6:17:57,  7.87s/it]
  8%|▊         | 244/3123 [32:23<6:16:24,  7.84s/it]
  8%|▊         | 245/3123 [32:30<6:15:11,  7.82s/it]
  8%|▊         | 246/3123 [32:38<6:18:30,  7.89s/it]
  8%|▊         | 247/3123 [32:46<6:16:44,  7.86s/it]
  8%|▊         | 248/3123 [32:54<6:15:16,  7.83s/it]
  8%|▊         | 249/3123 [33:02<6:18:36,  7.90s/it]
  8%|▊         | 250/3123 [33:10<6:16:34,  7.86s/it]
                                                    
{'loss': '0.01502', 'grad_norm': '22.12', 'learning_rate': '7.981e-05', 'ppl': '1.015', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '88.15', 'tokens/total': 4096000, 'tokens/trainable': 1287445, 'epoch': '0.24'}
+
  8%|▊         | 250/3123 [33:10<6:16:34,  7.86s/it]
  8%|▊         | 251/3123 [33:18<6:15:13,  7.84s/it]
  8%|▊         | 252/3123 [33:25<6:14:05,  7.82s/it]
  8%|▊         | 253/3123 [33:33<6:17:20,  7.89s/it]
  8%|▊         | 254/3123 [33:41<6:15:35,  7.85s/it]
  8%|▊         | 255/3123 [33:49<6:14:25,  7.83s/it]
  8%|▊         | 256/3123 [33:57<6:17:35,  7.90s/it]
  8%|▊         | 257/3123 [34:05<6:15:46,  7.87s/it]
  8%|▊         | 258/3123 [34:13<6:14:16,  7.84s/it]
  8%|▊         | 259/3123 [34:21<6:17:10,  7.90s/it]
  8%|▊         | 260/3123 [34:28<6:15:20,  7.87s/it]
                                                    
{'loss': '0.01258', 'grad_norm': '0.6836', 'learning_rate': '8.301e-05', 'ppl': '1.013', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '92.44', 'tokens/total': 4259840, 'tokens/trainable': 1339971, 'epoch': '0.2496'}
+
  8%|▊         | 260/3123 [34:29<6:15:20,  7.87s/it]
  8%|▊         | 261/3123 [34:36<6:13:59,  7.84s/it]
  8%|▊         | 262/3123 [34:44<6:12:52,  7.82s/it]
  8%|▊         | 263/3123 [34:52<6:16:24,  7.90s/it]
  8%|▊         | 264/3123 [35:00<6:14:41,  7.86s/it]
  8%|▊         | 265/3123 [35:08<6:13:16,  7.84s/it]
  9%|▊         | 266/3123 [35:16<6:16:15,  7.90s/it]
  9%|▊         | 267/3123 [35:23<6:14:17,  7.86s/it]
  9%|▊         | 268/3123 [35:31<6:12:54,  7.84s/it]
  9%|▊         | 269/3123 [35:39<6:11:49,  7.82s/it]
  9%|▊         | 270/3123 [35:47<6:15:13,  7.89s/it]
                                                    
{'loss': '0.01926', 'grad_norm': '5.375', 'learning_rate': '8.622e-05', 'ppl': '1.019', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '72.57', 'tokens/total': 4423680, 'tokens/trainable': 1391647, 'epoch': '0.2592'}
+
  9%|▊         | 270/3123 [35:47<6:15:13,  7.89s/it]
  9%|▊         | 271/3123 [35:55<6:13:21,  7.85s/it]
  9%|▊         | 272/3123 [36:03<6:12:02,  7.83s/it]
  9%|▊         | 273/3123 [36:11<6:15:00,  7.89s/it]
  9%|▉         | 274/3123 [36:18<6:13:09,  7.86s/it]
  9%|▉         | 275/3123 [36:26<6:11:49,  7.83s/it]
  9%|▉         | 276/3123 [36:34<6:14:59,  7.90s/it]
  9%|▉         | 277/3123 [36:42<6:12:49,  7.86s/it]
  9%|▉         | 278/3123 [36:50<6:11:19,  7.83s/it]
  9%|▉         | 279/3123 [36:58<6:10:15,  7.81s/it]
  9%|▉         | 280/3123 [37:06<6:13:33,  7.88s/it]
                                                    
{'loss': '0.03442', 'grad_norm': '0.6016', 'learning_rate': '8.942e-05', 'ppl': '1.035', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '80.7', 'tokens/total': 4587520, 'tokens/trainable': 1443110, 'epoch': '0.2688'}
+
  9%|▉         | 280/3123 [37:06<6:13:33,  7.88s/it]
  9%|▉         | 281/3123 [37:13<6:11:46,  7.85s/it]
  9%|▉         | 282/3123 [37:21<6:10:36,  7.83s/it]
  9%|▉         | 283/3123 [37:29<6:13:52,  7.90s/it]
  9%|▉         | 284/3123 [37:37<6:11:52,  7.86s/it]
  9%|▉         | 285/3123 [37:45<6:10:25,  7.83s/it]
  9%|▉         | 286/3123 [37:53<6:13:17,  7.89s/it]
  9%|▉         | 287/3123 [38:01<6:11:18,  7.86s/it]
  9%|▉         | 288/3123 [38:08<6:10:02,  7.83s/it]
  9%|▉         | 289/3123 [38:16<6:09:02,  7.81s/it]
  9%|▉         | 290/3123 [38:24<6:12:27,  7.89s/it]
                                                    
{'loss': '0.01742', 'grad_norm': '1.781', 'learning_rate': '9.263e-05', 'ppl': '1.018', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '87.54', 'tokens/total': 4751360, 'tokens/trainable': 1495030, 'epoch': '0.2784'}
+
  9%|▉         | 290/3123 [38:24<6:12:27,  7.89s/it]
  9%|▉         | 291/3123 [38:32<6:10:40,  7.85s/it]
  9%|▉         | 292/3123 [38:40<6:09:21,  7.83s/it]
  9%|▉         | 293/3123 [38:48<6:12:18,  7.89s/it]
  9%|▉         | 294/3123 [38:55<6:10:13,  7.85s/it]
  9%|▉         | 295/3123 [39:03<6:08:55,  7.83s/it]
  9%|▉         | 296/3123 [39:11<6:07:56,  7.81s/it]
 10%|▉         | 297/3123 [39:19<6:11:14,  7.88s/it]
 10%|▉         | 298/3123 [39:27<6:09:26,  7.85s/it]
 10%|▉         | 299/3123 [39:35<6:08:08,  7.82s/it]
 10%|▉         | 300/3123 [39:43<6:11:13,  7.89s/it]
                                                    
{'loss': '0.01093', 'grad_norm': '0.3438', 'learning_rate': '9.583e-05', 'ppl': '1.011', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '86.53', 'tokens/total': 4915200, 'tokens/trainable': 1545313, 'epoch': '0.288'}
+
 10%|▉         | 300/3123 [39:43<6:11:13,  7.89s/it]
 10%|▉         | 301/3123 [39:50<6:09:23,  7.85s/it]
 10%|▉         | 302/3123 [39:58<6:08:01,  7.83s/it]
 10%|▉         | 303/3123 [40:06<6:07:07,  7.81s/it]
 10%|▉         | 304/3123 [40:14<6:10:27,  7.89s/it]
 10%|▉         | 305/3123 [40:22<6:08:42,  7.85s/it]
 10%|▉         | 306/3123 [40:30<6:07:19,  7.82s/it]
 10%|▉         | 307/3123 [40:38<6:10:26,  7.89s/it]
 10%|▉         | 308/3123 [40:45<6:08:33,  7.86s/it]
 10%|▉         | 309/3123 [40:53<6:07:12,  7.83s/it]
 10%|▉         | 310/3123 [41:01<6:06:12,  7.81s/it]
                                                    
{'loss': '0.01098', 'grad_norm': '0.3125', 'learning_rate': '9.904e-05', 'ppl': '1.011', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '93.02', 'tokens/total': 5079040, 'tokens/trainable': 1596578, 'epoch': '0.2976'}
+
 10%|▉         | 310/3123 [41:01<6:06:12,  7.81s/it]
 10%|▉         | 311/3123 [41:09<6:09:24,  7.88s/it]
 10%|▉         | 312/3123 [41:17<6:07:37,  7.85s/it]
 10%|█         | 313/3123 [41:25<6:06:29,  7.83s/it]
 10%|█         | 314/3123 [41:33<6:09:24,  7.89s/it]
 10%|█         | 315/3123 [41:40<6:07:37,  7.86s/it]
 10%|█         | 316/3123 [41:48<6:06:15,  7.83s/it]
 10%|█         | 317/3123 [41:56<6:09:16,  7.90s/it]
 10%|█         | 318/3123 [42:04<6:07:19,  7.86s/it]
 10%|█         | 319/3123 [42:12<6:05:54,  7.83s/it]
 10%|█         | 320/3123 [42:19<6:05:01,  7.81s/it]
                                                    
{'loss': '0.007864', 'grad_norm': '0.2305', 'learning_rate': '0.0001', 'ppl': '1.008', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '86.5', 'tokens/total': 5242880, 'tokens/trainable': 1648111, 'epoch': '0.3072'}
+
 10%|█         | 320/3123 [42:20<6:05:01,  7.81s/it]
 10%|█         | 321/3123 [42:27<6:08:12,  7.88s/it]
 10%|█         | 322/3123 [42:35<6:06:30,  7.85s/it]
 10%|█         | 323/3123 [42:43<6:05:13,  7.83s/it]
 10%|█         | 324/3123 [42:51<6:08:15,  7.89s/it]
 10%|█         | 325/3123 [42:59<6:06:23,  7.86s/it]
 10%|█         | 326/3123 [43:07<6:04:59,  7.83s/it]
 10%|█         | 327/3123 [43:15<6:07:48,  7.89s/it]
 11%|█         | 328/3123 [43:22<6:06:01,  7.86s/it]
 11%|█         | 329/3123 [43:30<6:04:39,  7.83s/it]
 11%|█         | 330/3123 [43:38<6:03:37,  7.81s/it]
                                                    
{'loss': '0.005257', 'grad_norm': '0.4141', 'learning_rate': '0.0001', 'ppl': '1.005', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '89.96', 'tokens/total': 5406720, 'tokens/trainable': 1699588, 'epoch': '0.3168'}
+
 11%|█         | 330/3123 [43:38<6:03:37,  7.81s/it]
 11%|█         | 331/3123 [43:46<6:07:01,  7.89s/it]
 11%|█         | 332/3123 [43:54<6:05:23,  7.86s/it]
 11%|█         | 333/3123 [44:02<6:04:02,  7.83s/it]
 11%|█         | 334/3123 [44:10<6:07:06,  7.90s/it]
 11%|█         | 335/3123 [44:17<6:05:06,  7.86s/it]
 11%|█         | 336/3123 [44:25<6:03:50,  7.83s/it]
 11%|█         | 337/3123 [44:33<6:02:50,  7.81s/it]
 11%|█         | 338/3123 [44:41<6:06:00,  7.89s/it]
 11%|█         | 339/3123 [44:49<6:04:16,  7.85s/it]
 11%|█         | 340/3123 [44:57<6:03:04,  7.83s/it]
                                                    
{'loss': '0.009608', 'grad_norm': '0.2969', 'learning_rate': '0.0001', 'ppl': '1.01', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '91.73', 'tokens/total': 5570560, 'tokens/trainable': 1751730, 'epoch': '0.3264'}
+
 11%|█         | 340/3123 [44:57<6:03:04,  7.83s/it]
 11%|█         | 341/3123 [45:05<6:06:19,  7.90s/it]
 11%|█         | 342/3123 [45:12<6:04:21,  7.86s/it]
 11%|█         | 343/3123 [45:20<6:02:56,  7.83s/it]
 11%|█         | 344/3123 [45:28<6:06:07,  7.90s/it]
 11%|█         | 345/3123 [45:36<6:04:03,  7.86s/it]
 11%|█         | 346/3123 [45:44<6:02:36,  7.83s/it]
 11%|█         | 347/3123 [45:52<6:01:41,  7.82s/it]
 11%|█         | 348/3123 [46:00<6:04:52,  7.89s/it]
 11%|█         | 349/3123 [46:07<6:03:06,  7.85s/it]
 11%|█         | 350/3123 [46:15<6:01:48,  7.83s/it]
                                                    
{'loss': '0.007573', 'grad_norm': '0.3867', 'learning_rate': '0.0001', 'ppl': '1.008', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '90.04', 'tokens/total': 5734400, 'tokens/trainable': 1802979, 'epoch': '0.336'}
+
 11%|█         | 350/3123 [46:15<6:01:48,  7.83s/it]
 11%|█         | 351/3123 [46:23<6:05:01,  7.90s/it]
 11%|█▏        | 352/3123 [46:31<6:03:01,  7.86s/it]
 11%|█▏        | 353/3123 [46:39<6:01:40,  7.83s/it]
 11%|█▏        | 354/3123 [46:47<6:00:33,  7.81s/it]
 11%|█▏        | 355/3123 [46:55<6:03:49,  7.89s/it]
 11%|█▏        | 356/3123 [47:02<6:02:02,  7.85s/it]
 11%|█▏        | 357/3123 [47:10<6:00:44,  7.83s/it]
 11%|█▏        | 358/3123 [47:18<6:03:54,  7.90s/it]
 11%|█▏        | 359/3123 [47:26<6:02:00,  7.86s/it]
 12%|█▏        | 360/3123 [47:34<6:00:39,  7.83s/it]
                                                    
{'loss': '0.006454', 'grad_norm': '0.2148', 'learning_rate': '0.0001', 'ppl': '1.006', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '93.79', 'tokens/total': 5898240, 'tokens/trainable': 1853732, 'epoch': '0.3456'}
+
 12%|█▏        | 360/3123 [47:34<6:00:39,  7.83s/it]
 12%|█▏        | 361/3123 [47:42<6:03:36,  7.90s/it]
 12%|█▏        | 362/3123 [47:50<6:01:32,  7.86s/it]
 12%|█▏        | 363/3123 [47:57<6:00:14,  7.83s/it]
 12%|█▏        | 364/3123 [48:05<5:59:13,  7.81s/it]
 12%|█▏        | 365/3123 [48:13<6:02:26,  7.89s/it]
 12%|█▏        | 366/3123 [48:21<6:00:35,  7.85s/it]
 12%|█▏        | 367/3123 [48:29<5:59:20,  7.82s/it]
 12%|█▏        | 368/3123 [48:37<6:02:12,  7.89s/it]
 12%|█▏        | 369/3123 [48:44<6:00:25,  7.85s/it]
 12%|█▏        | 370/3123 [48:52<5:59:03,  7.83s/it]
                                                    
{'loss': '0.00607', 'grad_norm': '0.1699', 'learning_rate': '0.0001', 'ppl': '1.006', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '82.21', 'tokens/total': 6062080, 'tokens/trainable': 1904439, 'epoch': '0.3552'}
+
 12%|█▏        | 370/3123 [48:53<5:59:03,  7.83s/it]
 12%|█▏        | 371/3123 [49:00<5:58:08,  7.81s/it]
 12%|█▏        | 372/3123 [49:08<6:01:33,  7.89s/it]
 12%|█▏        | 373/3123 [49:16<5:59:44,  7.85s/it]
 12%|█▏        | 374/3123 [49:24<5:58:32,  7.83s/it]
 12%|█▏        | 375/3123 [49:32<6:01:28,  7.89s/it]
 12%|█▏        | 376/3123 [49:39<5:59:35,  7.85s/it]
 12%|█▏        | 377/3123 [49:47<5:58:17,  7.83s/it]
 12%|█▏        | 378/3123 [49:55<5:57:11,  7.81s/it]
 12%|█▏        | 379/3123 [50:03<6:00:21,  7.88s/it]
 12%|█▏        | 380/3123 [50:11<5:58:45,  7.85s/it]
                                                    
{'loss': '0.005776', 'grad_norm': '0.1436', 'learning_rate': '0.0001', 'ppl': '1.006', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '82.25', 'tokens/total': 6225920, 'tokens/trainable': 1955620, 'epoch': '0.3648'}
+
 12%|█▏        | 380/3123 [50:11<5:58:45,  7.85s/it]
 12%|█▏        | 381/3123 [50:19<5:57:39,  7.83s/it]
 12%|█▏        | 382/3123 [50:27<6:01:01,  7.90s/it]
 12%|█▏        | 383/3123 [50:34<5:58:58,  7.86s/it]
 12%|█▏        | 384/3123 [50:42<5:57:36,  7.83s/it]
 12%|█▏        | 385/3123 [50:50<6:00:37,  7.90s/it]
 12%|█▏        | 386/3123 [50:58<5:58:42,  7.86s/it]
 12%|█▏        | 387/3123 [51:06<5:57:17,  7.84s/it]
 12%|█▏        | 388/3123 [51:14<5:56:20,  7.82s/it]
 12%|█▏        | 389/3123 [51:22<5:59:34,  7.89s/it]
 12%|█▏        | 390/3123 [51:29<5:57:54,  7.86s/it]
                                                    
{'loss': '0.005018', 'grad_norm': '0.1514', 'learning_rate': '0.0001', 'ppl': '1.005', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '98.77', 'tokens/total': 6389760, 'tokens/trainable': 2007340, 'epoch': '0.3744'}
+
 12%|█▏        | 390/3123 [51:30<5:57:54,  7.86s/it]
 13%|█▎        | 391/3123 [51:37<5:56:34,  7.83s/it]
 13%|█▎        | 392/3123 [51:45<5:59:41,  7.90s/it]
 13%|█▎        | 393/3123 [51:53<5:57:57,  7.87s/it]
 13%|█▎        | 394/3123 [52:01<5:56:39,  7.84s/it]
 13%|█▎        | 395/3123 [52:09<5:55:41,  7.82s/it]
 13%|█▎        | 396/3123 [52:17<5:59:12,  7.90s/it]
 13%|█▎        | 397/3123 [52:24<5:57:18,  7.86s/it]
 13%|█▎        | 398/3123 [52:32<5:55:54,  7.84s/it]
 13%|█▎        | 399/3123 [52:40<5:58:51,  7.90s/it]
 13%|█▎        | 400/3123 [52:48<5:56:56,  7.86s/it]
                                                    
{'loss': '0.003989', 'grad_norm': '0.2383', 'learning_rate': '0.0001', 'ppl': '1.004', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '79.82', 'tokens/total': 6553600, 'tokens/trainable': 2059592, 'epoch': '0.384'}
+
 13%|█▎        | 400/3123 [52:48<5:56:56,  7.86s/it]
 13%|█▎        | 401/3123 [52:56<5:55:39,  7.84s/it]
 13%|█▎        | 402/3123 [53:04<5:54:41,  7.82s/it]
 13%|█▎        | 403/3123 [53:12<5:57:57,  7.90s/it]
 13%|█▎        | 404/3123 [53:19<5:56:10,  7.86s/it]
 13%|█▎        | 405/3123 [53:27<5:54:56,  7.84s/it]
 13%|█▎        | 406/3123 [53:35<5:58:08,  7.91s/it]
 13%|█▎        | 407/3123 [53:43<5:56:09,  7.87s/it]
 13%|█▎        | 408/3123 [53:51<5:54:42,  7.84s/it]
 13%|█▎        | 409/3123 [53:59<5:57:53,  7.91s/it]
 13%|█▎        | 410/3123 [54:07<5:55:49,  7.87s/it]
                                                    
{'loss': '0.004139', 'grad_norm': '0.3965', 'learning_rate': '0.0001', 'ppl': '1.004', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '81.19', 'tokens/total': 6717440, 'tokens/trainable': 2111025, 'epoch': '0.3936'}
+
 13%|█▎        | 410/3123 [54:07<5:55:49,  7.87s/it]
 13%|█▎        | 411/3123 [54:14<5:54:26,  7.84s/it]
 13%|█▎        | 412/3123 [54:22<5:53:20,  7.82s/it]
 13%|█▎        | 413/3123 [54:30<5:56:22,  7.89s/it]
 13%|█▎        | 414/3123 [54:38<5:54:38,  7.85s/it]
 13%|█▎        | 415/3123 [54:46<5:53:24,  7.83s/it]
 13%|█▎        | 416/3123 [54:54<5:56:52,  7.91s/it]
 13%|█▎        | 417/3123 [55:02<5:54:52,  7.87s/it]
 13%|█▎        | 418/3123 [55:09<5:53:25,  7.84s/it]
 13%|█▎        | 419/3123 [55:17<5:52:36,  7.82s/it]
 13%|█▎        | 420/3123 [55:25<5:55:48,  7.90s/it]
                                                    
{'loss': '0.004085', 'grad_norm': '0.2656', 'learning_rate': '0.0001', 'ppl': '1.004', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '90.93', 'tokens/total': 6881280, 'tokens/trainable': 2161939, 'epoch': '0.4032'}
+
 13%|█▎        | 420/3123 [55:26<5:55:48,  7.90s/it]
 13%|█▎        | 421/3123 [55:33<5:54:08,  7.86s/it]
 14%|█▎        | 422/3123 [55:41<5:52:50,  7.84s/it]
 14%|█▎        | 423/3123 [55:49<5:55:53,  7.91s/it]
 14%|█▎        | 424/3123 [55:57<5:53:56,  7.87s/it]
 14%|█▎        | 425/3123 [56:05<5:52:34,  7.84s/it]
 14%|█▎        | 426/3123 [56:12<5:51:28,  7.82s/it]
 14%|█▎        | 427/3123 [56:20<5:54:28,  7.89s/it]
 14%|█▎        | 428/3123 [56:28<5:52:49,  7.86s/it]
 14%|█▎        | 429/3123 [56:36<5:51:38,  7.83s/it]
 14%|█▍        | 430/3123 [56:44<5:54:42,  7.90s/it]
                                                    
{'loss': '0.003022', 'grad_norm': '0.1113', 'learning_rate': '0.0001', 'ppl': '1.003', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '73.3', 'tokens/total': 7045120, 'tokens/trainable': 2213313, 'epoch': '0.4128'}
+
 14%|█▍        | 430/3123 [56:44<5:54:42,  7.90s/it]
 14%|█▍        | 431/3123 [56:52<5:52:48,  7.86s/it]
 14%|█▍        | 432/3123 [56:59<5:51:30,  7.84s/it]
 14%|█▍        | 433/3123 [57:08<5:54:20,  7.90s/it]
 14%|█▍        | 434/3123 [57:15<5:52:24,  7.86s/it]
 14%|█▍        | 435/3123 [57:23<5:50:59,  7.83s/it]
 14%|█▍        | 436/3123 [57:31<5:49:58,  7.81s/it]
 14%|█▍        | 437/3123 [57:39<5:53:09,  7.89s/it]
 14%|█▍        | 438/3123 [57:47<5:51:27,  7.85s/it]
 14%|█▍        | 439/3123 [57:54<5:50:09,  7.83s/it]
 14%|█▍        | 440/3123 [58:03<5:53:01,  7.89s/it]
                                                    
{'loss': '0.002942', 'grad_norm': '0.1074', 'learning_rate': '0.0001', 'ppl': '1.003', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '82.12', 'tokens/total': 7208960, 'tokens/trainable': 2264334, 'epoch': '0.4224'}
+
 14%|█▍        | 440/3123 [58:03<5:53:01,  7.89s/it]
 14%|█▍        | 441/3123 [58:10<5:51:08,  7.86s/it]
 14%|█▍        | 442/3123 [58:18<5:49:57,  7.83s/it]
 14%|█▍        | 443/3123 [58:26<5:48:57,  7.81s/it]
 14%|█▍        | 444/3123 [58:34<5:52:06,  7.89s/it]
 14%|█▍        | 445/3123 [58:42<5:50:17,  7.85s/it]
 14%|█▍        | 446/3123 [58:49<5:49:05,  7.82s/it]
 14%|█▍        | 447/3123 [58:57<5:51:45,  7.89s/it]
 14%|█▍        | 448/3123 [59:05<5:49:57,  7.85s/it]
 14%|█▍        | 449/3123 [59:13<5:48:40,  7.82s/it]
 14%|█▍        | 450/3123 [59:21<5:51:36,  7.89s/it]
                                                    
{'loss': '0.003412', 'grad_norm': '0.1631', 'learning_rate': '0.0001', 'ppl': '1.003', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '78.49', 'tokens/total': 7372800, 'tokens/trainable': 2315460, 'epoch': '0.432'}
+
 14%|█▍        | 450/3123 [59:21<5:51:36,  7.89s/it]
 14%|█▍        | 451/3123 [59:29<5:49:48,  7.85s/it]
 14%|█▍        | 452/3123 [59:37<5:48:33,  7.83s/it]
 15%|█▍        | 453/3123 [59:44<5:47:32,  7.81s/it]
 15%|█▍        | 454/3123 [59:52<5:50:33,  7.88s/it]
 15%|█▍        | 455/3123 [1:00:00<5:48:54,  7.85s/it]
 15%|█▍        | 456/3123 [1:00:08<5:47:45,  7.82s/it]
 15%|█▍        | 457/3123 [1:00:16<5:50:44,  7.89s/it]
 15%|█▍        | 458/3123 [1:00:24<5:48:58,  7.86s/it]
 15%|█▍        | 459/3123 [1:00:31<5:47:39,  7.83s/it]
 15%|█▍        | 460/3123 [1:00:39<5:46:38,  7.81s/it]
                                                      
{'loss': '0.002535', 'grad_norm': '0.1514', 'learning_rate': '0.0001', 'ppl': '1.003', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '83.68', 'tokens/total': 7536640, 'tokens/trainable': 2366296, 'epoch': '0.4416'}
+
 15%|█▍        | 460/3123 [1:00:40<5:46:38,  7.81s/it]
 15%|█▍        | 461/3123 [1:00:47<5:49:30,  7.88s/it]
 15%|█▍        | 462/3123 [1:00:55<5:47:57,  7.85s/it]
 15%|█▍        | 463/3123 [1:01:03<5:46:48,  7.82s/it]
 15%|█▍        | 464/3123 [1:01:11<5:49:55,  7.90s/it]
 15%|█▍        | 465/3123 [1:01:19<5:48:06,  7.86s/it]
 15%|█▍        | 466/3123 [1:01:26<5:46:44,  7.83s/it]
 15%|█▍        | 467/3123 [1:01:34<5:45:43,  7.81s/it]
 15%|█▍        | 468/3123 [1:01:42<5:48:33,  7.88s/it]
 15%|█▌        | 469/3123 [1:01:50<5:47:01,  7.85s/it]
 15%|█▌        | 470/3123 [1:01:58<5:45:51,  7.82s/it]
                                                      
{'loss': '0.002229', 'grad_norm': '0.07373', 'learning_rate': '0.0001', 'ppl': '1.002', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '81.5', 'tokens/total': 7700480, 'tokens/trainable': 2417678, 'epoch': '0.4512'}
+
 15%|█▌        | 470/3123 [1:01:58<5:45:51,  7.82s/it]
 15%|█▌        | 471/3123 [1:02:06<5:48:53,  7.89s/it]
 15%|█▌        | 472/3123 [1:02:14<5:47:06,  7.86s/it]
 15%|█▌        | 473/3123 [1:02:21<5:45:49,  7.83s/it]
 15%|█▌        | 474/3123 [1:02:29<5:48:26,  7.89s/it]
 15%|█▌        | 475/3123 [1:02:37<5:46:37,  7.85s/it]
 15%|█▌        | 476/3123 [1:02:45<5:45:20,  7.83s/it]
 15%|█▌        | 477/3123 [1:02:53<5:44:28,  7.81s/it]
 15%|█▌        | 478/3123 [1:03:01<5:47:34,  7.88s/it]
 15%|█▌        | 479/3123 [1:03:09<5:45:52,  7.85s/it]
 15%|█▌        | 480/3123 [1:03:16<5:44:41,  7.82s/it]
                                                      
{'loss': '0.002742', 'grad_norm': '0.1494', 'learning_rate': '0.0001', 'ppl': '1.003', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '95.01', 'tokens/total': 7864320, 'tokens/trainable': 2469401, 'epoch': '0.4608'}
+
 15%|█▌        | 480/3123 [1:03:17<5:44:41,  7.82s/it]
 15%|█▌        | 481/3123 [1:03:24<5:47:30,  7.89s/it]
 15%|█▌        | 482/3123 [1:03:32<5:45:42,  7.85s/it]
 15%|█▌        | 483/3123 [1:03:40<5:44:29,  7.83s/it]
 15%|█▌        | 484/3123 [1:03:48<5:43:39,  7.81s/it]
 16%|█▌        | 485/3123 [1:03:56<5:46:50,  7.89s/it]
 16%|█▌        | 486/3123 [1:04:03<5:45:01,  7.85s/it]
 16%|█▌        | 487/3123 [1:04:11<5:43:48,  7.83s/it]
 16%|█▌        | 488/3123 [1:04:19<5:46:38,  7.89s/it]
 16%|█▌        | 489/3123 [1:04:27<5:44:48,  7.85s/it]
 16%|█▌        | 490/3123 [1:04:35<5:43:40,  7.83s/it]
                                                      
{'loss': '0.003123', 'grad_norm': '0.1885', 'learning_rate': '0.0001', 'ppl': '1.003', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '94.23', 'tokens/total': 8028160, 'tokens/trainable': 2520706, 'epoch': '0.4704'}
+
 16%|█▌        | 490/3123 [1:04:35<5:43:40,  7.83s/it]
 16%|█▌        | 491/3123 [1:04:43<5:46:33,  7.90s/it]
 16%|█▌        | 492/3123 [1:04:51<5:44:38,  7.86s/it]
 16%|█▌        | 493/3123 [1:04:58<5:43:17,  7.83s/it]
 16%|█▌        | 494/3123 [1:05:06<5:42:16,  7.81s/it]
 16%|█▌        | 495/3123 [1:05:14<5:45:12,  7.88s/it]
 16%|█▌        | 496/3123 [1:05:22<5:43:40,  7.85s/it]
 16%|█▌        | 497/3123 [1:05:30<5:42:31,  7.83s/it]
 16%|█▌        | 498/3123 [1:05:38<5:45:30,  7.90s/it]
 16%|█▌        | 499/3123 [1:05:46<5:43:46,  7.86s/it]
 16%|█▌        | 500/3123 [1:05:53<5:42:26,  7.83s/it]
                                                      
{'loss': '0.007013', 'grad_norm': '0.2578', 'learning_rate': '0.0001', 'ppl': '1.007', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '86.15', 'tokens/total': 8192000, 'tokens/trainable': 2571927, 'epoch': '0.48'}
+
 16%|█▌        | 500/3123 [1:05:54<5:42:26,  7.83s/it]
 16%|█▌        | 501/3123 [1:06:01<5:41:31,  7.82s/it]
 16%|█▌        | 502/3123 [1:06:09<5:44:34,  7.89s/it]
 16%|█▌        | 503/3123 [1:06:17<5:42:55,  7.85s/it]
 16%|█▌        | 504/3123 [1:06:25<5:41:46,  7.83s/it]
 16%|█▌        | 505/3123 [1:06:33<5:44:46,  7.90s/it]
 16%|█▌        | 506/3123 [1:06:41<5:42:56,  7.86s/it]
 16%|█▌        | 507/3123 [1:06:48<5:41:36,  7.84s/it]
 16%|█▋        | 508/3123 [1:06:56<5:40:37,  7.82s/it]
 16%|█▋        | 509/3123 [1:07:04<5:43:27,  7.88s/it]
 16%|█▋        | 510/3123 [1:07:12<5:41:49,  7.85s/it]
                                                      
{'loss': '0.007118', 'grad_norm': '0.1758', 'learning_rate': '0.0001', 'ppl': '1.007', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '86.57', 'tokens/total': 8355840, 'tokens/trainable': 2623451, 'epoch': '0.4896'}
+
 16%|█▋        | 510/3123 [1:07:12<5:41:49,  7.85s/it]
 16%|█▋        | 511/3123 [1:07:20<5:40:38,  7.82s/it]
 16%|█▋        | 512/3123 [1:07:28<5:43:44,  7.90s/it]
 16%|█▋        | 513/3123 [1:07:36<5:41:57,  7.86s/it]
 16%|█▋        | 514/3123 [1:07:43<5:40:35,  7.83s/it]
 16%|█▋        | 515/3123 [1:07:51<5:43:17,  7.90s/it]
 17%|█▋        | 516/3123 [1:07:59<5:41:20,  7.86s/it]
 17%|█▋        | 517/3123 [1:08:07<5:40:05,  7.83s/it]
 17%|█▋        | 518/3123 [1:08:15<5:39:10,  7.81s/it]
 17%|█▋        | 519/3123 [1:08:23<5:42:16,  7.89s/it]
 17%|█▋        | 520/3123 [1:08:31<5:40:38,  7.85s/it]
                                                      
{'loss': '0.004592', 'grad_norm': '0.249', 'learning_rate': '0.0001', 'ppl': '1.005', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'tokens/train_per_sec_per_gpu': '91.25', 'tokens/total': 8519680, 'tokens/trainable': 2674443, 'epoch': '0.4992'}
+
 17%|█▋        | 520/3123 [1:08:31<5:40:38,  7.85s/it]
 17%|█▋        | 521/3123 [1:08:38<5:39:23,  7.83s/it][2026-03-08 17:05:52,755] [INFO] [axolotl.core.trainers.base.evaluate:400] [PID:1659682] Running evaluation step...
+[2026-03-08 17:05:53,678] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:1659682] generate_batches time: 0.4639475345611572
+[2026-03-08 17:05:54,129] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:1659682] generate_batches time: 0.45036745071411133
+[2026-03-08 17:05:54,579] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:1659682] generate_batches time: 0.45003342628479004
+[2026-03-08 17:05:55,032] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:1659682] generate_batches time: 0.4526052474975586
+[2026-03-08 17:05:55,032] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:1659682] gather_len_batches: [34]
+
+
  0%|          | 0/34 [00:00<?, ?it/s]
+
  6%|▌         | 2/34 [00:00<00:03,  8.50it/s]
+
  9%|▉         | 3/34 [00:00<00:05,  5.95it/s]
+
 12%|█▏        | 4/34 [00:00<00:05,  5.14it/s]
+
 15%|█▍        | 5/34 [00:00<00:06,  4.76it/s]
+
 18%|█▊        | 6/34 [00:01<00:06,  4.56it/s]
+
 21%|██        | 7/34 [00:01<00:06,  4.44it/s]
+
 24%|██▎       | 8/34 [00:01<00:05,  4.36it/s]
+
 26%|██▋       | 9/34 [00:02<00:06,  3.70it/s]
+
 29%|██▉       | 10/34 [00:02<00:06,  3.93it/s]
+
 32%|███▏      | 11/34 [00:02<00:05,  4.00it/s]
+
 35%|███▌      | 12/34 [00:02<00:05,  4.06it/s]
+
 38%|███▊      | 13/34 [00:02<00:05,  4.10it/s]
+
 41%|████      | 14/34 [00:03<00:04,  4.13it/s]
+
 44%|████▍     | 15/34 [00:03<00:04,  4.15it/s]
+
 47%|████▋     | 16/34 [00:03<00:04,  4.17it/s]
+
 50%|█████     | 17/34 [00:03<00:04,  3.97it/s]
+
 53%|█████▎    | 18/34 [00:04<00:03,  4.09it/s]
+
 56%|█████▌    | 19/34 [00:04<00:03,  4.12it/s]
+
 59%|█████▉    | 20/34 [00:04<00:03,  4.14it/s]
+
 62%|██████▏   | 21/34 [00:04<00:03,  4.15it/s]
+
 65%|██████▍   | 22/34 [00:05<00:02,  4.17it/s]
+
 68%|██████▊   | 23/34 [00:05<00:02,  4.17it/s]
+
 71%|███████   | 24/34 [00:05<00:02,  4.18it/s]
+
 74%|███████▎  | 25/34 [00:05<00:02,  4.00it/s]
+
 76%|███████▋  | 26/34 [00:06<00:01,  4.12it/s]
+
 79%|███████▉  | 27/34 [00:06<00:01,  4.15it/s]
+
 82%|████████▏ | 28/34 [00:06<00:01,  4.16it/s]
+
 85%|████████▌ | 29/34 [00:06<00:01,  4.17it/s]
+
 88%|████████▊ | 30/34 [00:07<00:00,  4.18it/s]
+
 91%|█████████ | 31/34 [00:07<00:00,  4.18it/s]
+
 94%|█████████▍| 32/34 [00:07<00:00,  4.19it/s]
+
 97%|█████████▋| 33/34 [00:07<00:00,  3.95it/s]Traceback (most recent call last):
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/multiprocessing/util.py", line 300, in _run_finalizers
+    finalizer()
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/multiprocessing/util.py", line 224, in __call__
+    res = self._callback(*self._args, **self._kwargs)
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/multiprocessing/util.py", line 133, in _remove_temp_dir
+    rmtree(tempdir)
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/shutil.py", line 725, in rmtree
+    _rmtree_safe_fd(fd, path, onerror)
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/shutil.py", line 681, in _rmtree_safe_fd
+    onerror(os.unlink, fullname, sys.exc_info())
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/shutil.py", line 679, in _rmtree_safe_fd
+    os.unlink(entry.name, dir_fd=topfd)
+OSError: [Errno 16] Device or resource busy: '.nfs00000000000daa1000012506'
+
+
100%|██████████| 34/34 [00:08<00:00,  3.83it/s]
                                                      
+
                                               
{'eval_loss': '0.004909', 'eval_runtime': '9.006', 'eval_samples_per_second': '22.21', 'eval_steps_per_second': '22.21', 'eval_ppl': '1.005', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.5', 'epoch': '0.5001', 'tokens/train_per_sec_per_gpu': '83.41'}
+
 17%|█▋        | 521/3123 [1:08:50<5:39:23,  7.83s/it]
+
100%|██████████| 34/34 [00:08<00:00,  3.83it/s]
+
                                               
 17%|█▋        | 522/3123 [1:08:57<8:02:21, 11.13s/it]
 17%|█▋        | 523/3123 [1:09:05<7:18:25, 10.12s/it]
 17%|█▋        | 524/3123 [1:09:13<6:47:39,  9.41s/it]
 17%|█▋        | 525/3123 [1:09:21<6:30:28,  9.02s/it]
 17%|█▋        | 526/3123 [1:09:29<6:14:04,  8.64s/it]
 17%|█▋        | 527/3123 [1:09:36<6:02:32,  8.38s/it]
 17%|█▋        | 528/3123 [1:09:44<5:58:10,  8.28s/it]
 17%|█▋        | 529/3123 [1:09:52<5:51:21,  8.13s/it]
 17%|█▋        | 530/3123 [1:10:00<5:46:31,  8.02s/it]
                                                      
{'loss': '0.004099', 'grad_norm': '0.1855', 'learning_rate': '0.0001', 'ppl': '1.004', 'memory/max_active (GiB)': '33.96', 'memory/max_allocated (GiB)': '33.96', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '74.43', 'tokens/total': 8683520, 'tokens/trainable': 2725882, 'epoch': '0.5088'}
+
 17%|█▋        | 530/3123 [1:10:00<5:46:31,  8.02s/it]
 17%|█▋        | 531/3123 [1:10:08<5:43:04,  7.94s/it]
 17%|█▋        | 532/3123 [1:10:16<5:44:35,  7.98s/it]
 17%|█▋        | 533/3123 [1:10:23<5:41:40,  7.92s/it]
 17%|█▋        | 534/3123 [1:10:31<5:39:43,  7.87s/it]
 17%|█▋        | 535/3123 [1:10:39<5:41:51,  7.93s/it]
 17%|█▋        | 536/3123 [1:10:47<5:39:40,  7.88s/it]
 17%|█▋        | 537/3123 [1:10:55<5:38:09,  7.85s/it]
 17%|█▋        | 538/3123 [1:11:03<5:37:06,  7.82s/it]
 17%|█▋        | 539/3123 [1:11:11<5:39:48,  7.89s/it]
 17%|█▋        | 540/3123 [1:11:18<5:38:04,  7.85s/it]
                                                      
{'loss': '0.004386', 'grad_norm': '0.2578', 'learning_rate': '0.0001', 'ppl': '1.004', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '83.38', 'tokens/total': 8847360, 'tokens/trainable': 2777518, 'epoch': '0.5184'}
+
 17%|█▋        | 540/3123 [1:11:19<5:38:04,  7.85s/it]
 17%|█▋        | 541/3123 [1:11:26<5:36:51,  7.83s/it]
 17%|█▋        | 542/3123 [1:11:34<5:39:43,  7.90s/it]
 17%|█▋        | 543/3123 [1:11:42<5:37:56,  7.86s/it]
 17%|█▋        | 544/3123 [1:11:50<5:36:44,  7.83s/it]
 17%|█▋        | 545/3123 [1:11:58<5:35:51,  7.82s/it]
 17%|█▋        | 546/3123 [1:12:06<5:39:10,  7.90s/it]
 18%|█▊        | 547/3123 [1:12:13<5:37:29,  7.86s/it]
 18%|█▊        | 548/3123 [1:12:21<5:36:17,  7.84s/it]
 18%|█▊        | 549/3123 [1:12:29<5:39:23,  7.91s/it]
 18%|█▊        | 550/3123 [1:12:37<5:37:30,  7.87s/it]
                                                      
{'loss': '0.003662', 'grad_norm': '0.166', 'learning_rate': '0.0001', 'ppl': '1.004', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '92.66', 'tokens/total': 9011200, 'tokens/trainable': 2828761, 'epoch': '0.528'}
+
 18%|█▊        | 550/3123 [1:12:37<5:37:30,  7.87s/it]
 18%|█▊        | 551/3123 [1:12:45<5:36:10,  7.84s/it]
 18%|█▊        | 552/3123 [1:12:53<5:35:12,  7.82s/it]
 18%|█▊        | 553/3123 [1:13:01<5:38:11,  7.90s/it]
 18%|█▊        | 554/3123 [1:13:08<5:36:38,  7.86s/it]
 18%|█▊        | 555/3123 [1:13:16<5:35:27,  7.84s/it]
 18%|█▊        | 556/3123 [1:13:24<5:38:17,  7.91s/it]
 18%|█▊        | 557/3123 [1:13:32<5:36:27,  7.87s/it]
 18%|█▊        | 558/3123 [1:13:40<5:35:10,  7.84s/it]
 18%|█▊        | 559/3123 [1:13:48<5:37:46,  7.90s/it]
 18%|█▊        | 560/3123 [1:13:56<5:36:05,  7.87s/it]
                                                      
{'loss': '0.003376', 'grad_norm': '0.2617', 'learning_rate': '0.0001', 'ppl': '1.003', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '89.9', 'tokens/total': 9175040, 'tokens/trainable': 2879740, 'epoch': '0.5376'}
+
 18%|█▊        | 560/3123 [1:13:56<5:36:05,  7.87s/it]
 18%|█▊        | 561/3123 [1:14:03<5:34:45,  7.84s/it]
 18%|█▊        | 562/3123 [1:14:11<5:33:45,  7.82s/it]
 18%|█▊        | 563/3123 [1:14:19<5:36:58,  7.90s/it]
 18%|█▊        | 564/3123 [1:14:27<5:35:15,  7.86s/it]
 18%|█▊        | 565/3123 [1:14:35<5:33:58,  7.83s/it]
 18%|█▊        | 566/3123 [1:14:43<5:36:40,  7.90s/it]
 18%|█▊        | 567/3123 [1:14:51<5:35:02,  7.86s/it]
 18%|█▊        | 568/3123 [1:14:58<5:33:45,  7.84s/it]
 18%|█▊        | 569/3123 [1:15:06<5:32:48,  7.82s/it]
 18%|█▊        | 570/3123 [1:15:14<5:36:00,  7.90s/it]
                                                      
{'loss': '0.003272', 'grad_norm': '0.2012', 'learning_rate': '0.0001', 'ppl': '1.003', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '88.91', 'tokens/total': 9338880, 'tokens/trainable': 2931031, 'epoch': '0.5472'}
+
 18%|█▊        | 570/3123 [1:15:15<5:36:00,  7.90s/it]
 18%|█▊        | 571/3123 [1:15:22<5:34:21,  7.86s/it]
 18%|█▊        | 572/3123 [1:15:30<5:33:11,  7.84s/it]
 18%|█▊        | 573/3123 [1:15:38<5:35:50,  7.90s/it]
 18%|█▊        | 574/3123 [1:15:46<5:34:05,  7.86s/it]
 18%|█▊        | 575/3123 [1:15:54<5:32:52,  7.84s/it]
 18%|█▊        | 576/3123 [1:16:02<5:35:49,  7.91s/it]
 18%|█▊        | 577/3123 [1:16:09<5:33:51,  7.87s/it]
 19%|█▊        | 578/3123 [1:16:17<5:32:28,  7.84s/it]
 19%|█▊        | 579/3123 [1:16:25<5:31:30,  7.82s/it]
 19%|█▊        | 580/3123 [1:16:33<5:34:24,  7.89s/it]
                                                      
{'loss': '0.003928', 'grad_norm': '0.1128', 'learning_rate': '0.0001', 'ppl': '1.004', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '82.57', 'tokens/total': 9502720, 'tokens/trainable': 2981776, 'epoch': '0.5568'}
+
 19%|█▊        | 580/3123 [1:16:33<5:34:24,  7.89s/it]
 19%|█▊        | 581/3123 [1:16:41<5:32:45,  7.85s/it]
 19%|█▊        | 582/3123 [1:16:48<5:31:34,  7.83s/it]
 19%|█▊        | 583/3123 [1:16:57<5:34:28,  7.90s/it]
 19%|█▊        | 584/3123 [1:17:04<5:32:43,  7.86s/it]
 19%|█▊        | 585/3123 [1:17:12<5:31:27,  7.84s/it]
 19%|█▉        | 586/3123 [1:17:20<5:30:24,  7.81s/it]
 19%|█▉        | 587/3123 [1:17:28<5:33:12,  7.88s/it]
 19%|█▉        | 588/3123 [1:17:36<5:31:36,  7.85s/it]
 19%|█▉        | 589/3123 [1:17:43<5:30:33,  7.83s/it]
 19%|█▉        | 590/3123 [1:17:52<5:33:28,  7.90s/it]
                                                      
{'loss': '0.00318', 'grad_norm': '0.165', 'learning_rate': '0.0001', 'ppl': '1.003', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '82.6', 'tokens/total': 9666560, 'tokens/trainable': 3032962, 'epoch': '0.5664'}
+
 19%|█▉        | 590/3123 [1:17:52<5:33:28,  7.90s/it]
 19%|█▉        | 591/3123 [1:17:59<5:31:46,  7.86s/it]
 19%|█▉        | 592/3123 [1:18:07<5:30:27,  7.83s/it]
 19%|█▉        | 593/3123 [1:18:15<5:29:29,  7.81s/it]
 19%|█▉        | 594/3123 [1:18:23<5:32:25,  7.89s/it]
 19%|█▉        | 595/3123 [1:18:31<5:30:52,  7.85s/it]
 19%|█▉        | 596/3123 [1:18:38<5:29:42,  7.83s/it]
 19%|█▉        | 597/3123 [1:18:47<5:32:36,  7.90s/it]
 19%|█▉        | 598/3123 [1:18:54<5:30:48,  7.86s/it]
 19%|█▉        | 599/3123 [1:19:02<5:29:31,  7.83s/it]
 19%|█▉        | 600/3123 [1:19:10<5:32:06,  7.90s/it]
                                                      
{'loss': '0.002615', 'grad_norm': '0.2061', 'learning_rate': '0.0001', 'ppl': '1.003', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '79.05', 'tokens/total': 9830400, 'tokens/trainable': 3084477, 'epoch': '0.576'}
+
 19%|█▉        | 600/3123 [1:19:10<5:32:06,  7.90s/it]
 19%|█▉        | 601/3123 [1:19:18<5:30:15,  7.86s/it]
 19%|█▉        | 602/3123 [1:19:26<5:28:58,  7.83s/it]
 19%|█▉        | 603/3123 [1:19:33<5:28:03,  7.81s/it]
 19%|█▉        | 604/3123 [1:19:41<5:31:17,  7.89s/it]
 19%|█▉        | 605/3123 [1:19:49<5:29:37,  7.85s/it]
 19%|█▉        | 606/3123 [1:19:57<5:28:26,  7.83s/it]
 19%|█▉        | 607/3123 [1:20:05<5:31:27,  7.90s/it]
 19%|█▉        | 608/3123 [1:20:13<5:29:39,  7.86s/it]
 20%|█▉        | 609/3123 [1:20:21<5:28:20,  7.84s/it]
 20%|█▉        | 610/3123 [1:20:28<5:27:21,  7.82s/it]
                                                      
{'loss': '0.003223', 'grad_norm': '0.2754', 'learning_rate': '0.0001', 'ppl': '1.003', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '87.92', 'tokens/total': 9994240, 'tokens/trainable': 3135491, 'epoch': '0.5856'}
+
 20%|█▉        | 610/3123 [1:20:29<5:27:21,  7.82s/it]
 20%|█▉        | 611/3123 [1:20:36<5:30:16,  7.89s/it]
 20%|█▉        | 612/3123 [1:20:44<5:28:34,  7.85s/it]
 20%|█▉        | 613/3123 [1:20:52<5:27:24,  7.83s/it]
 20%|█▉        | 614/3123 [1:21:00<5:30:30,  7.90s/it]
 20%|█▉        | 615/3123 [1:21:08<5:28:43,  7.86s/it]
 20%|█▉        | 616/3123 [1:21:16<5:27:29,  7.84s/it]
 20%|█▉        | 617/3123 [1:21:23<5:26:34,  7.82s/it]
 20%|█▉        | 618/3123 [1:21:31<5:29:26,  7.89s/it]
 20%|█▉        | 619/3123 [1:21:39<5:27:48,  7.85s/it]
 20%|█▉        | 620/3123 [1:21:47<5:26:41,  7.83s/it]
                                                      
{'loss': '0.002825', 'grad_norm': '0.1699', 'learning_rate': '0.0001', 'ppl': '1.003', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '77.25', 'tokens/total': 10158080, 'tokens/trainable': 3186671, 'epoch': '0.5952'}
+
 20%|█▉        | 620/3123 [1:21:47<5:26:41,  7.83s/it]
 20%|█▉        | 621/3123 [1:21:55<5:29:13,  7.90s/it]
 20%|█▉        | 622/3123 [1:22:03<5:27:33,  7.86s/it]
 20%|█▉        | 623/3123 [1:22:11<5:26:23,  7.83s/it]
 20%|█▉        | 624/3123 [1:22:19<5:29:07,  7.90s/it]
 20%|██        | 625/3123 [1:22:26<5:27:13,  7.86s/it]
 20%|██        | 626/3123 [1:22:34<5:25:59,  7.83s/it]
 20%|██        | 627/3123 [1:22:42<5:25:10,  7.82s/it]
 20%|██        | 628/3123 [1:22:50<5:28:04,  7.89s/it]
 20%|██        | 629/3123 [1:22:58<5:26:29,  7.85s/it]
 20%|██        | 630/3123 [1:23:06<5:25:22,  7.83s/it]
                                                      
{'loss': '0.001928', 'grad_norm': '0.127', 'learning_rate': '0.0001', 'ppl': '1.002', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '94.54', 'tokens/total': 10321920, 'tokens/trainable': 3238286, 'epoch': '0.6048'}
+
 20%|██        | 630/3123 [1:23:06<5:25:22,  7.83s/it]
 20%|██        | 631/3123 [1:23:14<5:28:17,  7.90s/it]
 20%|██        | 632/3123 [1:23:21<5:26:37,  7.87s/it]
 20%|██        | 633/3123 [1:23:29<5:25:18,  7.84s/it]
 20%|██        | 634/3123 [1:23:37<5:24:28,  7.82s/it]
 20%|██        | 635/3123 [1:23:45<5:27:24,  7.90s/it]
 20%|██        | 636/3123 [1:23:53<5:25:47,  7.86s/it]
 20%|██        | 637/3123 [1:24:01<5:24:37,  7.83s/it]
 20%|██        | 638/3123 [1:24:09<5:27:29,  7.91s/it]
 20%|██        | 639/3123 [1:24:16<5:25:42,  7.87s/it]
 20%|██        | 640/3123 [1:24:24<5:24:25,  7.84s/it]
                                                      
{'loss': '0.002336', 'grad_norm': '0.09375', 'learning_rate': '0.0001', 'ppl': '1.002', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '83.42', 'tokens/total': 10485760, 'tokens/trainable': 3290171, 'epoch': '0.6144'}
+
 20%|██        | 640/3123 [1:24:25<5:24:25,  7.84s/it]
 21%|██        | 641/3123 [1:24:32<5:23:30,  7.82s/it]
 21%|██        | 642/3123 [1:24:40<5:26:09,  7.89s/it]
 21%|██        | 643/3123 [1:24:48<5:24:38,  7.85s/it]
 21%|██        | 644/3123 [1:24:56<5:23:27,  7.83s/it]
 21%|██        | 645/3123 [1:25:04<5:26:23,  7.90s/it]
 21%|██        | 646/3123 [1:25:11<5:24:41,  7.87s/it]
 21%|██        | 647/3123 [1:25:19<5:23:26,  7.84s/it]
 21%|██        | 648/3123 [1:25:27<5:22:32,  7.82s/it]
 21%|██        | 649/3123 [1:25:35<5:25:19,  7.89s/it]
 21%|██        | 650/3123 [1:25:43<5:23:45,  7.85s/it]
                                                      
{'loss': '0.002462', 'grad_norm': '0.1118', 'learning_rate': '0.0001', 'ppl': '1.002', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '82.64', 'tokens/total': 10649600, 'tokens/trainable': 3341228, 'epoch': '0.624'}
+
 21%|██        | 650/3123 [1:25:43<5:23:45,  7.85s/it]
 21%|██        | 651/3123 [1:25:51<5:22:40,  7.83s/it]
 21%|██        | 652/3123 [1:25:59<5:25:33,  7.90s/it]
 21%|██        | 653/3123 [1:26:06<5:23:40,  7.86s/it]
 21%|██        | 654/3123 [1:26:14<5:22:25,  7.84s/it]
 21%|██        | 655/3123 [1:26:22<5:25:03,  7.90s/it]
 21%|██        | 656/3123 [1:26:30<5:23:16,  7.86s/it]
 21%|██        | 657/3123 [1:26:38<5:22:00,  7.83s/it]
 21%|██        | 658/3123 [1:26:46<5:21:07,  7.82s/it]
 21%|██        | 659/3123 [1:26:54<5:24:17,  7.90s/it]
 21%|██        | 660/3123 [1:27:01<5:22:34,  7.86s/it]
                                                      
{'loss': '0.001495', 'grad_norm': '0.01733', 'learning_rate': '0.0001', 'ppl': '1.002', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '75.33', 'tokens/total': 10813440, 'tokens/trainable': 3392191, 'epoch': '0.6335'}
+
 21%|██        | 660/3123 [1:27:02<5:22:34,  7.86s/it]
 21%|██        | 661/3123 [1:27:09<5:21:26,  7.83s/it]
 21%|██        | 662/3123 [1:27:17<5:24:01,  7.90s/it]
 21%|██        | 663/3123 [1:27:25<5:22:19,  7.86s/it]
 21%|██▏       | 664/3123 [1:27:33<5:21:01,  7.83s/it]
 21%|██▏       | 665/3123 [1:27:41<5:20:11,  7.82s/it]
 21%|██▏       | 666/3123 [1:27:49<5:23:13,  7.89s/it]
 21%|██▏       | 667/3123 [1:27:56<5:21:40,  7.86s/it]
 21%|██▏       | 668/3123 [1:28:04<5:20:30,  7.83s/it]
 21%|██▏       | 669/3123 [1:28:12<5:23:07,  7.90s/it]
 21%|██▏       | 670/3123 [1:28:20<5:21:21,  7.86s/it]
                                                      
{'loss': '0.001687', 'grad_norm': '0.1069', 'learning_rate': '0.0001', 'ppl': '1.002', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '72.65', 'tokens/total': 10977280, 'tokens/trainable': 3442697, 'epoch': '0.6431'}
+
 21%|██▏       | 670/3123 [1:28:20<5:21:21,  7.86s/it]
 21%|██▏       | 671/3123 [1:28:28<5:20:09,  7.83s/it]
 22%|██▏       | 672/3123 [1:28:36<5:19:17,  7.82s/it]
 22%|██▏       | 673/3123 [1:28:44<5:22:17,  7.89s/it]
 22%|██▏       | 674/3123 [1:28:51<5:20:42,  7.86s/it]
 22%|██▏       | 675/3123 [1:28:59<5:19:37,  7.83s/it]
 22%|██▏       | 676/3123 [1:29:07<5:22:14,  7.90s/it]
 22%|██▏       | 677/3123 [1:29:15<5:20:35,  7.86s/it]
 22%|██▏       | 678/3123 [1:29:23<5:19:26,  7.84s/it]
 22%|██▏       | 679/3123 [1:29:31<5:18:32,  7.82s/it]
 22%|██▏       | 680/3123 [1:29:39<5:21:36,  7.90s/it]
                                                      
{'loss': '0.001429', 'grad_norm': '0.09033', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '94.79', 'tokens/total': 11141120, 'tokens/trainable': 3494178, 'epoch': '0.6527'}
+
 22%|██▏       | 680/3123 [1:29:39<5:21:36,  7.90s/it]
 22%|██▏       | 681/3123 [1:29:46<5:20:00,  7.86s/it]
 22%|██▏       | 682/3123 [1:29:54<5:18:46,  7.84s/it]
 22%|██▏       | 683/3123 [1:30:02<5:21:25,  7.90s/it]
 22%|██▏       | 684/3123 [1:30:10<5:19:44,  7.87s/it]
 22%|██▏       | 685/3123 [1:30:18<5:18:28,  7.84s/it]
 22%|██▏       | 686/3123 [1:30:26<5:21:23,  7.91s/it]
 22%|██▏       | 687/3123 [1:30:34<5:19:28,  7.87s/it]
 22%|██▏       | 688/3123 [1:30:41<5:18:08,  7.84s/it]
 22%|██▏       | 689/3123 [1:30:49<5:17:12,  7.82s/it]
 22%|██▏       | 690/3123 [1:30:57<5:20:02,  7.89s/it]
                                                      
{'loss': '0.001274', 'grad_norm': '0.1328', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '89.99', 'tokens/total': 11304960, 'tokens/trainable': 3545238, 'epoch': '0.6623'}
+
 22%|██▏       | 690/3123 [1:30:58<5:20:02,  7.89s/it]
 22%|██▏       | 691/3123 [1:31:05<5:18:27,  7.86s/it]
 22%|██▏       | 692/3123 [1:31:13<5:17:20,  7.83s/it]
 22%|██▏       | 693/3123 [1:31:21<5:20:10,  7.91s/it]
 22%|██▏       | 694/3123 [1:31:29<5:18:27,  7.87s/it]
 22%|██▏       | 695/3123 [1:31:36<5:17:09,  7.84s/it]
 22%|██▏       | 696/3123 [1:31:44<5:16:19,  7.82s/it]
 22%|██▏       | 697/3123 [1:31:52<5:19:15,  7.90s/it]
 22%|██▏       | 698/3123 [1:32:00<5:17:46,  7.86s/it]
 22%|██▏       | 699/3123 [1:32:08<5:16:37,  7.84s/it]
 22%|██▏       | 700/3123 [1:32:16<5:19:33,  7.91s/it]
                                                      
{'loss': '0.00168', 'grad_norm': '0.09668', 'learning_rate': '0.0001', 'ppl': '1.002', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '81.06', 'tokens/total': 11468800, 'tokens/trainable': 3595974, 'epoch': '0.6719'}
+
 22%|██▏       | 700/3123 [1:32:16<5:19:33,  7.91s/it]
 22%|██▏       | 701/3123 [1:32:24<5:17:43,  7.87s/it]
 22%|██▏       | 702/3123 [1:32:32<5:16:23,  7.84s/it]
 23%|██▎       | 703/3123 [1:32:39<5:15:27,  7.82s/it]
 23%|██▎       | 704/3123 [1:32:47<5:18:16,  7.89s/it]
 23%|██▎       | 705/3123 [1:32:55<5:16:47,  7.86s/it]
 23%|██▎       | 706/3123 [1:33:03<5:15:36,  7.83s/it]
 23%|██▎       | 707/3123 [1:33:11<5:18:18,  7.90s/it]
 23%|██▎       | 708/3123 [1:33:19<5:16:36,  7.87s/it]
 23%|██▎       | 709/3123 [1:33:27<5:15:24,  7.84s/it]
 23%|██▎       | 710/3123 [1:33:35<5:17:56,  7.91s/it]
                                                      
{'loss': '0.001274', 'grad_norm': '0.1099', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '72.36', 'tokens/total': 11632640, 'tokens/trainable': 3646406, 'epoch': '0.6815'}
+
 23%|██▎       | 710/3123 [1:33:35<5:17:56,  7.91s/it]
 23%|██▎       | 711/3123 [1:33:42<5:16:18,  7.87s/it]
 23%|██▎       | 712/3123 [1:33:50<5:15:00,  7.84s/it]
 23%|██▎       | 713/3123 [1:33:58<5:14:05,  7.82s/it]
 23%|██▎       | 714/3123 [1:34:06<5:17:05,  7.90s/it]
 23%|██▎       | 715/3123 [1:34:14<5:15:34,  7.86s/it]
 23%|██▎       | 716/3123 [1:34:22<5:14:24,  7.84s/it]
 23%|██▎       | 717/3123 [1:34:30<5:16:57,  7.90s/it]
 23%|██▎       | 718/3123 [1:34:37<5:15:11,  7.86s/it]
 23%|██▎       | 719/3123 [1:34:45<5:14:00,  7.84s/it]
 23%|██▎       | 720/3123 [1:34:53<5:13:11,  7.82s/it]
                                                      
{'loss': '0.001683', 'grad_norm': '0.1172', 'learning_rate': '0.0001', 'ppl': '1.002', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '83.24', 'tokens/total': 11796480, 'tokens/trainable': 3697893, 'epoch': '0.6911'}
+
 23%|██▎       | 720/3123 [1:34:53<5:13:11,  7.82s/it]
 23%|██▎       | 721/3123 [1:35:01<5:16:15,  7.90s/it]
 23%|██▎       | 722/3123 [1:35:09<5:14:40,  7.86s/it]
 23%|██▎       | 723/3123 [1:35:17<5:13:29,  7.84s/it]
 23%|██▎       | 724/3123 [1:35:25<5:15:53,  7.90s/it]
 23%|██▎       | 725/3123 [1:35:32<5:14:10,  7.86s/it]
 23%|██▎       | 726/3123 [1:35:40<5:12:59,  7.83s/it]
 23%|██▎       | 727/3123 [1:35:48<5:12:08,  7.82s/it]
 23%|██▎       | 728/3123 [1:35:56<5:15:01,  7.89s/it]
 23%|██▎       | 729/3123 [1:36:04<5:13:27,  7.86s/it]
 23%|██▎       | 730/3123 [1:36:12<5:12:19,  7.83s/it]
                                                      
{'loss': '0.001029', 'grad_norm': '0.08643', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '72.87', 'tokens/total': 11960320, 'tokens/trainable': 3749410, 'epoch': '0.7007'}
+
 23%|██▎       | 730/3123 [1:36:12<5:12:19,  7.83s/it]
 23%|██▎       | 731/3123 [1:36:20<5:14:59,  7.90s/it]
 23%|██▎       | 732/3123 [1:36:27<5:13:21,  7.86s/it]
 23%|██▎       | 733/3123 [1:36:35<5:12:13,  7.84s/it]
 24%|██▎       | 734/3123 [1:36:43<5:15:05,  7.91s/it]
 24%|██▎       | 735/3123 [1:36:51<5:13:23,  7.87s/it]
 24%|██▎       | 736/3123 [1:36:59<5:12:06,  7.85s/it]
 24%|██▎       | 737/3123 [1:37:07<5:11:13,  7.83s/it]
 24%|██▎       | 738/3123 [1:37:15<5:14:13,  7.90s/it]
 24%|██▎       | 739/3123 [1:37:22<5:12:30,  7.86s/it]
 24%|██▎       | 740/3123 [1:37:30<5:11:16,  7.84s/it]
                                                      
{'loss': '0.001321', 'grad_norm': '0.07812', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '72.19', 'tokens/total': 12124160, 'tokens/trainable': 3800410, 'epoch': '0.7103'}
+
 24%|██▎       | 740/3123 [1:37:31<5:11:16,  7.84s/it]
 24%|██▎       | 741/3123 [1:37:38<5:13:55,  7.91s/it]
 24%|██▍       | 742/3123 [1:37:46<5:12:07,  7.87s/it]
 24%|██▍       | 743/3123 [1:37:54<5:10:49,  7.84s/it]
 24%|██▍       | 744/3123 [1:38:02<5:09:54,  7.82s/it]
 24%|██▍       | 745/3123 [1:38:10<5:12:40,  7.89s/it]
 24%|██▍       | 746/3123 [1:38:17<5:11:07,  7.85s/it]
 24%|██▍       | 747/3123 [1:38:25<5:10:03,  7.83s/it]
 24%|██▍       | 748/3123 [1:38:33<5:12:45,  7.90s/it]
 24%|██▍       | 749/3123 [1:38:41<5:10:59,  7.86s/it]
 24%|██▍       | 750/3123 [1:38:49<5:09:44,  7.83s/it]
                                                      
{'loss': '0.5068', 'grad_norm': '3.547', 'learning_rate': '0.0001', 'ppl': '1.66', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '88.22', 'tokens/total': 12288000, 'tokens/trainable': 3852024, 'epoch': '0.7199'}
+
 24%|██▍       | 750/3123 [1:38:49<5:09:44,  7.83s/it]
 24%|██▍       | 751/3123 [1:38:57<5:12:12,  7.90s/it]
 24%|██▍       | 752/3123 [1:39:05<5:10:36,  7.86s/it]
 24%|██▍       | 753/3123 [1:39:12<5:09:23,  7.83s/it]
 24%|██▍       | 754/3123 [1:39:20<5:08:29,  7.81s/it]
 24%|██▍       | 755/3123 [1:39:28<5:11:31,  7.89s/it]
 24%|██▍       | 756/3123 [1:39:36<5:10:06,  7.86s/it]
 24%|██▍       | 757/3123 [1:39:44<5:08:58,  7.84s/it]
 24%|██▍       | 758/3123 [1:39:52<5:11:34,  7.90s/it]
 24%|██▍       | 759/3123 [1:40:00<5:09:47,  7.86s/it]
 24%|██▍       | 760/3123 [1:40:07<5:08:39,  7.84s/it]
                                                      
{'loss': '0.0424', 'grad_norm': '1.609', 'learning_rate': '0.0001', 'ppl': '1.043', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '91.02', 'tokens/total': 12451840, 'tokens/trainable': 3903571, 'epoch': '0.7295'}
+
 24%|██▍       | 760/3123 [1:40:08<5:08:39,  7.84s/it]
 24%|██▍       | 761/3123 [1:40:15<5:07:46,  7.82s/it]
 24%|██▍       | 762/3123 [1:40:23<5:10:22,  7.89s/it]
 24%|██▍       | 763/3123 [1:40:31<5:08:53,  7.85s/it]
 24%|██▍       | 764/3123 [1:40:39<5:07:47,  7.83s/it]
 24%|██▍       | 765/3123 [1:40:47<5:10:25,  7.90s/it]
 25%|██▍       | 766/3123 [1:40:55<5:08:44,  7.86s/it]
 25%|██▍       | 767/3123 [1:41:02<5:07:34,  7.83s/it]
 25%|██▍       | 768/3123 [1:41:10<5:06:40,  7.81s/it]
 25%|██▍       | 769/3123 [1:41:18<5:09:30,  7.89s/it]
 25%|██▍       | 770/3123 [1:41:26<5:08:06,  7.86s/it]
                                                      
{'loss': '0.05415', 'grad_norm': '0.4434', 'learning_rate': '0.0001', 'ppl': '1.056', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '88.71', 'tokens/total': 12615680, 'tokens/trainable': 3954797, 'epoch': '0.7391'}
+
 25%|██▍       | 770/3123 [1:41:26<5:08:06,  7.86s/it]
 25%|██▍       | 771/3123 [1:41:34<5:07:05,  7.83s/it]
 25%|██▍       | 772/3123 [1:41:42<5:09:33,  7.90s/it]
 25%|██▍       | 773/3123 [1:41:50<5:07:58,  7.86s/it]
 25%|██▍       | 774/3123 [1:41:57<5:06:41,  7.83s/it]
 25%|██▍       | 775/3123 [1:42:05<5:09:16,  7.90s/it]
 25%|██▍       | 776/3123 [1:42:13<5:07:34,  7.86s/it]
 25%|██▍       | 777/3123 [1:42:21<5:06:23,  7.84s/it]
 25%|██▍       | 778/3123 [1:42:29<5:05:24,  7.81s/it]
 25%|██▍       | 779/3123 [1:42:37<5:08:12,  7.89s/it]
 25%|██▍       | 780/3123 [1:42:45<5:06:34,  7.85s/it]
                                                      
{'loss': '0.01558', 'grad_norm': '0.6211', 'learning_rate': '0.0001', 'ppl': '1.016', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '75.27', 'tokens/total': 12779520, 'tokens/trainable': 4005775, 'epoch': '0.7487'}
+
 25%|██▍       | 780/3123 [1:42:45<5:06:34,  7.85s/it]
 25%|██▌       | 781/3123 [1:42:52<5:05:30,  7.83s/it]
 25%|██▌       | 782/3123 [1:43:00<5:07:53,  7.89s/it]
 25%|██▌       | 783/3123 [1:43:08<5:06:22,  7.86s/it]
 25%|██▌       | 784/3123 [1:43:16<5:05:17,  7.83s/it]
 25%|██▌       | 785/3123 [1:43:24<5:04:29,  7.81s/it]
 25%|██▌       | 786/3123 [1:43:32<5:07:22,  7.89s/it]
 25%|██▌       | 787/3123 [1:43:40<5:05:52,  7.86s/it]
 25%|██▌       | 788/3123 [1:43:47<5:04:36,  7.83s/it]
 25%|██▌       | 789/3123 [1:43:55<5:07:22,  7.90s/it]
 25%|██▌       | 790/3123 [1:44:03<5:05:43,  7.86s/it]
                                                      
{'loss': '0.02899', 'grad_norm': '0.9766', 'learning_rate': '0.0001', 'ppl': '1.029', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '89.23', 'tokens/total': 12943360, 'tokens/trainable': 4056679, 'epoch': '0.7583'}
+
 25%|██▌       | 790/3123 [1:44:03<5:05:43,  7.86s/it]
 25%|██▌       | 791/3123 [1:44:11<5:04:32,  7.84s/it]
 25%|██▌       | 792/3123 [1:44:19<5:03:35,  7.81s/it]
 25%|██▌       | 793/3123 [1:44:27<5:06:19,  7.89s/it]
 25%|██▌       | 794/3123 [1:44:35<5:04:53,  7.85s/it]
 25%|██▌       | 795/3123 [1:44:42<5:03:47,  7.83s/it]
 25%|██▌       | 796/3123 [1:44:50<5:06:22,  7.90s/it]
 26%|██▌       | 797/3123 [1:44:58<5:04:39,  7.86s/it]
 26%|██▌       | 798/3123 [1:45:06<5:03:34,  7.83s/it]
 26%|██▌       | 799/3123 [1:45:14<5:02:45,  7.82s/it]
 26%|██▌       | 800/3123 [1:45:22<5:05:28,  7.89s/it]
                                                      
{'loss': '0.01191', 'grad_norm': '0.291', 'learning_rate': '0.0001', 'ppl': '1.012', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '93.23', 'tokens/total': 13107200, 'tokens/trainable': 4107997, 'epoch': '0.7679'}
+
 26%|██▌       | 800/3123 [1:45:22<5:05:28,  7.89s/it]
 26%|██▌       | 801/3123 [1:45:30<5:03:58,  7.85s/it]
 26%|██▌       | 802/3123 [1:45:37<5:02:56,  7.83s/it]
 26%|██▌       | 803/3123 [1:45:45<5:05:07,  7.89s/it]
 26%|██▌       | 804/3123 [1:45:53<5:03:37,  7.86s/it]
 26%|██▌       | 805/3123 [1:46:01<5:02:30,  7.83s/it]
 26%|██▌       | 806/3123 [1:46:09<5:05:01,  7.90s/it]
 26%|██▌       | 807/3123 [1:46:17<5:03:13,  7.86s/it]
 26%|██▌       | 808/3123 [1:46:24<5:02:05,  7.83s/it]
 26%|██▌       | 809/3123 [1:46:32<5:01:14,  7.81s/it]
 26%|██▌       | 810/3123 [1:46:40<5:03:59,  7.89s/it]
                                                      
{'loss': '0.006751', 'grad_norm': '0.2344', 'learning_rate': '0.0001', 'ppl': '1.007', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '79.2', 'tokens/total': 13271040, 'tokens/trainable': 4158984, 'epoch': '0.7775'}
+
 26%|██▌       | 810/3123 [1:46:41<5:03:59,  7.89s/it]
 26%|██▌       | 811/3123 [1:46:48<5:02:32,  7.85s/it]
 26%|██▌       | 812/3123 [1:46:56<5:01:29,  7.83s/it]
 26%|██▌       | 813/3123 [1:47:04<5:03:55,  7.89s/it]
 26%|██▌       | 814/3123 [1:47:12<5:02:19,  7.86s/it]
 26%|██▌       | 815/3123 [1:47:19<5:01:10,  7.83s/it]
 26%|██▌       | 816/3123 [1:47:27<5:00:25,  7.81s/it]
 26%|██▌       | 817/3123 [1:47:35<5:03:02,  7.89s/it]
 26%|██▌       | 818/3123 [1:47:43<5:01:37,  7.85s/it]
 26%|██▌       | 819/3123 [1:47:51<5:00:28,  7.82s/it]
 26%|██▋       | 820/3123 [1:47:59<5:03:12,  7.90s/it]
                                                      
{'loss': '0.005448', 'grad_norm': '0.2949', 'learning_rate': '0.0001', 'ppl': '1.005', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '91.82', 'tokens/total': 13434880, 'tokens/trainable': 4210162, 'epoch': '0.7871'}
+
 26%|██▋       | 820/3123 [1:47:59<5:03:12,  7.90s/it]
 26%|██▋       | 821/3123 [1:48:07<5:01:33,  7.86s/it]
 26%|██▋       | 822/3123 [1:48:14<5:00:22,  7.83s/it]
 26%|██▋       | 823/3123 [1:48:22<4:59:29,  7.81s/it]
 26%|██▋       | 824/3123 [1:48:30<5:02:01,  7.88s/it]
 26%|██▋       | 825/3123 [1:48:38<5:00:38,  7.85s/it]
 26%|██▋       | 826/3123 [1:48:46<4:59:28,  7.82s/it]
 26%|██▋       | 827/3123 [1:48:54<5:02:12,  7.90s/it]
 27%|██▋       | 828/3123 [1:49:02<5:00:27,  7.85s/it]
 27%|██▋       | 829/3123 [1:49:09<4:59:19,  7.83s/it]
 27%|██▋       | 830/3123 [1:49:17<5:01:26,  7.89s/it]
                                                      
{'loss': '0.005318', 'grad_norm': '0.3574', 'learning_rate': '0.0001', 'ppl': '1.005', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '77.35', 'tokens/total': 13598720, 'tokens/trainable': 4261207, 'epoch': '0.7967'}
+
 27%|██▋       | 830/3123 [1:49:18<5:01:26,  7.89s/it]
 27%|██▋       | 831/3123 [1:49:25<4:59:57,  7.85s/it]
 27%|██▋       | 832/3123 [1:49:33<4:58:50,  7.83s/it]
 27%|██▋       | 833/3123 [1:49:41<4:58:04,  7.81s/it]
 27%|██▋       | 834/3123 [1:49:49<5:00:48,  7.88s/it]
 27%|██▋       | 835/3123 [1:49:57<4:59:20,  7.85s/it]
 27%|██▋       | 836/3123 [1:50:04<4:58:11,  7.82s/it]
 27%|██▋       | 837/3123 [1:50:12<5:00:37,  7.89s/it]
 27%|██▋       | 838/3123 [1:50:20<4:59:04,  7.85s/it]
 27%|██▋       | 839/3123 [1:50:28<4:57:58,  7.83s/it]
 27%|██▋       | 840/3123 [1:50:36<4:57:09,  7.81s/it]
                                                      
{'loss': '0.004299', 'grad_norm': '0.543', 'learning_rate': '0.0001', 'ppl': '1.004', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '82.04', 'tokens/total': 13762560, 'tokens/trainable': 4312249, 'epoch': '0.8063'}
+
 27%|██▋       | 840/3123 [1:50:36<4:57:09,  7.81s/it]
 27%|██▋       | 841/3123 [1:50:44<4:59:56,  7.89s/it]
 27%|██▋       | 842/3123 [1:50:51<4:58:30,  7.85s/it]
 27%|██▋       | 843/3123 [1:50:59<4:57:28,  7.83s/it]
 27%|██▋       | 844/3123 [1:51:07<4:59:50,  7.89s/it]
 27%|██▋       | 845/3123 [1:51:15<4:58:19,  7.86s/it]
 27%|██▋       | 846/3123 [1:51:23<4:57:18,  7.83s/it]
 27%|██▋       | 847/3123 [1:51:31<4:56:29,  7.82s/it]
 27%|██▋       | 848/3123 [1:51:39<4:59:15,  7.89s/it]
 27%|██▋       | 849/3123 [1:51:46<4:57:43,  7.86s/it]
 27%|██▋       | 850/3123 [1:51:54<4:56:39,  7.83s/it]
                                                      
{'loss': '0.006567', 'grad_norm': '0.3691', 'learning_rate': '0.0001', 'ppl': '1.007', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '90.01', 'tokens/total': 13926400, 'tokens/trainable': 4363740, 'epoch': '0.8159'}
+
 27%|██▋       | 850/3123 [1:51:55<4:56:39,  7.83s/it]
 27%|██▋       | 851/3123 [1:52:02<4:58:59,  7.90s/it]
 27%|██▋       | 852/3123 [1:52:10<4:57:30,  7.86s/it]
 27%|██▋       | 853/3123 [1:52:18<4:56:19,  7.83s/it]
 27%|██▋       | 854/3123 [1:52:26<4:55:32,  7.82s/it]
 27%|██▋       | 855/3123 [1:52:34<4:58:11,  7.89s/it]
 27%|██▋       | 856/3123 [1:52:41<4:56:43,  7.85s/it]
 27%|██▋       | 857/3123 [1:52:49<4:55:34,  7.83s/it]
 27%|██▋       | 858/3123 [1:52:57<4:58:02,  7.90s/it]
 28%|██▊       | 859/3123 [1:53:05<4:56:33,  7.86s/it]
 28%|██▊       | 860/3123 [1:53:13<4:55:24,  7.83s/it]
                                                      
{'loss': '0.005271', 'grad_norm': '0.2139', 'learning_rate': '0.0001', 'ppl': '1.005', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '85.22', 'tokens/total': 14090240, 'tokens/trainable': 4414546, 'epoch': '0.8255'}
+
 28%|██▊       | 860/3123 [1:53:13<4:55:24,  7.83s/it]
 28%|██▊       | 861/3123 [1:53:21<4:57:56,  7.90s/it]
 28%|██▊       | 862/3123 [1:53:29<4:56:17,  7.86s/it]
 28%|██▊       | 863/3123 [1:53:36<4:55:09,  7.84s/it]
 28%|██▊       | 864/3123 [1:53:44<4:54:20,  7.82s/it]
 28%|██▊       | 865/3123 [1:53:52<4:57:10,  7.90s/it]
 28%|██▊       | 866/3123 [1:54:00<4:55:34,  7.86s/it]
 28%|██▊       | 867/3123 [1:54:08<4:54:26,  7.83s/it]
 28%|██▊       | 868/3123 [1:54:16<4:57:00,  7.90s/it]
 28%|██▊       | 869/3123 [1:54:24<4:55:20,  7.86s/it]
 28%|██▊       | 870/3123 [1:54:31<4:54:11,  7.83s/it]
                                                      
{'loss': '0.004132', 'grad_norm': '0.252', 'learning_rate': '0.0001', 'ppl': '1.004', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '91.62', 'tokens/total': 14254080, 'tokens/trainable': 4465604, 'epoch': '0.8351'}
+
 28%|██▊       | 870/3123 [1:54:32<4:54:11,  7.83s/it]
 28%|██▊       | 871/3123 [1:54:39<4:53:23,  7.82s/it]
 28%|██▊       | 872/3123 [1:54:47<4:55:54,  7.89s/it]
 28%|██▊       | 873/3123 [1:54:55<4:54:29,  7.85s/it]
 28%|██▊       | 874/3123 [1:55:03<4:53:24,  7.83s/it]
 28%|██▊       | 875/3123 [1:55:11<4:56:16,  7.91s/it]
 28%|██▊       | 876/3123 [1:55:19<4:54:36,  7.87s/it]
 28%|██▊       | 877/3123 [1:55:26<4:53:31,  7.84s/it]
 28%|██▊       | 878/3123 [1:55:34<4:52:34,  7.82s/it]
 28%|██▊       | 879/3123 [1:55:42<4:54:58,  7.89s/it]
 28%|██▊       | 880/3123 [1:55:50<4:53:37,  7.85s/it]
                                                      
{'loss': '0.003341', 'grad_norm': '0.1436', 'learning_rate': '0.0001', 'ppl': '1.003', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '92.62', 'tokens/total': 14417920, 'tokens/trainable': 4516755, 'epoch': '0.8447'}
+
 28%|██▊       | 880/3123 [1:55:50<4:53:37,  7.85s/it]
 28%|██▊       | 881/3123 [1:55:58<4:52:42,  7.83s/it]
 28%|██▊       | 882/3123 [1:56:06<4:55:17,  7.91s/it]
 28%|██▊       | 883/3123 [1:56:14<4:53:44,  7.87s/it]
 28%|██▊       | 884/3123 [1:56:21<4:52:31,  7.84s/it]
 28%|██▊       | 885/3123 [1:56:29<4:51:39,  7.82s/it]
 28%|██▊       | 886/3123 [1:56:37<4:54:01,  7.89s/it]
 28%|██▊       | 887/3123 [1:56:45<4:52:42,  7.85s/it]
 28%|██▊       | 888/3123 [1:56:53<4:51:40,  7.83s/it]
 28%|██▊       | 889/3123 [1:57:01<4:54:08,  7.90s/it]
 28%|██▊       | 890/3123 [1:57:09<4:52:36,  7.86s/it]
                                                      
{'loss': '0.002202', 'grad_norm': '0.1338', 'learning_rate': '0.0001', 'ppl': '1.002', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '93.75', 'tokens/total': 14581760, 'tokens/trainable': 4568307, 'epoch': '0.8543'}
+
 28%|██▊       | 890/3123 [1:57:09<4:52:36,  7.86s/it]
 29%|██▊       | 891/3123 [1:57:16<4:51:31,  7.84s/it]
 29%|██▊       | 892/3123 [1:57:24<4:50:37,  7.82s/it]
 29%|██▊       | 893/3123 [1:57:32<4:53:01,  7.88s/it]
 29%|██▊       | 894/3123 [1:57:40<4:51:34,  7.85s/it]
 29%|██▊       | 895/3123 [1:57:48<4:50:34,  7.83s/it]
 29%|██▊       | 896/3123 [1:57:56<4:53:03,  7.90s/it]
 29%|██▊       | 897/3123 [1:58:04<4:51:29,  7.86s/it]
 29%|██▉       | 898/3123 [1:58:11<4:50:22,  7.83s/it]
 29%|██▉       | 899/3123 [1:58:19<4:52:38,  7.90s/it]
 29%|██▉       | 900/3123 [1:58:27<4:51:05,  7.86s/it]
                                                      
{'loss': '0.001743', 'grad_norm': '0.104', 'learning_rate': '0.0001', 'ppl': '1.002', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '74.67', 'tokens/total': 14745600, 'tokens/trainable': 4619967, 'epoch': '0.8639'}
+
 29%|██▉       | 900/3123 [1:58:27<4:51:05,  7.86s/it]
 29%|██▉       | 901/3123 [1:58:35<4:49:59,  7.83s/it]
 29%|██▉       | 902/3123 [1:58:43<4:49:11,  7.81s/it]
 29%|██▉       | 903/3123 [1:58:51<4:51:47,  7.89s/it]
 29%|██▉       | 904/3123 [1:58:59<4:50:20,  7.85s/it]
 29%|██▉       | 905/3123 [1:59:06<4:49:16,  7.83s/it]
 29%|██▉       | 906/3123 [1:59:14<4:51:45,  7.90s/it]
 29%|██▉       | 907/3123 [1:59:22<4:50:14,  7.86s/it]
 29%|██▉       | 908/3123 [1:59:30<4:49:08,  7.83s/it]
 29%|██▉       | 909/3123 [1:59:38<4:48:16,  7.81s/it]
 29%|██▉       | 910/3123 [1:59:46<4:50:43,  7.88s/it]
                                                      
{'loss': '0.001868', 'grad_norm': '0.09668', 'learning_rate': '0.0001', 'ppl': '1.002', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '93.81', 'tokens/total': 14909440, 'tokens/trainable': 4671271, 'epoch': '0.8735'}
+
 29%|██▉       | 910/3123 [1:59:46<4:50:43,  7.88s/it]
 29%|██▉       | 911/3123 [1:59:53<4:49:25,  7.85s/it]
 29%|██▉       | 912/3123 [2:00:01<4:48:23,  7.83s/it]
 29%|██▉       | 913/3123 [2:00:09<4:51:00,  7.90s/it]
 29%|██▉       | 914/3123 [2:00:17<4:49:28,  7.86s/it]
 29%|██▉       | 915/3123 [2:00:25<4:48:18,  7.83s/it]
 29%|██▉       | 916/3123 [2:00:33<4:47:29,  7.82s/it]
 29%|██▉       | 917/3123 [2:00:41<4:50:09,  7.89s/it]
 29%|██▉       | 918/3123 [2:00:48<4:48:43,  7.86s/it]
 29%|██▉       | 919/3123 [2:00:56<4:47:37,  7.83s/it]
 29%|██▉       | 920/3123 [2:01:04<4:49:55,  7.90s/it]
                                                      
{'loss': '0.001774', 'grad_norm': '0.06494', 'learning_rate': '0.0001', 'ppl': '1.002', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '91.69', 'tokens/total': 15073280, 'tokens/trainable': 4723106, 'epoch': '0.8831'}
+
 29%|██▉       | 920/3123 [2:01:05<4:49:55,  7.90s/it]
 29%|██▉       | 921/3123 [2:01:12<4:48:24,  7.86s/it]
 30%|██▉       | 922/3123 [2:01:20<4:47:19,  7.83s/it]
 30%|██▉       | 923/3123 [2:01:28<4:49:41,  7.90s/it]
 30%|██▉       | 924/3123 [2:01:36<4:48:09,  7.86s/it]
 30%|██▉       | 925/3123 [2:01:43<4:46:58,  7.83s/it]
 30%|██▉       | 926/3123 [2:01:51<4:46:06,  7.81s/it]
 30%|██▉       | 927/3123 [2:01:59<4:48:37,  7.89s/it]
 30%|██▉       | 928/3123 [2:02:07<4:47:08,  7.85s/it]
 30%|██▉       | 929/3123 [2:02:15<4:46:04,  7.82s/it]
 30%|██▉       | 930/3123 [2:02:23<4:48:30,  7.89s/it]
                                                      
{'loss': '0.002928', 'grad_norm': '0.1572', 'learning_rate': '0.0001', 'ppl': '1.003', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '85.04', 'tokens/total': 15237120, 'tokens/trainable': 4774009, 'epoch': '0.8927'}
+
 30%|██▉       | 930/3123 [2:02:23<4:48:30,  7.89s/it]
 30%|██▉       | 931/3123 [2:02:31<4:47:05,  7.86s/it]
 30%|██▉       | 932/3123 [2:02:38<4:46:00,  7.83s/it]
 30%|██▉       | 933/3123 [2:02:46<4:45:10,  7.81s/it]
 30%|██▉       | 934/3123 [2:02:54<4:47:33,  7.88s/it]
 30%|██▉       | 935/3123 [2:03:02<4:46:12,  7.85s/it]
 30%|██▉       | 936/3123 [2:03:10<4:45:17,  7.83s/it]
 30%|███       | 937/3123 [2:03:18<4:47:39,  7.90s/it]
 30%|███       | 938/3123 [2:03:26<4:46:05,  7.86s/it]
 30%|███       | 939/3123 [2:03:33<4:45:02,  7.83s/it]
 30%|███       | 940/3123 [2:03:41<4:47:14,  7.89s/it]
                                                      
{'loss': '0.002748', 'grad_norm': '0.09326', 'learning_rate': '0.0001', 'ppl': '1.003', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '89.04', 'tokens/total': 15400960, 'tokens/trainable': 4825081, 'epoch': '0.9023'}
+
 30%|███       | 940/3123 [2:03:42<4:47:14,  7.89s/it]
 30%|███       | 941/3123 [2:03:49<4:45:46,  7.86s/it]
 30%|███       | 942/3123 [2:03:57<4:44:41,  7.83s/it]
 30%|███       | 943/3123 [2:04:05<4:43:55,  7.81s/it]
 30%|███       | 944/3123 [2:04:13<4:46:32,  7.89s/it]
 30%|███       | 945/3123 [2:04:21<4:45:09,  7.86s/it]
 30%|███       | 946/3123 [2:04:28<4:44:06,  7.83s/it]
 30%|███       | 947/3123 [2:04:36<4:46:32,  7.90s/it]
 30%|███       | 948/3123 [2:04:44<4:45:03,  7.86s/it]
 30%|███       | 949/3123 [2:04:52<4:43:54,  7.84s/it]
 30%|███       | 950/3123 [2:05:00<4:43:00,  7.81s/it]
                                                      
{'loss': '0.001598', 'grad_norm': '0.103', 'learning_rate': '0.0001', 'ppl': '1.002', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '94.27', 'tokens/total': 15564800, 'tokens/trainable': 4877201, 'epoch': '0.9119'}
+
 30%|███       | 950/3123 [2:05:00<4:43:00,  7.81s/it]
 30%|███       | 951/3123 [2:05:08<4:45:29,  7.89s/it]
 30%|███       | 952/3123 [2:05:16<4:44:06,  7.85s/it]
 31%|███       | 953/3123 [2:05:23<4:43:00,  7.83s/it]
 31%|███       | 954/3123 [2:05:31<4:45:15,  7.89s/it]
 31%|███       | 955/3123 [2:05:39<4:43:53,  7.86s/it]
 31%|███       | 956/3123 [2:05:47<4:42:53,  7.83s/it]
 31%|███       | 957/3123 [2:05:55<4:42:06,  7.81s/it]
 31%|███       | 958/3123 [2:06:03<4:44:41,  7.89s/it]
 31%|███       | 959/3123 [2:06:10<4:43:16,  7.85s/it]
 31%|███       | 960/3123 [2:06:18<4:42:15,  7.83s/it]
                                                      
{'loss': '0.001896', 'grad_norm': '0.08105', 'learning_rate': '0.0001', 'ppl': '1.002', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '90.29', 'tokens/total': 15728640, 'tokens/trainable': 4929476, 'epoch': '0.9215'}
+
 31%|███       | 960/3123 [2:06:19<4:42:15,  7.83s/it]
 31%|███       | 961/3123 [2:06:26<4:44:27,  7.89s/it]
 31%|███       | 962/3123 [2:06:34<4:42:59,  7.86s/it]
 31%|███       | 963/3123 [2:06:42<4:42:01,  7.83s/it]
 31%|███       | 964/3123 [2:06:50<4:41:15,  7.82s/it]
 31%|███       | 965/3123 [2:06:58<4:43:54,  7.89s/it]
 31%|███       | 966/3123 [2:07:05<4:42:30,  7.86s/it]
 31%|███       | 967/3123 [2:07:13<4:41:28,  7.83s/it]
 31%|███       | 968/3123 [2:07:21<4:43:57,  7.91s/it]
 31%|███       | 969/3123 [2:07:29<4:42:25,  7.87s/it]
 31%|███       | 970/3123 [2:07:37<4:41:17,  7.84s/it]
                                                      
{'loss': '0.001703', 'grad_norm': '0.1484', 'learning_rate': '0.0001', 'ppl': '1.002', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '89.73', 'tokens/total': 15892480, 'tokens/trainable': 4981090, 'epoch': '0.9311'}
+
 31%|███       | 970/3123 [2:07:37<4:41:17,  7.84s/it]
 31%|███       | 971/3123 [2:07:45<4:43:22,  7.90s/it]
 31%|███       | 972/3123 [2:07:53<4:41:54,  7.86s/it]
 31%|███       | 973/3123 [2:08:00<4:40:44,  7.83s/it]
 31%|███       | 974/3123 [2:08:08<4:39:58,  7.82s/it]
 31%|███       | 975/3123 [2:08:16<4:42:18,  7.89s/it]
 31%|███▏      | 976/3123 [2:08:24<4:40:57,  7.85s/it]
 31%|███▏      | 977/3123 [2:08:32<4:39:58,  7.83s/it]
 31%|███▏      | 978/3123 [2:08:40<4:42:31,  7.90s/it]
 31%|███▏      | 979/3123 [2:08:48<4:41:04,  7.87s/it]
 31%|███▏      | 980/3123 [2:08:55<4:39:57,  7.84s/it]
                                                      
{'loss': '0.001243', 'grad_norm': '0.03809', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '86.38', 'tokens/total': 16056320, 'tokens/trainable': 5032187, 'epoch': '0.9407'}
+
 31%|███▏      | 980/3123 [2:08:56<4:39:57,  7.84s/it]
 31%|███▏      | 981/3123 [2:09:03<4:39:06,  7.82s/it]
 31%|███▏      | 982/3123 [2:09:11<4:41:29,  7.89s/it]
 31%|███▏      | 983/3123 [2:09:19<4:40:05,  7.85s/it]
 32%|███▏      | 984/3123 [2:09:27<4:39:03,  7.83s/it]
 32%|███▏      | 985/3123 [2:09:35<4:41:21,  7.90s/it]
 32%|███▏      | 986/3123 [2:09:43<4:39:54,  7.86s/it]
 32%|███▏      | 987/3123 [2:09:50<4:38:51,  7.83s/it]
 32%|███▏      | 988/3123 [2:09:58<4:38:02,  7.81s/it]
 32%|███▏      | 989/3123 [2:10:06<4:40:28,  7.89s/it]
 32%|███▏      | 990/3123 [2:10:14<4:39:03,  7.85s/it]
                                                      
{'loss': '0.001601', 'grad_norm': '0.1768', 'learning_rate': '0.0001', 'ppl': '1.002', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '92.52', 'tokens/total': 16220160, 'tokens/trainable': 5083255, 'epoch': '0.9503'}
+
 32%|███▏      | 990/3123 [2:10:14<4:39:03,  7.85s/it]
 32%|███▏      | 991/3123 [2:10:22<4:38:05,  7.83s/it]
 32%|███▏      | 992/3123 [2:10:30<4:40:20,  7.89s/it]
 32%|███▏      | 993/3123 [2:10:38<4:38:52,  7.86s/it]
 32%|███▏      | 994/3123 [2:10:45<4:37:45,  7.83s/it]
 32%|███▏      | 995/3123 [2:10:53<4:39:57,  7.89s/it]
 32%|███▏      | 996/3123 [2:11:01<4:38:25,  7.85s/it]
 32%|███▏      | 997/3123 [2:11:09<4:37:26,  7.83s/it]
 32%|███▏      | 998/3123 [2:11:17<4:36:36,  7.81s/it]
 32%|███▏      | 999/3123 [2:11:25<4:39:15,  7.89s/it]
 32%|███▏      | 1000/3123 [2:11:33<4:37:44,  7.85s/it]
                                                       
{'loss': '0.001584', 'grad_norm': '0.1738', 'learning_rate': '0.0001', 'ppl': '1.002', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '80.4', 'tokens/total': 16384000, 'tokens/trainable': 5133643, 'epoch': '0.9599'}
+
 32%|███▏      | 1000/3123 [2:11:33<4:37:44,  7.85s/it]
 32%|███▏      | 1001/3123 [2:11:40<4:36:48,  7.83s/it]
 32%|███▏      | 1002/3123 [2:11:48<4:38:59,  7.89s/it]
 32%|███▏      | 1003/3123 [2:11:56<4:37:41,  7.86s/it]
 32%|███▏      | 1004/3123 [2:12:04<4:36:31,  7.83s/it]
 32%|███▏      | 1005/3123 [2:12:12<4:35:45,  7.81s/it]
 32%|███▏      | 1006/3123 [2:12:20<4:38:18,  7.89s/it]
 32%|███▏      | 1007/3123 [2:12:28<4:36:59,  7.85s/it]
 32%|███▏      | 1008/3123 [2:12:35<4:35:58,  7.83s/it]
 32%|███▏      | 1009/3123 [2:12:43<4:38:29,  7.90s/it]
 32%|███▏      | 1010/3123 [2:12:51<4:36:55,  7.86s/it]
                                                       
{'loss': '0.002674', 'grad_norm': '0.07812', 'learning_rate': '0.0001', 'ppl': '1.003', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '76.93', 'tokens/total': 16547840, 'tokens/trainable': 5185486, 'epoch': '0.9695'}
+
 32%|███▏      | 1010/3123 [2:12:51<4:36:55,  7.86s/it]
 32%|███▏      | 1011/3123 [2:12:59<4:35:53,  7.84s/it]
 32%|███▏      | 1012/3123 [2:13:07<4:35:04,  7.82s/it]
 32%|███▏      | 1013/3123 [2:13:15<4:37:33,  7.89s/it]
 32%|███▏      | 1014/3123 [2:13:23<4:36:09,  7.86s/it]
 33%|███▎      | 1015/3123 [2:13:30<4:35:04,  7.83s/it]
 33%|███▎      | 1016/3123 [2:13:38<4:37:23,  7.90s/it]
 33%|███▎      | 1017/3123 [2:13:46<4:35:57,  7.86s/it]
 33%|███▎      | 1018/3123 [2:13:54<4:34:57,  7.84s/it]
 33%|███▎      | 1019/3123 [2:14:02<4:34:11,  7.82s/it]
 33%|███▎      | 1020/3123 [2:14:10<4:36:48,  7.90s/it]
                                                       
{'loss': '0.001443', 'grad_norm': '0.03064', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '73.7', 'tokens/total': 16711680, 'tokens/trainable': 5236078, 'epoch': '0.9791'}
+
 33%|███▎      | 1020/3123 [2:14:10<4:36:48,  7.90s/it]
 33%|███▎      | 1021/3123 [2:14:18<4:35:23,  7.86s/it]
 33%|███▎      | 1022/3123 [2:14:25<4:34:28,  7.84s/it]
 33%|███▎      | 1023/3123 [2:14:33<4:36:45,  7.91s/it]
 33%|███▎      | 1024/3123 [2:14:41<4:35:19,  7.87s/it]
 33%|███▎      | 1025/3123 [2:14:49<4:34:12,  7.84s/it]
 33%|███▎      | 1026/3123 [2:14:57<4:36:55,  7.92s/it]
 33%|███▎      | 1027/3123 [2:15:05<4:35:13,  7.88s/it]
 33%|███▎      | 1028/3123 [2:15:13<4:34:02,  7.85s/it]
 33%|███▎      | 1029/3123 [2:15:20<4:33:01,  7.82s/it]
 33%|███▎      | 1030/3123 [2:15:28<4:35:24,  7.90s/it]
                                                       
{'loss': '0.001552', 'grad_norm': '0.04492', 'learning_rate': '0.0001', 'ppl': '1.002', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '84.83', 'tokens/total': 16875520, 'tokens/trainable': 5287369, 'epoch': '0.9887'}
+
 33%|███▎      | 1030/3123 [2:15:29<4:35:24,  7.90s/it]
 33%|███▎      | 1031/3123 [2:15:36<4:34:01,  7.86s/it]
 33%|███▎      | 1032/3123 [2:15:44<4:33:00,  7.83s/it]
 33%|███▎      | 1033/3123 [2:15:52<4:35:28,  7.91s/it]
 33%|███▎      | 1034/3123 [2:16:00<4:34:00,  7.87s/it]
 33%|███▎      | 1035/3123 [2:16:08<4:32:50,  7.84s/it]
 33%|███▎      | 1036/3123 [2:16:15<4:31:56,  7.82s/it]
 33%|███▎      | 1037/3123 [2:16:23<4:34:18,  7.89s/it]
 33%|███▎      | 1038/3123 [2:16:31<4:33:01,  7.86s/it]
 33%|███▎      | 1039/3123 [2:16:39<4:32:03,  7.83s/it]
 33%|███▎      | 1040/3123 [2:16:47<4:34:30,  7.91s/it]
                                                       
{'loss': '0.001351', 'grad_norm': '0.09424', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '89.41', 'tokens/total': 17039360, 'tokens/trainable': 5338389, 'epoch': '0.9983'}
+
 33%|███▎      | 1040/3123 [2:16:47<4:34:30,  7.91s/it]
 33%|███▎      | 1041/3123 [2:16:55<4:33:01,  7.87s/it][2026-03-08 18:14:09,370] [INFO] [axolotl.core.trainers.base._save:721] [PID:1659682] Saving model checkpoint to /home/jiaruil5/math_rl/mix_teachers/r3lit_rl/mix_teachers/checkpoints/math_operations/primitive_atomic_full_sft_50k_lr1e4_t20260308/checkpoint-1041
+
+
Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]
+
Writing model shards: 100%|██████████| 1/1 [00:21<00:00, 21.22s/it]
Writing model shards: 100%|██████████| 1/1 [00:21<00:00, 21.22s/it]Traceback (most recent call last):
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/multiprocessing/util.py", line 300, in _run_finalizers
+    finalizer()
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/multiprocessing/util.py", line 224, in __call__
+    res = self._callback(*self._args, **self._kwargs)
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/multiprocessing/util.py", line 133, in _remove_temp_dir
+    rmtree(tempdir)
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/shutil.py", line 725, in rmtree
+    _rmtree_safe_fd(fd, path, onerror)
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/shutil.py", line 681, in _rmtree_safe_fd
+    onerror(os.unlink, fullname, sys.exc_info())
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/shutil.py", line 679, in _rmtree_safe_fd
+    os.unlink(entry.name, dir_fd=topfd)
+OSError: [Errno 16] Device or resource busy: '.nfs00000000000dace200012507'
+
+
 33%|███▎      | 1042/3123 [2:18:20<17:52:06, 30.91s/it][2026-03-08 18:15:33,998] [INFO] [axolotl.core.trainers.base.evaluate:400] [PID:1659682] Running evaluation step...
+[2026-03-08 18:15:35,012] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:1659682] generate_batches time: 0.47913289070129395
+[2026-03-08 18:15:35,476] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:1659682] generate_batches time: 0.4633975028991699
+[2026-03-08 18:15:35,941] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:1659682] generate_batches time: 0.46491360664367676
+[2026-03-08 18:15:36,415] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:1659682] generate_batches time: 0.4733400344848633
+[2026-03-08 18:15:36,415] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:1659682] gather_len_batches: [34]
+
+
  0%|          | 0/34 [00:00<?, ?it/s]
+
  6%|▌         | 2/34 [00:00<00:03,  8.58it/s]
+
  9%|▉         | 3/34 [00:00<00:05,  6.01it/s]
+
 12%|█▏        | 4/34 [00:00<00:05,  5.19it/s]
+
 15%|█▍        | 5/34 [00:00<00:06,  4.82it/s]
+
 18%|█▊        | 6/34 [00:01<00:06,  4.60it/s]
+
 21%|██        | 7/34 [00:01<00:06,  4.48it/s]
+
 24%|██▎       | 8/34 [00:01<00:05,  4.40it/s]
+
 26%|██▋       | 9/34 [00:01<00:06,  3.87it/s]
+
 29%|██▉       | 10/34 [00:02<00:05,  4.03it/s]
+
 32%|███▏      | 11/34 [00:02<00:05,  4.09it/s]
+
 35%|███▌      | 12/34 [00:02<00:05,  4.13it/s]
+
 38%|███▊      | 13/34 [00:02<00:05,  4.16it/s]
+
 41%|████      | 14/34 [00:03<00:04,  4.19it/s]
+
 44%|████▍     | 15/34 [00:03<00:04,  4.21it/s]
+
 47%|████▋     | 16/34 [00:03<00:04,  4.21it/s]
+
 50%|█████     | 17/34 [00:03<00:04,  4.11it/s]
+
 53%|█████▎    | 18/34 [00:04<00:03,  4.18it/s]
+
 56%|█████▌    | 19/34 [00:04<00:03,  4.20it/s]
+
 59%|█████▉    | 20/34 [00:04<00:03,  4.21it/s]
+
 62%|██████▏   | 21/34 [00:04<00:03,  4.21it/s]
+
 65%|██████▍   | 22/34 [00:05<00:02,  4.22it/s]
+
 68%|██████▊   | 23/34 [00:05<00:02,  4.23it/s]
+
 71%|███████   | 24/34 [00:05<00:02,  4.23it/s]
+
 74%|███████▎  | 25/34 [00:05<00:02,  4.07it/s]
+
 76%|███████▋  | 26/34 [00:06<00:01,  4.17it/s]
+
 79%|███████▉  | 27/34 [00:06<00:01,  4.20it/s]
+
 82%|████████▏ | 28/34 [00:06<00:01,  4.21it/s]
+
 85%|████████▌ | 29/34 [00:06<00:01,  4.22it/s]
+
 88%|████████▊ | 30/34 [00:06<00:00,  4.23it/s]
+
 91%|█████████ | 31/34 [00:07<00:00,  4.23it/s]
+
 94%|█████████▍| 32/34 [00:07<00:00,  4.24it/s]
+
 97%|█████████▋| 33/34 [00:07<00:00,  4.10it/s]Traceback (most recent call last):
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/multiprocessing/util.py", line 300, in _run_finalizers
+    finalizer()
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/multiprocessing/util.py", line 224, in __call__
+    res = self._callback(*self._args, **self._kwargs)
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/multiprocessing/util.py", line 133, in _remove_temp_dir
+    rmtree(tempdir)
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/shutil.py", line 725, in rmtree
+    _rmtree_safe_fd(fd, path, onerror)
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/shutil.py", line 681, in _rmtree_safe_fd
+    onerror(os.unlink, fullname, sys.exc_info())
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/shutil.py", line 679, in _rmtree_safe_fd
+    os.unlink(entry.name, dir_fd=topfd)
+OSError: [Errno 16] Device or resource busy: '.nfs00000000000dafbc00012508'
+
+
100%|██████████| 34/34 [00:07<00:00,  4.00it/s]
                                                        
+
                                               
{'eval_loss': '0.001464', 'eval_runtime': '8.873', 'eval_samples_per_second': '22.54', 'eval_steps_per_second': '22.54', 'eval_ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'epoch': '1.001', 'tokens/train_per_sec_per_gpu': '76.74'}
+
 33%|███▎      | 1042/3123 [2:18:31<17:52:06, 30.91s/it]
+
100%|██████████| 34/34 [00:07<00:00,  4.00it/s]
+
                                               
 33%|███▎      | 1043/3123 [2:18:38<15:45:09, 27.26s/it]
 33%|███▎      | 1044/3123 [2:18:46<12:21:31, 21.40s/it]
 33%|███▎      | 1045/3123 [2:18:54<9:59:10, 17.30s/it] 
 33%|███▎      | 1046/3123 [2:19:02<8:23:02, 14.53s/it]
 34%|███▎      | 1047/3123 [2:19:10<7:12:24, 12.50s/it]
 34%|███▎      | 1048/3123 [2:19:17<6:22:57, 11.07s/it]
 34%|███▎      | 1049/3123 [2:19:25<5:51:26, 10.17s/it]
 34%|███▎      | 1050/3123 [2:19:33<5:26:17,  9.44s/it]
                                                       
{'loss': '0.001145', 'grad_norm': '0.05249', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.96', 'memory/max_allocated (GiB)': '33.96', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '100.7', 'tokens/total': 17213440, 'tokens/trainable': 5392886, 'epoch': '1.009'}
+
 34%|███▎      | 1050/3123 [2:19:33<5:26:17,  9.44s/it]
 34%|███▎      | 1051/3123 [2:19:41<5:08:43,  8.94s/it]
 34%|███▎      | 1052/3123 [2:19:49<4:56:21,  8.59s/it]
 34%|███▎      | 1053/3123 [2:19:57<4:50:33,  8.42s/it]
 34%|███▎      | 1054/3123 [2:20:04<4:43:40,  8.23s/it]
 34%|███▍      | 1055/3123 [2:20:12<4:38:42,  8.09s/it]
 34%|███▍      | 1056/3123 [2:20:20<4:38:19,  8.08s/it]
 34%|███▍      | 1057/3123 [2:20:28<4:34:50,  7.98s/it]
 34%|███▍      | 1058/3123 [2:20:36<4:32:28,  7.92s/it]
 34%|███▍      | 1059/3123 [2:20:44<4:30:42,  7.87s/it]
 34%|███▍      | 1060/3123 [2:20:52<4:32:36,  7.93s/it]
                                                       
{'loss': '0.00112', 'grad_norm': '0.09375', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '84.8', 'tokens/total': 17377280, 'tokens/trainable': 5444495, 'epoch': '1.018'}
+
 34%|███▍      | 1060/3123 [2:20:52<4:32:36,  7.93s/it]
 34%|███▍      | 1061/3123 [2:20:59<4:30:47,  7.88s/it]
 34%|███▍      | 1062/3123 [2:21:07<4:29:27,  7.84s/it]
 34%|███▍      | 1063/3123 [2:21:15<4:31:22,  7.90s/it]
 34%|███▍      | 1064/3123 [2:21:23<4:29:49,  7.86s/it]
 34%|███▍      | 1065/3123 [2:21:31<4:28:43,  7.83s/it]
 34%|███▍      | 1066/3123 [2:21:39<4:27:55,  7.82s/it]
 34%|███▍      | 1067/3123 [2:21:47<4:30:16,  7.89s/it]
 34%|███▍      | 1068/3123 [2:21:54<4:28:51,  7.85s/it]
 34%|███▍      | 1069/3123 [2:22:02<4:27:54,  7.83s/it]
 34%|███▍      | 1070/3123 [2:22:10<4:30:14,  7.90s/it]
                                                       
{'loss': '0.001095', 'grad_norm': '0.07129', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '84.68', 'tokens/total': 17541120, 'tokens/trainable': 5496208, 'epoch': '1.028'}
+
 34%|███▍      | 1070/3123 [2:22:10<4:30:14,  7.90s/it]
 34%|███▍      | 1071/3123 [2:22:18<4:28:49,  7.86s/it]
 34%|███▍      | 1072/3123 [2:22:26<4:27:45,  7.83s/it]
 34%|███▍      | 1073/3123 [2:22:33<4:26:56,  7.81s/it]
 34%|███▍      | 1074/3123 [2:22:42<4:29:08,  7.88s/it]
 34%|███▍      | 1075/3123 [2:22:49<4:27:49,  7.85s/it]
 34%|███▍      | 1076/3123 [2:22:57<4:26:50,  7.82s/it]
 34%|███▍      | 1077/3123 [2:23:05<4:29:09,  7.89s/it]
 35%|███▍      | 1078/3123 [2:23:13<4:27:38,  7.85s/it]
 35%|███▍      | 1079/3123 [2:23:21<4:26:33,  7.82s/it]
 35%|███▍      | 1080/3123 [2:23:29<4:28:47,  7.89s/it]
                                                       
{'loss': '0.0009318', 'grad_norm': '0.01843', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '75.63', 'tokens/total': 17704960, 'tokens/trainable': 5547530, 'epoch': '1.037'}
+
 35%|███▍      | 1080/3123 [2:23:29<4:28:47,  7.89s/it]
 35%|███▍      | 1081/3123 [2:23:36<4:27:17,  7.85s/it]
 35%|███▍      | 1082/3123 [2:23:44<4:26:12,  7.83s/it]
 35%|███▍      | 1083/3123 [2:23:52<4:25:23,  7.81s/it]
 35%|███▍      | 1084/3123 [2:24:00<4:27:39,  7.88s/it]
 35%|███▍      | 1085/3123 [2:24:08<4:26:21,  7.84s/it]
 35%|███▍      | 1086/3123 [2:24:16<4:25:25,  7.82s/it]
 35%|███▍      | 1087/3123 [2:24:24<4:27:43,  7.89s/it]
 35%|███▍      | 1088/3123 [2:24:31<4:26:17,  7.85s/it]
 35%|███▍      | 1089/3123 [2:24:39<4:25:13,  7.82s/it]
 35%|███▍      | 1090/3123 [2:24:47<4:24:24,  7.80s/it]
                                                       
{'loss': '0.0007877', 'grad_norm': '0.09912', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '83.9', 'tokens/total': 17868800, 'tokens/trainable': 5598017, 'epoch': '1.047'}
+
 35%|███▍      | 1090/3123 [2:24:47<4:24:24,  7.80s/it]
 35%|███▍      | 1091/3123 [2:24:55<4:26:59,  7.88s/it]
 35%|███▍      | 1092/3123 [2:25:03<4:25:40,  7.85s/it]
 35%|███▍      | 1093/3123 [2:25:10<4:24:40,  7.82s/it]
 35%|███▌      | 1094/3123 [2:25:19<4:26:55,  7.89s/it]
 35%|███▌      | 1095/3123 [2:25:26<4:25:27,  7.85s/it]
 35%|███▌      | 1096/3123 [2:25:34<4:24:25,  7.83s/it]
 35%|███▌      | 1097/3123 [2:25:42<4:23:37,  7.81s/it]
 35%|███▌      | 1098/3123 [2:25:50<4:26:16,  7.89s/it]
 35%|███▌      | 1099/3123 [2:25:58<4:24:53,  7.85s/it]
 35%|███▌      | 1100/3123 [2:26:05<4:23:52,  7.83s/it]
                                                       
{'loss': '0.000617', 'grad_norm': '0.02905', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '83.14', 'tokens/total': 18032640, 'tokens/trainable': 5649223, 'epoch': '1.057'}
+
 35%|███▌      | 1100/3123 [2:26:06<4:23:52,  7.83s/it]
 35%|███▌      | 1101/3123 [2:26:13<4:26:17,  7.90s/it]
 35%|███▌      | 1102/3123 [2:26:21<4:24:41,  7.86s/it]
 35%|███▌      | 1103/3123 [2:26:29<4:23:36,  7.83s/it]
 35%|███▌      | 1104/3123 [2:26:37<4:25:46,  7.90s/it]
 35%|███▌      | 1105/3123 [2:26:45<4:24:16,  7.86s/it]
 35%|███▌      | 1106/3123 [2:26:53<4:23:08,  7.83s/it]
 35%|███▌      | 1107/3123 [2:27:00<4:22:20,  7.81s/it]
 35%|███▌      | 1108/3123 [2:27:08<4:24:47,  7.88s/it]
 36%|███▌      | 1109/3123 [2:27:16<4:23:23,  7.85s/it]
 36%|███▌      | 1110/3123 [2:27:24<4:22:28,  7.82s/it]
                                                       
{'loss': '0.0008214', 'grad_norm': '0.05444', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '95.55', 'tokens/total': 18196480, 'tokens/trainable': 5701017, 'epoch': '1.066'}
+
 36%|███▌      | 1110/3123 [2:27:24<4:22:28,  7.82s/it]
 36%|███▌      | 1111/3123 [2:27:32<4:21:46,  7.81s/it]
 36%|███▌      | 1112/3123 [2:27:40<4:24:14,  7.88s/it]
 36%|███▌      | 1113/3123 [2:27:48<4:22:49,  7.85s/it]
 36%|███▌      | 1114/3123 [2:27:55<4:21:50,  7.82s/it]
 36%|███▌      | 1115/3123 [2:28:03<4:24:01,  7.89s/it]
 36%|███▌      | 1116/3123 [2:28:11<4:22:38,  7.85s/it]
 36%|███▌      | 1117/3123 [2:28:19<4:21:38,  7.83s/it]
 36%|███▌      | 1118/3123 [2:28:27<4:20:57,  7.81s/it]
 36%|███▌      | 1119/3123 [2:28:35<4:23:17,  7.88s/it]
 36%|███▌      | 1120/3123 [2:28:42<4:21:57,  7.85s/it]
                                                       
{'loss': '0.0007146', 'grad_norm': '0.0791', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '79.14', 'tokens/total': 18360320, 'tokens/trainable': 5752591, 'epoch': '1.076'}
+
 36%|███▌      | 1120/3123 [2:28:43<4:21:57,  7.85s/it]
 36%|███▌      | 1121/3123 [2:28:50<4:20:56,  7.82s/it]
 36%|███▌      | 1122/3123 [2:28:58<4:23:16,  7.89s/it]
 36%|███▌      | 1123/3123 [2:29:06<4:21:45,  7.85s/it]
 36%|███▌      | 1124/3123 [2:29:14<4:20:42,  7.83s/it]
 36%|███▌      | 1125/3123 [2:29:22<4:19:54,  7.81s/it]
 36%|███▌      | 1126/3123 [2:29:30<4:22:09,  7.88s/it]
 36%|███▌      | 1127/3123 [2:29:37<4:20:51,  7.84s/it]
 36%|███▌      | 1128/3123 [2:29:45<4:19:49,  7.81s/it]
 36%|███▌      | 1129/3123 [2:29:53<4:22:06,  7.89s/it]
 36%|███▌      | 1130/3123 [2:30:01<4:20:44,  7.85s/it]
                                                       
{'loss': '0.001074', 'grad_norm': '0.09375', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '94.42', 'tokens/total': 18524160, 'tokens/trainable': 5803980, 'epoch': '1.085'}
+
 36%|███▌      | 1130/3123 [2:30:01<4:20:44,  7.85s/it]
 36%|███▌      | 1131/3123 [2:30:09<4:19:43,  7.82s/it]
 36%|███▌      | 1132/3123 [2:30:17<4:21:46,  7.89s/it]
 36%|███▋      | 1133/3123 [2:30:24<4:20:16,  7.85s/it]
 36%|███▋      | 1134/3123 [2:30:32<4:19:11,  7.82s/it]
 36%|███▋      | 1135/3123 [2:30:40<4:18:28,  7.80s/it]
 36%|███▋      | 1136/3123 [2:30:48<4:20:38,  7.87s/it]
 36%|███▋      | 1137/3123 [2:30:56<4:19:23,  7.84s/it]
 36%|███▋      | 1138/3123 [2:31:04<4:18:29,  7.81s/it]
 36%|███▋      | 1139/3123 [2:31:12<4:20:51,  7.89s/it]
 37%|███▋      | 1140/3123 [2:31:19<4:19:24,  7.85s/it]
                                                       
{'loss': '0.001066', 'grad_norm': '0.06738', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '80.02', 'tokens/total': 18688000, 'tokens/trainable': 5854955, 'epoch': '1.095'}
+
 37%|███▋      | 1140/3123 [2:31:20<4:19:24,  7.85s/it]
 37%|███▋      | 1141/3123 [2:31:27<4:18:26,  7.82s/it]
 37%|███▋      | 1142/3123 [2:31:35<4:17:35,  7.80s/it]
 37%|███▋      | 1143/3123 [2:31:43<4:19:49,  7.87s/it]
 37%|███▋      | 1144/3123 [2:31:51<4:18:37,  7.84s/it]
 37%|███▋      | 1145/3123 [2:31:58<4:17:45,  7.82s/it]
 37%|███▋      | 1146/3123 [2:32:07<4:20:01,  7.89s/it]
 37%|███▋      | 1147/3123 [2:32:14<4:18:37,  7.85s/it]
 37%|███▋      | 1148/3123 [2:32:22<4:17:35,  7.83s/it]
 37%|███▋      | 1149/3123 [2:32:30<4:16:51,  7.81s/it]
 37%|███▋      | 1150/3123 [2:32:38<4:19:11,  7.88s/it]
                                                       
{'loss': '0.0005198', 'grad_norm': '0.02576', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '73.42', 'tokens/total': 18851840, 'tokens/trainable': 5905676, 'epoch': '1.105'}
+
 37%|███▋      | 1150/3123 [2:32:38<4:19:11,  7.88s/it]
 37%|███▋      | 1151/3123 [2:32:46<4:17:54,  7.85s/it]
 37%|███▋      | 1152/3123 [2:32:53<4:16:54,  7.82s/it]
 37%|███▋      | 1153/3123 [2:33:01<4:19:04,  7.89s/it]
 37%|███▋      | 1154/3123 [2:33:09<4:17:46,  7.86s/it]
 37%|███▋      | 1155/3123 [2:33:17<4:16:45,  7.83s/it]
 37%|███▋      | 1156/3123 [2:33:25<4:16:00,  7.81s/it]
 37%|███▋      | 1157/3123 [2:33:33<4:18:16,  7.88s/it]
 37%|███▋      | 1158/3123 [2:33:41<4:17:02,  7.85s/it]
 37%|███▋      | 1159/3123 [2:33:48<4:16:02,  7.82s/it]
 37%|███▋      | 1160/3123 [2:33:56<4:18:22,  7.90s/it]
                                                       
{'loss': '0.0009939', 'grad_norm': '0.01721', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '71.21', 'tokens/total': 19015680, 'tokens/trainable': 5956999, 'epoch': '1.114'}
+
 37%|███▋      | 1160/3123 [2:33:57<4:18:22,  7.90s/it]
 37%|███▋      | 1161/3123 [2:34:04<4:16:51,  7.85s/it]
 37%|███▋      | 1162/3123 [2:34:12<4:15:52,  7.83s/it]
 37%|███▋      | 1163/3123 [2:34:20<4:17:48,  7.89s/it]
 37%|███▋      | 1164/3123 [2:34:28<4:16:28,  7.86s/it]
 37%|███▋      | 1165/3123 [2:34:35<4:15:27,  7.83s/it]
 37%|███▋      | 1166/3123 [2:34:43<4:14:46,  7.81s/it]
 37%|███▋      | 1167/3123 [2:34:51<4:17:00,  7.88s/it]
 37%|███▋      | 1168/3123 [2:34:59<4:15:43,  7.85s/it]
 37%|███▋      | 1169/3123 [2:35:07<4:14:48,  7.82s/it]
 37%|███▋      | 1170/3123 [2:35:15<4:17:01,  7.90s/it]
                                                       
{'loss': '0.0008747', 'grad_norm': '0.08301', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '75.55', 'tokens/total': 19179520, 'tokens/trainable': 6008446, 'epoch': '1.124'}
+
 37%|███▋      | 1170/3123 [2:35:15<4:17:01,  7.90s/it]
 37%|███▋      | 1171/3123 [2:35:23<4:15:44,  7.86s/it]
 38%|███▊      | 1172/3123 [2:35:30<4:14:45,  7.83s/it]
 38%|███▊      | 1173/3123 [2:35:38<4:14:01,  7.82s/it]
 38%|███▊      | 1174/3123 [2:35:46<4:16:23,  7.89s/it]
 38%|███▊      | 1175/3123 [2:35:54<4:15:01,  7.85s/it]
 38%|███▊      | 1176/3123 [2:36:02<4:14:00,  7.83s/it]
 38%|███▊      | 1177/3123 [2:36:10<4:15:56,  7.89s/it]
 38%|███▊      | 1178/3123 [2:36:18<4:14:30,  7.85s/it]
 38%|███▊      | 1179/3123 [2:36:25<4:13:36,  7.83s/it]
 38%|███▊      | 1180/3123 [2:36:33<4:12:53,  7.81s/it]
                                                       
{'loss': '0.0009802', 'grad_norm': '0.1299', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '83.64', 'tokens/total': 19343360, 'tokens/trainable': 6060444, 'epoch': '1.133'}
+
 38%|███▊      | 1180/3123 [2:36:33<4:12:53,  7.81s/it]
 38%|███▊      | 1181/3123 [2:36:41<4:15:21,  7.89s/it]
 38%|███▊      | 1182/3123 [2:36:49<4:13:57,  7.85s/it]
 38%|███▊      | 1183/3123 [2:36:57<4:13:03,  7.83s/it]
 38%|███▊      | 1184/3123 [2:37:05<4:15:00,  7.89s/it]
 38%|███▊      | 1185/3123 [2:37:13<4:13:42,  7.85s/it]
 38%|███▊      | 1186/3123 [2:37:20<4:12:51,  7.83s/it]
 38%|███▊      | 1187/3123 [2:37:28<4:12:07,  7.81s/it]
 38%|███▊      | 1188/3123 [2:37:36<4:14:23,  7.89s/it]
 38%|███▊      | 1189/3123 [2:37:44<4:13:02,  7.85s/it]
 38%|███▊      | 1190/3123 [2:37:52<4:12:02,  7.82s/it]
                                                       
{'loss': '0.0006263', 'grad_norm': '0.127', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '94.44', 'tokens/total': 19507200, 'tokens/trainable': 6112318, 'epoch': '1.143'}
+
 38%|███▊      | 1190/3123 [2:37:52<4:12:02,  7.82s/it]
 38%|███▊      | 1191/3123 [2:38:00<4:14:17,  7.90s/it]
 38%|███▊      | 1192/3123 [2:38:08<4:12:54,  7.86s/it]
 38%|███▊      | 1193/3123 [2:38:15<4:11:54,  7.83s/it]
 38%|███▊      | 1194/3123 [2:38:23<4:11:10,  7.81s/it]
 38%|███▊      | 1195/3123 [2:38:31<4:13:30,  7.89s/it]
 38%|███▊      | 1196/3123 [2:38:39<4:12:07,  7.85s/it]
 38%|███▊      | 1197/3123 [2:38:47<4:11:11,  7.83s/it]
 38%|███▊      | 1198/3123 [2:38:55<4:13:20,  7.90s/it]
 38%|███▊      | 1199/3123 [2:39:03<4:11:59,  7.86s/it]
 38%|███▊      | 1200/3123 [2:39:10<4:10:57,  7.83s/it]
                                                       
{'loss': '0.0008735', 'grad_norm': '0.05469', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '84.73', 'tokens/total': 19671040, 'tokens/trainable': 6163992, 'epoch': '1.153'}
+
 38%|███▊      | 1200/3123 [2:39:11<4:10:57,  7.83s/it]
 38%|███▊      | 1201/3123 [2:39:18<4:10:11,  7.81s/it]
 38%|███▊      | 1202/3123 [2:39:26<4:12:32,  7.89s/it]
 39%|███▊      | 1203/3123 [2:39:34<4:11:13,  7.85s/it]
 39%|███▊      | 1204/3123 [2:39:42<4:10:09,  7.82s/it]
 39%|███▊      | 1205/3123 [2:39:50<4:11:59,  7.88s/it]
 39%|███▊      | 1206/3123 [2:39:57<4:10:39,  7.85s/it]
 39%|███▊      | 1207/3123 [2:40:05<4:09:41,  7.82s/it]
 39%|███▊      | 1208/3123 [2:40:13<4:11:49,  7.89s/it]
 39%|███▊      | 1209/3123 [2:40:21<4:10:25,  7.85s/it]
 39%|███▊      | 1210/3123 [2:40:29<4:09:23,  7.82s/it]
                                                       
{'loss': '0.0009971', 'grad_norm': '0.2734', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '78.27', 'tokens/total': 19834880, 'tokens/trainable': 6214313, 'epoch': '1.162'}
+
 39%|███▊      | 1210/3123 [2:40:29<4:09:23,  7.82s/it]
 39%|███▉      | 1211/3123 [2:40:37<4:08:40,  7.80s/it]
 39%|███▉      | 1212/3123 [2:40:45<4:10:52,  7.88s/it]
 39%|███▉      | 1213/3123 [2:40:52<4:09:30,  7.84s/it]
 39%|███▉      | 1214/3123 [2:41:00<4:08:40,  7.82s/it]
 39%|███▉      | 1215/3123 [2:41:08<4:10:49,  7.89s/it]
 39%|███▉      | 1216/3123 [2:41:16<4:09:30,  7.85s/it]
 39%|███▉      | 1217/3123 [2:41:24<4:08:28,  7.82s/it]
 39%|███▉      | 1218/3123 [2:41:31<4:07:47,  7.80s/it]
 39%|███▉      | 1219/3123 [2:41:39<4:09:57,  7.88s/it]
 39%|███▉      | 1220/3123 [2:41:47<4:08:43,  7.84s/it]
                                                       
{'loss': '0.0009465', 'grad_norm': '0.04883', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '90.83', 'tokens/total': 19998720, 'tokens/trainable': 6265730, 'epoch': '1.172'}
+
 39%|███▉      | 1220/3123 [2:41:48<4:08:43,  7.84s/it]
 39%|███▉      | 1221/3123 [2:41:55<4:07:48,  7.82s/it]
 39%|███▉      | 1222/3123 [2:42:03<4:09:54,  7.89s/it]
 39%|███▉      | 1223/3123 [2:42:11<4:08:33,  7.85s/it]
 39%|███▉      | 1224/3123 [2:42:19<4:07:34,  7.82s/it]
 39%|███▉      | 1225/3123 [2:42:26<4:06:54,  7.81s/it]
 39%|███▉      | 1226/3123 [2:42:34<4:08:59,  7.88s/it]
 39%|███▉      | 1227/3123 [2:42:42<4:07:48,  7.84s/it]
 39%|███▉      | 1228/3123 [2:42:50<4:06:53,  7.82s/it]
 39%|███▉      | 1229/3123 [2:42:58<4:09:01,  7.89s/it]
 39%|███▉      | 1230/3123 [2:43:06<4:07:40,  7.85s/it]
                                                       
{'loss': '0.00154', 'grad_norm': '0.05151', 'learning_rate': '0.0001', 'ppl': '1.002', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '95.27', 'tokens/total': 20162560, 'tokens/trainable': 6317338, 'epoch': '1.181'}
+
 39%|███▉      | 1230/3123 [2:43:06<4:07:40,  7.85s/it]
 39%|███▉      | 1231/3123 [2:43:13<4:06:39,  7.82s/it]
 39%|███▉      | 1232/3123 [2:43:21<4:05:54,  7.80s/it]
 39%|███▉      | 1233/3123 [2:43:29<4:08:06,  7.88s/it]
 40%|███▉      | 1234/3123 [2:43:37<4:06:48,  7.84s/it]
 40%|███▉      | 1235/3123 [2:43:45<4:06:00,  7.82s/it]
 40%|███▉      | 1236/3123 [2:43:53<4:08:09,  7.89s/it]
 40%|███▉      | 1237/3123 [2:44:01<4:06:48,  7.85s/it]
 40%|███▉      | 1238/3123 [2:44:08<4:05:43,  7.82s/it]
 40%|███▉      | 1239/3123 [2:44:16<4:04:59,  7.80s/it]
 40%|███▉      | 1240/3123 [2:44:24<4:07:07,  7.87s/it]
                                                       
{'loss': '0.001302', 'grad_norm': '0.08057', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '67.09', 'tokens/total': 20326400, 'tokens/trainable': 6368478, 'epoch': '1.191'}
+
 40%|███▉      | 1240/3123 [2:44:24<4:07:07,  7.87s/it]
 40%|███▉      | 1241/3123 [2:44:32<4:05:55,  7.84s/it]
 40%|███▉      | 1242/3123 [2:44:40<4:04:59,  7.81s/it]
 40%|███▉      | 1243/3123 [2:44:48<4:07:13,  7.89s/it]
 40%|███▉      | 1244/3123 [2:44:55<4:05:46,  7.85s/it]
 40%|███▉      | 1245/3123 [2:45:03<4:04:47,  7.82s/it]
 40%|███▉      | 1246/3123 [2:45:11<4:06:35,  7.88s/it]
 40%|███▉      | 1247/3123 [2:45:19<4:05:18,  7.85s/it]
 40%|███▉      | 1248/3123 [2:45:27<4:04:21,  7.82s/it]
 40%|███▉      | 1249/3123 [2:45:35<4:03:41,  7.80s/it]
 40%|████      | 1250/3123 [2:45:43<4:05:48,  7.87s/it]
                                                       
{'loss': '0.0007991', 'grad_norm': '0.01709', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '84.8', 'tokens/total': 20490240, 'tokens/trainable': 6420144, 'epoch': '1.201'}
+
 40%|████      | 1250/3123 [2:45:43<4:05:48,  7.87s/it]
 40%|████      | 1251/3123 [2:45:50<4:04:33,  7.84s/it]
 40%|████      | 1252/3123 [2:45:58<4:03:41,  7.81s/it]
 40%|████      | 1253/3123 [2:46:06<4:05:33,  7.88s/it]
 40%|████      | 1254/3123 [2:46:14<4:04:20,  7.84s/it]
 40%|████      | 1255/3123 [2:46:22<4:03:21,  7.82s/it]
 40%|████      | 1256/3123 [2:46:29<4:02:36,  7.80s/it]
 40%|████      | 1257/3123 [2:46:37<4:04:42,  7.87s/it]
 40%|████      | 1258/3123 [2:46:45<4:03:32,  7.83s/it]
 40%|████      | 1259/3123 [2:46:53<4:02:41,  7.81s/it]
 40%|████      | 1260/3123 [2:47:01<4:04:35,  7.88s/it]
                                                       
{'loss': '0.001166', 'grad_norm': '0.05908', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '94.27', 'tokens/total': 20654080, 'tokens/trainable': 6471183, 'epoch': '1.21'}
+
 40%|████      | 1260/3123 [2:47:01<4:04:35,  7.88s/it]
 40%|████      | 1261/3123 [2:47:09<4:03:18,  7.84s/it]
 40%|████      | 1262/3123 [2:47:16<4:02:23,  7.81s/it]
 40%|████      | 1263/3123 [2:47:24<4:01:42,  7.80s/it]
 40%|████      | 1264/3123 [2:47:32<4:04:03,  7.88s/it]
 41%|████      | 1265/3123 [2:47:40<4:02:45,  7.84s/it]
 41%|████      | 1266/3123 [2:47:48<4:01:53,  7.82s/it]
 41%|████      | 1267/3123 [2:47:56<4:03:42,  7.88s/it]
 41%|████      | 1268/3123 [2:48:04<4:02:30,  7.84s/it]
 41%|████      | 1269/3123 [2:48:11<4:01:31,  7.82s/it]
 41%|████      | 1270/3123 [2:48:19<4:00:53,  7.80s/it]
                                                       
{'loss': '0.0007856', 'grad_norm': '0.09131', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '90.41', 'tokens/total': 20817920, 'tokens/trainable': 6522523, 'epoch': '1.22'}
+
 41%|████      | 1270/3123 [2:48:19<4:00:53,  7.80s/it]
 41%|████      | 1271/3123 [2:48:27<4:03:04,  7.87s/it]
 41%|████      | 1272/3123 [2:48:35<4:01:49,  7.84s/it]
 41%|████      | 1273/3123 [2:48:43<4:00:53,  7.81s/it]
 41%|████      | 1274/3123 [2:48:51<4:02:52,  7.88s/it]
 41%|████      | 1275/3123 [2:48:58<4:01:35,  7.84s/it]
 41%|████      | 1276/3123 [2:49:06<4:00:38,  7.82s/it]
 41%|████      | 1277/3123 [2:49:14<4:02:37,  7.89s/it]
 41%|████      | 1278/3123 [2:49:22<4:01:18,  7.85s/it]
 41%|████      | 1279/3123 [2:49:30<4:00:18,  7.82s/it]
 41%|████      | 1280/3123 [2:49:38<3:59:35,  7.80s/it]
                                                       
{'loss': '0.0009364', 'grad_norm': '0.03711', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '80.42', 'tokens/total': 20981760, 'tokens/trainable': 6573574, 'epoch': '1.229'}
+
 41%|████      | 1280/3123 [2:49:38<3:59:35,  7.80s/it]
 41%|████      | 1281/3123 [2:49:46<4:01:37,  7.87s/it]
 41%|████      | 1282/3123 [2:49:53<4:00:25,  7.84s/it]
 41%|████      | 1283/3123 [2:50:01<3:59:32,  7.81s/it]
 41%|████      | 1284/3123 [2:50:09<4:01:40,  7.89s/it]
 41%|████      | 1285/3123 [2:50:17<4:00:21,  7.85s/it]
 41%|████      | 1286/3123 [2:50:25<3:59:22,  7.82s/it]
 41%|████      | 1287/3123 [2:50:32<3:58:40,  7.80s/it]
 41%|████      | 1288/3123 [2:50:40<4:00:46,  7.87s/it]
 41%|████▏     | 1289/3123 [2:50:48<3:59:30,  7.84s/it]
 41%|████▏     | 1290/3123 [2:50:56<3:58:38,  7.81s/it]
                                                       
{'loss': '0.0008233', 'grad_norm': '0.04053', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '81.89', 'tokens/total': 21145600, 'tokens/trainable': 6624754, 'epoch': '1.239'}
+
 41%|████▏     | 1290/3123 [2:50:56<3:58:38,  7.81s/it]
 41%|████▏     | 1291/3123 [2:51:04<4:00:41,  7.88s/it]
 41%|████▏     | 1292/3123 [2:51:12<3:59:19,  7.84s/it]
 41%|████▏     | 1293/3123 [2:51:20<3:58:29,  7.82s/it]
 41%|████▏     | 1294/3123 [2:51:27<3:57:47,  7.80s/it]
 41%|████▏     | 1295/3123 [2:51:35<4:00:01,  7.88s/it]
 41%|████▏     | 1296/3123 [2:51:43<3:58:43,  7.84s/it]
 42%|████▏     | 1297/3123 [2:51:51<3:57:50,  7.82s/it]
 42%|████▏     | 1298/3123 [2:51:59<3:59:38,  7.88s/it]
 42%|████▏     | 1299/3123 [2:52:07<3:58:27,  7.84s/it]
 42%|████▏     | 1300/3123 [2:52:14<3:57:31,  7.82s/it]
                                                       
{'loss': '0.0007726', 'grad_norm': '0.04517', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '85.65', 'tokens/total': 21309440, 'tokens/trainable': 6676467, 'epoch': '1.249'}
+
 42%|████▏     | 1300/3123 [2:52:15<3:57:31,  7.82s/it]
 42%|████▏     | 1301/3123 [2:52:22<3:56:50,  7.80s/it]
 42%|████▏     | 1302/3123 [2:52:30<3:58:43,  7.87s/it]
 42%|████▏     | 1303/3123 [2:52:38<3:57:34,  7.83s/it]
 42%|████▏     | 1304/3123 [2:52:46<3:56:40,  7.81s/it]
 42%|████▏     | 1305/3123 [2:52:54<3:58:46,  7.88s/it]
 42%|████▏     | 1306/3123 [2:53:01<3:57:31,  7.84s/it]
 42%|████▏     | 1307/3123 [2:53:09<3:56:37,  7.82s/it]
 42%|████▏     | 1308/3123 [2:53:17<3:58:27,  7.88s/it]
 42%|████▏     | 1309/3123 [2:53:25<3:57:13,  7.85s/it]
 42%|████▏     | 1310/3123 [2:53:33<3:56:19,  7.82s/it]
                                                       
{'loss': '0.0007265', 'grad_norm': '0.02893', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '68.01', 'tokens/total': 21473280, 'tokens/trainable': 6727717, 'epoch': '1.258'}
+
 42%|████▏     | 1310/3123 [2:53:33<3:56:19,  7.82s/it]
 42%|████▏     | 1311/3123 [2:53:41<3:55:39,  7.80s/it]
 42%|████▏     | 1312/3123 [2:53:49<3:57:52,  7.88s/it]
 42%|████▏     | 1313/3123 [2:53:56<3:56:37,  7.84s/it]
 42%|████▏     | 1314/3123 [2:54:04<3:55:40,  7.82s/it]
 42%|████▏     | 1315/3123 [2:54:12<3:57:38,  7.89s/it]
 42%|████▏     | 1316/3123 [2:54:20<3:56:22,  7.85s/it]
 42%|████▏     | 1317/3123 [2:54:28<3:55:28,  7.82s/it]
 42%|████▏     | 1318/3123 [2:54:35<3:54:50,  7.81s/it]
 42%|████▏     | 1319/3123 [2:54:44<3:56:58,  7.88s/it]
 42%|████▏     | 1320/3123 [2:54:51<3:55:46,  7.85s/it]
                                                       
{'loss': '0.001054', 'grad_norm': '0.08105', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '81.8', 'tokens/total': 21637120, 'tokens/trainable': 6778400, 'epoch': '1.268'}
+
 42%|████▏     | 1320/3123 [2:54:52<3:55:46,  7.85s/it]
 42%|████▏     | 1321/3123 [2:54:59<3:54:55,  7.82s/it]
 42%|████▏     | 1322/3123 [2:55:07<3:56:44,  7.89s/it]
 42%|████▏     | 1323/3123 [2:55:15<3:55:31,  7.85s/it]
 42%|████▏     | 1324/3123 [2:55:23<3:54:37,  7.82s/it]
 42%|████▏     | 1325/3123 [2:55:30<3:53:55,  7.81s/it]
 42%|████▏     | 1326/3123 [2:55:38<3:56:08,  7.88s/it]
 42%|████▏     | 1327/3123 [2:55:46<3:54:55,  7.85s/it]
 43%|████▎     | 1328/3123 [2:55:54<3:54:05,  7.82s/it]
 43%|████▎     | 1329/3123 [2:56:02<3:55:55,  7.89s/it]
 43%|████▎     | 1330/3123 [2:56:10<3:54:39,  7.85s/it]
                                                       
{'loss': '0.0007948', 'grad_norm': '0.1226', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '81.15', 'tokens/total': 21800960, 'tokens/trainable': 6829680, 'epoch': '1.277'}
+
 43%|████▎     | 1330/3123 [2:56:10<3:54:39,  7.85s/it]
 43%|████▎     | 1331/3123 [2:56:18<3:53:35,  7.82s/it]
 43%|████▎     | 1332/3123 [2:56:25<3:52:55,  7.80s/it]
 43%|████▎     | 1333/3123 [2:56:33<3:55:05,  7.88s/it]
 43%|████▎     | 1334/3123 [2:56:41<3:53:52,  7.84s/it]
 43%|████▎     | 1335/3123 [2:56:49<3:52:59,  7.82s/it]
 43%|████▎     | 1336/3123 [2:56:57<3:54:56,  7.89s/it]
 43%|████▎     | 1337/3123 [2:57:05<3:53:40,  7.85s/it]
 43%|████▎     | 1338/3123 [2:57:12<3:52:47,  7.83s/it]
 43%|████▎     | 1339/3123 [2:57:20<3:52:07,  7.81s/it]
 43%|████▎     | 1340/3123 [2:57:28<3:54:17,  7.88s/it]
                                                       
{'loss': '0.001016', 'grad_norm': '0.05225', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '85.85', 'tokens/total': 21964800, 'tokens/trainable': 6880865, 'epoch': '1.287'}
+
 43%|████▎     | 1340/3123 [2:57:29<3:54:17,  7.88s/it]
 43%|████▎     | 1341/3123 [2:57:36<3:53:07,  7.85s/it]
 43%|████▎     | 1342/3123 [2:57:44<3:52:12,  7.82s/it]
 43%|████▎     | 1343/3123 [2:57:52<3:54:02,  7.89s/it]
 43%|████▎     | 1344/3123 [2:58:00<3:52:42,  7.85s/it]
 43%|████▎     | 1345/3123 [2:58:07<3:51:51,  7.82s/it]
 43%|████▎     | 1346/3123 [2:58:15<3:53:49,  7.90s/it]
 43%|████▎     | 1347/3123 [2:58:23<3:52:32,  7.86s/it]
 43%|████▎     | 1348/3123 [2:58:31<3:51:34,  7.83s/it]
 43%|████▎     | 1349/3123 [2:58:39<3:50:51,  7.81s/it]
 43%|████▎     | 1350/3123 [2:58:47<3:52:40,  7.87s/it]
                                                       
{'loss': '0.0007738', 'grad_norm': '0.07471', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '74.48', 'tokens/total': 22128640, 'tokens/trainable': 6932720, 'epoch': '1.297'}
+
 43%|████▎     | 1350/3123 [2:58:47<3:52:40,  7.87s/it]
 43%|████▎     | 1351/3123 [2:58:55<3:51:40,  7.84s/it]
 43%|████▎     | 1352/3123 [2:59:02<3:50:47,  7.82s/it]
 43%|████▎     | 1353/3123 [2:59:10<3:52:51,  7.89s/it]
 43%|████▎     | 1354/3123 [2:59:18<3:51:34,  7.85s/it]
 43%|████▎     | 1355/3123 [2:59:26<3:50:38,  7.83s/it]
 43%|████▎     | 1356/3123 [2:59:34<3:49:55,  7.81s/it]
 43%|████▎     | 1357/3123 [2:59:42<3:51:56,  7.88s/it]
 43%|████▎     | 1358/3123 [2:59:49<3:50:45,  7.84s/it]
 44%|████▎     | 1359/3123 [2:59:57<3:49:54,  7.82s/it]
 44%|████▎     | 1360/3123 [3:00:05<3:51:49,  7.89s/it]
                                                       
{'loss': '0.0004303', 'grad_norm': '0.01361', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '76.58', 'tokens/total': 22292480, 'tokens/trainable': 6983558, 'epoch': '1.306'}
+
 44%|████▎     | 1360/3123 [3:00:06<3:51:49,  7.89s/it]
 44%|████▎     | 1361/3123 [3:00:13<3:50:37,  7.85s/it]
 44%|████▎     | 1362/3123 [3:00:21<3:49:40,  7.83s/it]
 44%|████▎     | 1363/3123 [3:00:29<3:49:00,  7.81s/it]
 44%|████▎     | 1364/3123 [3:00:37<3:51:05,  7.88s/it]
 44%|████▎     | 1365/3123 [3:00:44<3:49:56,  7.85s/it]
 44%|████▎     | 1366/3123 [3:00:52<3:49:05,  7.82s/it]
 44%|████▍     | 1367/3123 [3:01:00<3:51:00,  7.89s/it]
 44%|████▍     | 1368/3123 [3:01:08<3:49:44,  7.85s/it]
 44%|████▍     | 1369/3123 [3:01:16<3:48:47,  7.83s/it]
 44%|████▍     | 1370/3123 [3:01:24<3:50:38,  7.89s/it]
                                                       
{'loss': '0.0005287', 'grad_norm': '0.06689', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '80.43', 'tokens/total': 22456320, 'tokens/trainable': 7035633, 'epoch': '1.316'}
+
 44%|████▍     | 1370/3123 [3:01:24<3:50:38,  7.89s/it]
 44%|████▍     | 1371/3123 [3:01:32<3:49:23,  7.86s/it]
 44%|████▍     | 1372/3123 [3:01:39<3:48:25,  7.83s/it]
 44%|████▍     | 1373/3123 [3:01:47<3:47:43,  7.81s/it]
 44%|████▍     | 1374/3123 [3:01:55<3:49:56,  7.89s/it]
 44%|████▍     | 1375/3123 [3:02:03<3:48:43,  7.85s/it]
 44%|████▍     | 1376/3123 [3:02:11<3:47:50,  7.83s/it]
 44%|████▍     | 1377/3123 [3:02:19<3:49:41,  7.89s/it]
 44%|████▍     | 1378/3123 [3:02:27<3:48:25,  7.85s/it]
 44%|████▍     | 1379/3123 [3:02:34<3:47:30,  7.83s/it]
 44%|████▍     | 1380/3123 [3:02:42<3:46:54,  7.81s/it]
                                                       
{'loss': '0.001019', 'grad_norm': '0.03174', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '89.46', 'tokens/total': 22620160, 'tokens/trainable': 7087613, 'epoch': '1.325'}
+
 44%|████▍     | 1380/3123 [3:02:42<3:46:54,  7.81s/it]
 44%|████▍     | 1381/3123 [3:02:50<3:48:59,  7.89s/it]
 44%|████▍     | 1382/3123 [3:02:58<3:47:46,  7.85s/it]
 44%|████▍     | 1383/3123 [3:03:06<3:46:49,  7.82s/it]
 44%|████▍     | 1384/3123 [3:03:14<3:48:45,  7.89s/it]
 44%|████▍     | 1385/3123 [3:03:21<3:47:28,  7.85s/it]
 44%|████▍     | 1386/3123 [3:03:29<3:46:37,  7.83s/it]
 44%|████▍     | 1387/3123 [3:03:37<3:45:55,  7.81s/it]
 44%|████▍     | 1388/3123 [3:03:45<3:48:06,  7.89s/it]
 44%|████▍     | 1389/3123 [3:03:53<3:46:49,  7.85s/it]
 45%|████▍     | 1390/3123 [3:04:01<3:45:59,  7.82s/it]
                                                       
{'loss': '0.0008692', 'grad_norm': '0.1797', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '90.63', 'tokens/total': 22784000, 'tokens/trainable': 7138863, 'epoch': '1.335'}
+
 45%|████▍     | 1390/3123 [3:04:01<3:45:59,  7.82s/it]
 45%|████▍     | 1391/3123 [3:04:09<3:47:51,  7.89s/it]
 45%|████▍     | 1392/3123 [3:04:16<3:46:35,  7.85s/it]
 45%|████▍     | 1393/3123 [3:04:24<3:45:35,  7.82s/it]
 45%|████▍     | 1394/3123 [3:04:32<3:44:54,  7.81s/it]
 45%|████▍     | 1395/3123 [3:04:40<3:46:53,  7.88s/it]
 45%|████▍     | 1396/3123 [3:04:48<3:45:46,  7.84s/it]
 45%|████▍     | 1397/3123 [3:04:55<3:44:55,  7.82s/it]
 45%|████▍     | 1398/3123 [3:05:04<3:46:46,  7.89s/it]
 45%|████▍     | 1399/3123 [3:05:11<3:45:34,  7.85s/it]
 45%|████▍     | 1400/3123 [3:05:19<3:44:39,  7.82s/it]
                                                       
{'loss': '0.0008631', 'grad_norm': '0.1416', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '85.32', 'tokens/total': 22947840, 'tokens/trainable': 7190354, 'epoch': '1.345'}
+
 45%|████▍     | 1400/3123 [3:05:19<3:44:39,  7.82s/it]
 45%|████▍     | 1401/3123 [3:05:27<3:46:38,  7.90s/it]
 45%|████▍     | 1402/3123 [3:05:35<3:45:18,  7.86s/it]
 45%|████▍     | 1403/3123 [3:05:43<3:44:22,  7.83s/it]
 45%|████▍     | 1404/3123 [3:05:50<3:43:38,  7.81s/it]
 45%|████▍     | 1405/3123 [3:05:58<3:45:37,  7.88s/it]
 45%|████▌     | 1406/3123 [3:06:06<3:44:24,  7.84s/it]
 45%|████▌     | 1407/3123 [3:06:14<3:43:40,  7.82s/it]
 45%|████▌     | 1408/3123 [3:06:22<3:45:35,  7.89s/it]
 45%|████▌     | 1409/3123 [3:06:30<3:44:21,  7.85s/it]
 45%|████▌     | 1410/3123 [3:06:38<3:43:23,  7.82s/it]
                                                       
{'loss': '0.001051', 'grad_norm': '0.02856', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '89.66', 'tokens/total': 23111680, 'tokens/trainable': 7241829, 'epoch': '1.354'}
+
 45%|████▌     | 1410/3123 [3:06:38<3:43:23,  7.82s/it]
 45%|████▌     | 1411/3123 [3:06:45<3:42:45,  7.81s/it]
 45%|████▌     | 1412/3123 [3:06:53<3:44:35,  7.88s/it]
 45%|████▌     | 1413/3123 [3:07:01<3:43:31,  7.84s/it]
 45%|████▌     | 1414/3123 [3:07:09<3:42:45,  7.82s/it]
 45%|████▌     | 1415/3123 [3:07:17<3:44:41,  7.89s/it]
 45%|████▌     | 1416/3123 [3:07:25<3:43:24,  7.85s/it]
 45%|████▌     | 1417/3123 [3:07:32<3:42:30,  7.83s/it]
 45%|████▌     | 1418/3123 [3:07:40<3:41:49,  7.81s/it]
 45%|████▌     | 1419/3123 [3:07:48<3:43:43,  7.88s/it]
 45%|████▌     | 1420/3123 [3:07:56<3:42:30,  7.84s/it]
                                                       
{'loss': '0.0008184', 'grad_norm': '0.06836', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '79.39', 'tokens/total': 23275520, 'tokens/trainable': 7292651, 'epoch': '1.364'}
+
 45%|████▌     | 1420/3123 [3:07:56<3:42:30,  7.84s/it]
 46%|████▌     | 1421/3123 [3:08:04<3:41:45,  7.82s/it]
 46%|████▌     | 1422/3123 [3:08:12<3:43:36,  7.89s/it]
 46%|████▌     | 1423/3123 [3:08:20<3:42:27,  7.85s/it]
 46%|████▌     | 1424/3123 [3:08:27<3:41:32,  7.82s/it]
 46%|████▌     | 1425/3123 [3:08:35<3:40:55,  7.81s/it]
 46%|████▌     | 1426/3123 [3:08:43<3:42:53,  7.88s/it]
 46%|████▌     | 1427/3123 [3:08:51<3:41:49,  7.85s/it]
 46%|████▌     | 1428/3123 [3:08:59<3:41:00,  7.82s/it]
 46%|████▌     | 1429/3123 [3:09:07<3:42:53,  7.89s/it]
 46%|████▌     | 1430/3123 [3:09:15<3:41:44,  7.86s/it]
                                                       
{'loss': '0.0009149', 'grad_norm': '0.08838', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '87.58', 'tokens/total': 23439360, 'tokens/trainable': 7344149, 'epoch': '1.373'}
+
 46%|████▌     | 1430/3123 [3:09:15<3:41:44,  7.86s/it]
 46%|████▌     | 1431/3123 [3:09:22<3:40:49,  7.83s/it]
 46%|████▌     | 1432/3123 [3:09:30<3:40:08,  7.81s/it]
 46%|████▌     | 1433/3123 [3:09:38<3:42:01,  7.88s/it]
 46%|████▌     | 1434/3123 [3:09:46<3:40:55,  7.85s/it]
 46%|████▌     | 1435/3123 [3:09:54<3:40:02,  7.82s/it]
 46%|████▌     | 1436/3123 [3:10:02<3:41:59,  7.90s/it]
 46%|████▌     | 1437/3123 [3:10:10<3:40:46,  7.86s/it]
 46%|████▌     | 1438/3123 [3:10:17<3:39:55,  7.83s/it]
 46%|████▌     | 1439/3123 [3:10:25<3:41:37,  7.90s/it]
 46%|████▌     | 1440/3123 [3:10:33<3:40:23,  7.86s/it]
                                                       
{'loss': '0.0007012', 'grad_norm': '0.03223', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '77.76', 'tokens/total': 23603200, 'tokens/trainable': 7394829, 'epoch': '1.383'}
+
 46%|████▌     | 1440/3123 [3:10:33<3:40:23,  7.86s/it]
 46%|████▌     | 1441/3123 [3:10:41<3:39:28,  7.83s/it]
 46%|████▌     | 1442/3123 [3:10:49<3:38:50,  7.81s/it]
 46%|████▌     | 1443/3123 [3:10:57<3:40:41,  7.88s/it]
 46%|████▌     | 1444/3123 [3:11:04<3:39:36,  7.85s/it]
 46%|████▋     | 1445/3123 [3:11:12<3:38:44,  7.82s/it]
 46%|████▋     | 1446/3123 [3:11:20<3:40:27,  7.89s/it]
 46%|████▋     | 1447/3123 [3:11:28<3:39:21,  7.85s/it]
 46%|████▋     | 1448/3123 [3:11:36<3:38:29,  7.83s/it]
 46%|████▋     | 1449/3123 [3:11:44<3:37:53,  7.81s/it]
 46%|████▋     | 1450/3123 [3:11:52<3:39:53,  7.89s/it]
                                                       
{'loss': '0.0005787', 'grad_norm': '0.1816', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '89.97', 'tokens/total': 23767040, 'tokens/trainable': 7446945, 'epoch': '1.393'}
+
 46%|████▋     | 1450/3123 [3:11:52<3:39:53,  7.89s/it]
 46%|████▋     | 1451/3123 [3:11:59<3:38:44,  7.85s/it]
 46%|████▋     | 1452/3123 [3:12:07<3:37:54,  7.82s/it]
 47%|████▋     | 1453/3123 [3:12:15<3:39:40,  7.89s/it]
 47%|████▋     | 1454/3123 [3:12:23<3:38:25,  7.85s/it]
 47%|████▋     | 1455/3123 [3:12:31<3:37:31,  7.82s/it]
 47%|████▋     | 1456/3123 [3:12:38<3:36:55,  7.81s/it]
 47%|████▋     | 1457/3123 [3:12:47<3:38:58,  7.89s/it]
 47%|████▋     | 1458/3123 [3:12:54<3:37:49,  7.85s/it]
 47%|████▋     | 1459/3123 [3:13:02<3:37:00,  7.82s/it]
 47%|████▋     | 1460/3123 [3:13:10<3:38:42,  7.89s/it]
                                                       
{'loss': '0.000568', 'grad_norm': '0.04785', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '75.61', 'tokens/total': 23930880, 'tokens/trainable': 7498437, 'epoch': '1.402'}
+
 47%|████▋     | 1460/3123 [3:13:10<3:38:42,  7.89s/it]
 47%|████▋     | 1461/3123 [3:13:18<3:37:32,  7.85s/it]
 47%|████▋     | 1462/3123 [3:13:26<3:36:40,  7.83s/it]
 47%|████▋     | 1463/3123 [3:13:33<3:36:02,  7.81s/it]
 47%|████▋     | 1464/3123 [3:13:41<3:38:00,  7.88s/it]
 47%|████▋     | 1465/3123 [3:13:49<3:36:54,  7.85s/it]
 47%|████▋     | 1466/3123 [3:13:57<3:36:00,  7.82s/it]
 47%|████▋     | 1467/3123 [3:14:05<3:37:46,  7.89s/it]
 47%|████▋     | 1468/3123 [3:14:13<3:36:35,  7.85s/it]
 47%|████▋     | 1469/3123 [3:14:21<3:35:48,  7.83s/it]
 47%|████▋     | 1470/3123 [3:14:29<3:37:33,  7.90s/it]
                                                       
{'loss': '0.001166', 'grad_norm': '0.06494', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '67.36', 'tokens/total': 24094720, 'tokens/trainable': 7549736, 'epoch': '1.412'}
+
 47%|████▋     | 1470/3123 [3:14:29<3:37:33,  7.90s/it]
 47%|████▋     | 1471/3123 [3:14:36<3:36:20,  7.86s/it]
 47%|████▋     | 1472/3123 [3:14:44<3:35:22,  7.83s/it]
 47%|████▋     | 1473/3123 [3:14:52<3:34:45,  7.81s/it]
 47%|████▋     | 1474/3123 [3:15:00<3:36:40,  7.88s/it]
 47%|████▋     | 1475/3123 [3:15:08<3:35:32,  7.85s/it]
 47%|████▋     | 1476/3123 [3:15:16<3:34:45,  7.82s/it]
 47%|████▋     | 1477/3123 [3:15:24<3:36:40,  7.90s/it]
 47%|████▋     | 1478/3123 [3:15:31<3:35:25,  7.86s/it]
 47%|████▋     | 1479/3123 [3:15:39<3:34:35,  7.83s/it]
 47%|████▋     | 1480/3123 [3:15:47<3:33:54,  7.81s/it]
                                                       
{'loss': '0.0007651', 'grad_norm': '0.03979', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '96.94', 'tokens/total': 24258560, 'tokens/trainable': 7601091, 'epoch': '1.421'}
+
 47%|████▋     | 1480/3123 [3:15:47<3:33:54,  7.81s/it]
 47%|████▋     | 1481/3123 [3:15:55<3:35:49,  7.89s/it]
 47%|████▋     | 1482/3123 [3:16:03<3:34:42,  7.85s/it]
 47%|████▋     | 1483/3123 [3:16:11<3:33:51,  7.82s/it]
 48%|████▊     | 1484/3123 [3:16:19<3:35:43,  7.90s/it]
 48%|████▊     | 1485/3123 [3:16:26<3:34:31,  7.86s/it]
 48%|████▊     | 1486/3123 [3:16:34<3:33:39,  7.83s/it]
 48%|████▊     | 1487/3123 [3:16:42<3:33:02,  7.81s/it]
 48%|████▊     | 1488/3123 [3:16:50<3:34:51,  7.88s/it]
 48%|████▊     | 1489/3123 [3:16:58<3:33:46,  7.85s/it]
 48%|████▊     | 1490/3123 [3:17:05<3:32:57,  7.82s/it]
                                                       
{'loss': '0.000851', 'grad_norm': '0.09277', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '90.11', 'tokens/total': 24422400, 'tokens/trainable': 7652714, 'epoch': '1.431'}
+
 48%|████▊     | 1490/3123 [3:17:06<3:32:57,  7.82s/it]
 48%|████▊     | 1491/3123 [3:17:14<3:34:47,  7.90s/it]
 48%|████▊     | 1492/3123 [3:17:21<3:33:31,  7.86s/it]
 48%|████▊     | 1493/3123 [3:17:29<3:32:37,  7.83s/it]
 48%|████▊     | 1494/3123 [3:17:37<3:31:56,  7.81s/it]
 48%|████▊     | 1495/3123 [3:17:45<3:33:40,  7.87s/it]
 48%|████▊     | 1496/3123 [3:17:53<3:32:38,  7.84s/it]
 48%|████▊     | 1497/3123 [3:18:00<3:31:51,  7.82s/it]
 48%|████▊     | 1498/3123 [3:18:08<3:33:49,  7.90s/it]
 48%|████▊     | 1499/3123 [3:18:16<3:32:36,  7.85s/it]
 48%|████▊     | 1500/3123 [3:18:24<3:31:43,  7.83s/it]
                                                       
{'loss': '0.0009757', 'grad_norm': '0.08057', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '88.12', 'tokens/total': 24586240, 'tokens/trainable': 7703464, 'epoch': '1.441'}
+
 48%|████▊     | 1500/3123 [3:18:24<3:31:43,  7.83s/it]
 48%|████▊     | 1501/3123 [3:18:32<3:31:01,  7.81s/it]
 48%|████▊     | 1502/3123 [3:18:40<3:32:49,  7.88s/it]
 48%|████▊     | 1503/3123 [3:18:48<3:31:43,  7.84s/it]
 48%|████▊     | 1504/3123 [3:18:55<3:30:57,  7.82s/it]
 48%|████▊     | 1505/3123 [3:19:03<3:32:47,  7.89s/it]
 48%|████▊     | 1506/3123 [3:19:11<3:31:38,  7.85s/it]
 48%|████▊     | 1507/3123 [3:19:19<3:30:44,  7.82s/it]
 48%|████▊     | 1508/3123 [3:19:27<3:30:07,  7.81s/it]
 48%|████▊     | 1509/3123 [3:19:35<3:31:45,  7.87s/it]
 48%|████▊     | 1510/3123 [3:19:42<3:30:39,  7.84s/it]
                                                       
{'loss': '0.00125', 'grad_norm': '0.2637', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '82.07', 'tokens/total': 24750080, 'tokens/trainable': 7754782, 'epoch': '1.45'}
+
 48%|████▊     | 1510/3123 [3:19:43<3:30:39,  7.84s/it]
 48%|████▊     | 1511/3123 [3:19:50<3:29:59,  7.82s/it]
 48%|████▊     | 1512/3123 [3:19:58<3:31:52,  7.89s/it]
 48%|████▊     | 1513/3123 [3:20:06<3:30:42,  7.85s/it]
 48%|████▊     | 1514/3123 [3:20:14<3:29:52,  7.83s/it]
 49%|████▊     | 1515/3123 [3:20:22<3:31:29,  7.89s/it]
 49%|████▊     | 1516/3123 [3:20:30<3:30:20,  7.85s/it]
 49%|████▊     | 1517/3123 [3:20:37<3:29:33,  7.83s/it]
 49%|████▊     | 1518/3123 [3:20:45<3:28:58,  7.81s/it]
 49%|████▊     | 1519/3123 [3:20:53<3:30:48,  7.89s/it]
 49%|████▊     | 1520/3123 [3:21:01<3:29:40,  7.85s/it]
                                                       
{'loss': '0.001424', 'grad_norm': '0.09326', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '72.32', 'tokens/total': 24913920, 'tokens/trainable': 7806224, 'epoch': '1.46'}
+
 49%|████▊     | 1520/3123 [3:21:01<3:29:40,  7.85s/it]
 49%|████▊     | 1521/3123 [3:21:09<3:28:52,  7.82s/it]
 49%|████▊     | 1522/3123 [3:21:17<3:30:37,  7.89s/it]
 49%|████▉     | 1523/3123 [3:21:25<3:29:27,  7.85s/it]
 49%|████▉     | 1524/3123 [3:21:32<3:28:35,  7.83s/it]
 49%|████▉     | 1525/3123 [3:21:40<3:27:59,  7.81s/it]
 49%|████▉     | 1526/3123 [3:21:48<3:29:48,  7.88s/it]
 49%|████▉     | 1527/3123 [3:21:56<3:28:46,  7.85s/it]
 49%|████▉     | 1528/3123 [3:22:04<3:27:55,  7.82s/it]
 49%|████▉     | 1529/3123 [3:22:12<3:29:41,  7.89s/it]
 49%|████▉     | 1530/3123 [3:22:19<3:28:29,  7.85s/it]
                                                       
{'loss': '0.001188', 'grad_norm': '0.04907', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '86.82', 'tokens/total': 25077760, 'tokens/trainable': 7857372, 'epoch': '1.469'}
+
 49%|████▉     | 1530/3123 [3:22:20<3:28:29,  7.85s/it]
 49%|████▉     | 1531/3123 [3:22:27<3:27:43,  7.83s/it]
 49%|████▉     | 1532/3123 [3:22:35<3:27:08,  7.81s/it]
 49%|████▉     | 1533/3123 [3:22:43<3:29:02,  7.89s/it]
 49%|████▉     | 1534/3123 [3:22:51<3:27:58,  7.85s/it]
 49%|████▉     | 1535/3123 [3:22:59<3:27:09,  7.83s/it]
 49%|████▉     | 1536/3123 [3:23:07<3:28:43,  7.89s/it]
 49%|████▉     | 1537/3123 [3:23:14<3:27:35,  7.85s/it]
 49%|████▉     | 1538/3123 [3:23:22<3:26:47,  7.83s/it]
 49%|████▉     | 1539/3123 [3:23:30<3:28:37,  7.90s/it]
 49%|████▉     | 1540/3123 [3:23:38<3:27:24,  7.86s/it]
                                                       
{'loss': '0.0008189', 'grad_norm': '0.07227', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '92.38', 'tokens/total': 25241600, 'tokens/trainable': 7909316, 'epoch': '1.479'}
+
 49%|████▉     | 1540/3123 [3:23:38<3:27:24,  7.86s/it]
 49%|████▉     | 1541/3123 [3:23:46<3:26:33,  7.83s/it]
 49%|████▉     | 1542/3123 [3:23:54<3:25:50,  7.81s/it]
 49%|████▉     | 1543/3123 [3:24:02<3:27:35,  7.88s/it]
 49%|████▉     | 1544/3123 [3:24:09<3:26:29,  7.85s/it]
 49%|████▉     | 1545/3123 [3:24:17<3:25:45,  7.82s/it]
 50%|████▉     | 1546/3123 [3:24:25<3:27:30,  7.89s/it]
 50%|████▉     | 1547/3123 [3:24:33<3:26:20,  7.86s/it]
 50%|████▉     | 1548/3123 [3:24:41<3:25:24,  7.83s/it]
 50%|████▉     | 1549/3123 [3:24:48<3:24:43,  7.80s/it]
 50%|████▉     | 1550/3123 [3:24:57<3:26:36,  7.88s/it]
                                                       
{'loss': '0.0008214', 'grad_norm': '0.1387', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '81.78', 'tokens/total': 25405440, 'tokens/trainable': 7960179, 'epoch': '1.489'}
+
 50%|████▉     | 1550/3123 [3:24:57<3:26:36,  7.88s/it]
 50%|████▉     | 1551/3123 [3:25:04<3:25:34,  7.85s/it]
 50%|████▉     | 1552/3123 [3:25:12<3:24:46,  7.82s/it]
 50%|████▉     | 1553/3123 [3:25:20<3:26:31,  7.89s/it]
 50%|████▉     | 1554/3123 [3:25:28<3:25:22,  7.85s/it]
 50%|████▉     | 1555/3123 [3:25:36<3:24:37,  7.83s/it]
 50%|████▉     | 1556/3123 [3:25:43<3:24:01,  7.81s/it]
 50%|████▉     | 1557/3123 [3:25:52<3:25:56,  7.89s/it]
 50%|████▉     | 1558/3123 [3:25:59<3:24:46,  7.85s/it]
 50%|████▉     | 1559/3123 [3:26:07<3:23:57,  7.82s/it]
 50%|████▉     | 1560/3123 [3:26:15<3:25:33,  7.89s/it]
                                                       
{'loss': '0.001014', 'grad_norm': '0.1216', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '94.45', 'tokens/total': 25569280, 'tokens/trainable': 8011395, 'epoch': '1.498'}
+
 50%|████▉     | 1560/3123 [3:26:15<3:25:33,  7.89s/it]
 50%|████▉     | 1561/3123 [3:26:23<3:24:28,  7.85s/it]
 50%|█████     | 1562/3123 [3:26:31<3:23:39,  7.83s/it]
 50%|█████     | 1563/3123 [3:26:39<3:25:07,  7.89s/it][2026-03-08 19:23:53,093] [INFO] [axolotl.core.trainers.base.evaluate:400] [PID:1659682] Running evaluation step...
+[2026-03-08 19:23:54,016] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:1659682] generate_batches time: 0.4609494209289551
+[2026-03-08 19:23:54,482] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:1659682] generate_batches time: 0.46578168869018555
+[2026-03-08 19:23:54,937] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:1659682] generate_batches time: 0.45462942123413086
+[2026-03-08 19:23:55,411] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:1659682] generate_batches time: 0.47313761711120605
+[2026-03-08 19:23:55,411] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:1659682] gather_len_batches: [34]
+
+
  0%|          | 0/34 [00:00<?, ?it/s]
+
  6%|▌         | 2/34 [00:00<00:03,  8.54it/s]
+
  9%|▉         | 3/34 [00:00<00:05,  5.97it/s]
+
 12%|█▏        | 4/34 [00:00<00:05,  5.15it/s]
+
 15%|█▍        | 5/34 [00:00<00:06,  4.77it/s]
+
 18%|█▊        | 6/34 [00:01<00:06,  4.57it/s]
+
 21%|██        | 7/34 [00:01<00:06,  4.45it/s]
+
 24%|██▎       | 8/34 [00:01<00:05,  4.37it/s]
+
 26%|██▋       | 9/34 [00:02<00:06,  3.69it/s]
+
 29%|██▉       | 10/34 [00:02<00:06,  3.84it/s]
+
 32%|███▏      | 11/34 [00:02<00:05,  3.94it/s]
+
 35%|███��      | 12/34 [00:02<00:05,  4.01it/s]
+
 38%|███▊      | 13/34 [00:02<00:05,  4.07it/s]
+
 41%|████      | 14/34 [00:03<00:04,  4.11it/s]
+
 44%|████▍     | 15/34 [00:03<00:04,  4.14it/s]
+
 47%|████▋     | 16/34 [00:03<00:04,  4.16it/s]
+
 50%|█████     | 17/34 [00:04<00:05,  3.14it/s]
+
 53%|█████▎    | 18/34 [00:04<00:04,  3.42it/s]
+
 56%|█████▌    | 19/34 [00:04<00:04,  3.63it/s]
+
 59%|█████▉    | 20/34 [00:04<00:03,  3.78it/s]
+
 62%|██████▏   | 21/34 [00:05<00:03,  3.89it/s]
+
 65%|██████▍   | 22/34 [00:05<00:03,  3.99it/s]
+
 68%|██████▊   | 23/34 [00:05<00:02,  4.06it/s]
+
 71%|███████   | 24/34 [00:05<00:02,  4.10it/s]
+
 74%|███████▎  | 25/34 [00:06<00:02,  3.89it/s]
+
 76%|███████▋  | 26/34 [00:06<00:02,  3.99it/s]
+
 79%|███████▉  | 27/34 [00:06<00:01,  4.06it/s]
+
 82%|████████▏ | 28/34 [00:06<00:01,  4.10it/s]
+
 85%|████████▌ | 29/34 [00:07<00:01,  4.13it/s]
+
 88%|████████▊ | 30/34 [00:07<00:00,  4.15it/s]
+
 91%|█████████ | 31/34 [00:07<00:00,  4.16it/s]
+
 94%|█████████▍| 32/34 [00:07<00:00,  4.17it/s]
+
 97%|█████████▋| 33/34 [00:08<00:00,  4.00it/s]Traceback (most recent call last):
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/multiprocessing/util.py", line 300, in _run_finalizers
+    finalizer()
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/multiprocessing/util.py", line 224, in __call__
+    res = self._callback(*self._args, **self._kwargs)
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/multiprocessing/util.py", line 133, in _remove_temp_dir
+    rmtree(tempdir)
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/shutil.py", line 725, in rmtree
+    _rmtree_safe_fd(fd, path, onerror)
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/shutil.py", line 681, in _rmtree_safe_fd
+    onerror(os.unlink, fullname, sys.exc_info())
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/shutil.py", line 679, in _rmtree_safe_fd
+    os.unlink(entry.name, dir_fd=topfd)
+OSError: [Errno 16] Device or resource busy: '.nfs00000000000dae9c0001250a'
+
+
100%|██████████| 34/34 [00:08<00:00,  3.87it/s]
                                                       
+
                                               
{'eval_loss': '0.0007254', 'eval_runtime': '9.252', 'eval_samples_per_second': '21.62', 'eval_steps_per_second': '21.62', 'eval_ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'epoch': '1.501', 'tokens/train_per_sec_per_gpu': '75.59'}
+
 50%|█████     | 1563/3123 [3:26:50<3:25:07,  7.89s/it]
+
100%|██████████| 34/34 [00:08<00:00,  3.87it/s]
+
                                               
 50%|█████     | 1564/3123 [3:26:58<4:52:10, 11.24s/it]
 50%|█████     | 1565/3123 [3:27:05<4:24:47, 10.20s/it]
 50%|█████     | 1566/3123 [3:27:13<4:05:35,  9.46s/it]
 50%|█████     | 1567/3123 [3:27:21<3:54:48,  9.05s/it]
 50%|█████     | 1568/3123 [3:27:29<3:44:30,  8.66s/it]
 50%|█████     | 1569/3123 [3:27:37<3:37:20,  8.39s/it]
 50%|█████     | 1570/3123 [3:27:45<3:32:16,  8.20s/it]
                                                       
{'loss': '0.0008709', 'grad_norm': '0.1299', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.96', 'memory/max_allocated (GiB)': '33.96', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '98.39', 'tokens/total': 25733120, 'tokens/trainable': 8062743, 'epoch': '1.508'}
+
 50%|█████     | 1570/3123 [3:27:45<3:32:16,  8.20s/it]
 50%|█████     | 1571/3123 [3:27:53<3:30:55,  8.15s/it]
 50%|█████     | 1572/3123 [3:28:00<3:27:43,  8.04s/it]
 50%|█████     | 1573/3123 [3:28:08<3:25:26,  7.95s/it]
 50%|█████     | 1574/3123 [3:28:16<3:26:10,  7.99s/it]
 50%|█████     | 1575/3123 [3:28:24<3:24:17,  7.92s/it]
 50%|█████     | 1576/3123 [3:28:32<3:22:54,  7.87s/it]
 50%|█████     | 1577/3123 [3:28:40<3:24:06,  7.92s/it]
 51%|█████     | 1578/3123 [3:28:48<3:22:39,  7.87s/it]
 51%|█████     | 1579/3123 [3:28:55<3:21:42,  7.84s/it]
 51%|█████     | 1580/3123 [3:29:03<3:20:59,  7.82s/it]
                                                       
{'loss': '0.0003492', 'grad_norm': '0.004059', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '94.7', 'tokens/total': 25896960, 'tokens/trainable': 8113834, 'epoch': '1.517'}
+
 51%|█████     | 1580/3123 [3:29:03<3:20:59,  7.82s/it]
 51%|█████     | 1581/3123 [3:29:11<3:22:47,  7.89s/it]
 51%|█████     | 1582/3123 [3:29:19<3:21:37,  7.85s/it]
 51%|█████     | 1583/3123 [3:29:27<3:20:48,  7.82s/it]
 51%|█████     | 1584/3123 [3:29:35<3:22:22,  7.89s/it]
 51%|█████     | 1585/3123 [3:29:42<3:21:13,  7.85s/it]
 51%|█████     | 1586/3123 [3:29:50<3:20:24,  7.82s/it]
 51%|█████     | 1587/3123 [3:29:58<3:19:49,  7.81s/it]
 51%|█████     | 1588/3123 [3:30:06<3:21:39,  7.88s/it]
 51%|█████     | 1589/3123 [3:30:14<3:20:37,  7.85s/it]
 51%|█████     | 1590/3123 [3:30:22<3:19:49,  7.82s/it]
                                                       
{'loss': '0.0003433', 'grad_norm': '0.003052', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '77.96', 'tokens/total': 26060800, 'tokens/trainable': 8165964, 'epoch': '1.527'}
+
 51%|█████     | 1590/3123 [3:30:22<3:19:49,  7.82s/it]
 51%|█████     | 1591/3123 [3:30:30<3:21:23,  7.89s/it]
 51%|█████     | 1592/3123 [3:30:37<3:20:18,  7.85s/it]
 51%|█████     | 1593/3123 [3:30:45<3:19:32,  7.83s/it]
 51%|█████     | 1594/3123 [3:30:53<3:21:14,  7.90s/it]
 51%|█████     | 1595/3123 [3:31:01<3:20:02,  7.86s/it]
 51%|█████     | 1596/3123 [3:31:09<3:19:11,  7.83s/it]
 51%|█████     | 1597/3123 [3:31:16<3:18:32,  7.81s/it]
 51%|█████     | 1598/3123 [3:31:24<3:20:09,  7.88s/it]
 51%|█████     | 1599/3123 [3:31:32<3:19:09,  7.84s/it]
 51%|█████     | 1600/3123 [3:31:40<3:18:27,  7.82s/it]
                                                       
{'loss': '0.000548', 'grad_norm': '0.03296', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '95.18', 'tokens/total': 26224640, 'tokens/trainable': 8217121, 'epoch': '1.537'}
+
 51%|█████     | 1600/3123 [3:31:40<3:18:27,  7.82s/it]
 51%|█████▏    | 1601/3123 [3:31:48<3:20:09,  7.89s/it]
 51%|█████▏    | 1602/3123 [3:31:56<3:19:01,  7.85s/it]
 51%|█████▏    | 1603/3123 [3:32:04<3:18:12,  7.82s/it]
 51%|█████▏    | 1604/3123 [3:32:11<3:17:37,  7.81s/it]
 51%|█████▏    | 1605/3123 [3:32:19<3:19:16,  7.88s/it]
 51%|█████▏    | 1606/3123 [3:32:27<3:18:15,  7.84s/it]
 51%|█████▏    | 1607/3123 [3:32:35<3:17:28,  7.82s/it]
 51%|█████▏    | 1608/3123 [3:32:43<3:19:05,  7.88s/it]
 52%|█████▏    | 1609/3123 [3:32:51<3:18:02,  7.85s/it]
 52%|█████▏    | 1610/3123 [3:32:58<3:17:15,  7.82s/it]
                                                       
{'loss': '0.0005881', 'grad_norm': '0.123', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '91.48', 'tokens/total': 26388480, 'tokens/trainable': 8268301, 'epoch': '1.546'}
+
 52%|█████▏    | 1610/3123 [3:32:59<3:17:15,  7.82s/it]
 52%|█████▏    | 1611/3123 [3:33:06<3:16:42,  7.81s/it]
 52%|█████▏    | 1612/3123 [3:33:14<3:18:26,  7.88s/it]
 52%|█████▏    | 1613/3123 [3:33:22<3:17:26,  7.85s/it]
 52%|█████▏    | 1614/3123 [3:33:30<3:16:40,  7.82s/it]
 52%|█████▏    | 1615/3123 [3:33:38<3:18:19,  7.89s/it]
 52%|█████▏    | 1616/3123 [3:33:46<3:17:11,  7.85s/it]
 52%|█████▏    | 1617/3123 [3:33:53<3:16:23,  7.82s/it]
 52%|█████▏    | 1618/3123 [3:34:01<3:15:47,  7.81s/it]
 52%|█████▏    | 1619/3123 [3:34:09<3:17:30,  7.88s/it]
 52%|█████▏    | 1620/3123 [3:34:17<3:16:28,  7.84s/it]
                                                       
{'loss': '0.0004819', 'grad_norm': '0.08594', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '85.61', 'tokens/total': 26552320, 'tokens/trainable': 8320320, 'epoch': '1.556'}
+
 52%|█████▏    | 1620/3123 [3:34:17<3:16:28,  7.84s/it]
 52%|█████▏    | 1621/3123 [3:34:25<3:15:44,  7.82s/it]
 52%|█████▏    | 1622/3123 [3:34:33<3:17:13,  7.88s/it]
 52%|█████▏    | 1623/3123 [3:34:41<3:16:12,  7.85s/it]
 52%|█████▏    | 1624/3123 [3:34:48<3:15:24,  7.82s/it]
 52%|█████▏    | 1625/3123 [3:34:56<3:16:59,  7.89s/it]
 52%|█████▏    | 1626/3123 [3:35:04<3:15:53,  7.85s/it]
 52%|█████▏    | 1627/3123 [3:35:12<3:15:05,  7.82s/it]
 52%|█████▏    | 1628/3123 [3:35:20<3:14:32,  7.81s/it]
 52%|█████▏    | 1629/3123 [3:35:28<3:16:10,  7.88s/it]
 52%|█████▏    | 1630/3123 [3:35:35<3:15:13,  7.85s/it]
                                                       
{'loss': '0.0007268', 'grad_norm': '0.09277', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '86.85', 'tokens/total': 26716160, 'tokens/trainable': 8372031, 'epoch': '1.565'}
+
 52%|█████▏    | 1630/3123 [3:35:36<3:15:13,  7.85s/it]
 52%|█████▏    | 1631/3123 [3:35:43<3:14:30,  7.82s/it]
 52%|█████▏    | 1632/3123 [3:35:51<3:16:08,  7.89s/it]
 52%|█████▏    | 1633/3123 [3:35:59<3:14:59,  7.85s/it]
 52%|█████▏    | 1634/3123 [3:36:07<3:14:09,  7.82s/it]
 52%|█████▏    | 1635/3123 [3:36:15<3:13:31,  7.80s/it]
 52%|█████▏    | 1636/3123 [3:36:23<3:15:09,  7.87s/it]
 52%|█████▏    | 1637/3123 [3:36:30<3:14:11,  7.84s/it]
 52%|█████▏    | 1638/3123 [3:36:38<3:13:26,  7.82s/it]
 52%|█████▏    | 1639/3123 [3:36:46<3:15:07,  7.89s/it]
 53%|█████▎    | 1640/3123 [3:36:54<3:14:03,  7.85s/it]
                                                       
{'loss': '0.0006106', 'grad_norm': '0.01477', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '91.07', 'tokens/total': 26880000, 'tokens/trainable': 8422723, 'epoch': '1.575'}
+
 53%|█████▎    | 1640/3123 [3:36:54<3:14:03,  7.85s/it]
 53%|█████▎    | 1641/3123 [3:37:02<3:13:14,  7.82s/it]
 53%|█████▎    | 1642/3123 [3:37:09<3:12:39,  7.81s/it]
 53%|█████▎    | 1643/3123 [3:37:18<3:14:15,  7.88s/it]
 53%|█████▎    | 1644/3123 [3:37:25<3:13:16,  7.84s/it]
 53%|█████▎    | 1645/3123 [3:37:33<3:12:34,  7.82s/it]
 53%|█████▎    | 1646/3123 [3:37:41<3:14:06,  7.89s/it]
 53%|█████▎    | 1647/3123 [3:37:49<3:13:06,  7.85s/it]
 53%|█████▎    | 1648/3123 [3:37:57<3:12:22,  7.83s/it]
 53%|█████▎    | 1649/3123 [3:38:04<3:11:44,  7.80s/it]
 53%|█████▎    | 1650/3123 [3:38:12<3:13:25,  7.88s/it]
                                                       
{'loss': '0.0009039', 'grad_norm': '0.08203', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '95.02', 'tokens/total': 27043840, 'tokens/trainable': 8474243, 'epoch': '1.585'}
+
 53%|█████▎    | 1650/3123 [3:38:13<3:13:25,  7.88s/it]
 53%|█████▎    | 1651/3123 [3:38:20<3:12:29,  7.85s/it]
 53%|█████▎    | 1652/3123 [3:38:28<3:11:44,  7.82s/it]
 53%|█████▎    | 1653/3123 [3:38:36<3:13:26,  7.90s/it]
 53%|█████▎    | 1654/3123 [3:38:44<3:12:17,  7.85s/it]
 53%|█████▎    | 1655/3123 [3:38:52<3:11:27,  7.83s/it]
 53%|█████▎    | 1656/3123 [3:39:00<3:12:57,  7.89s/it]
 53%|█████▎    | 1657/3123 [3:39:07<3:11:52,  7.85s/it]
 53%|█████▎    | 1658/3123 [3:39:15<3:11:07,  7.83s/it]
 53%|█████▎    | 1659/3123 [3:39:23<3:10:31,  7.81s/it]
 53%|█████▎    | 1660/3123 [3:39:31<3:12:16,  7.89s/it]
                                                       
{'loss': '0.001328', 'grad_norm': '0.04761', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '77.22', 'tokens/total': 27207680, 'tokens/trainable': 8524834, 'epoch': '1.594'}
+
 53%|█████▎    | 1660/3123 [3:39:31<3:12:16,  7.89s/it]
 53%|█████▎    | 1661/3123 [3:39:39<3:11:14,  7.85s/it]
 53%|█████▎    | 1662/3123 [3:39:46<3:10:30,  7.82s/it]
 53%|█████▎    | 1663/3123 [3:39:55<3:11:59,  7.89s/it]
 53%|█████▎    | 1664/3123 [3:40:02<3:10:56,  7.85s/it]
 53%|█████▎    | 1665/3123 [3:40:10<3:10:07,  7.82s/it]
 53%|█████▎    | 1666/3123 [3:40:18<3:09:30,  7.80s/it]
 53%|█████▎    | 1667/3123 [3:40:26<3:11:14,  7.88s/it]
 53%|█████▎    | 1668/3123 [3:40:34<3:10:13,  7.84s/it]
 53%|█████▎    | 1669/3123 [3:40:41<3:09:28,  7.82s/it]
 53%|█████▎    | 1670/3123 [3:40:49<3:10:57,  7.89s/it]
                                                       
{'loss': '0.001924', 'grad_norm': '0.1416', 'learning_rate': '0.0001', 'ppl': '1.002', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '81.6', 'tokens/total': 27371520, 'tokens/trainable': 8576115, 'epoch': '1.604'}
+
 53%|█████▎    | 1670/3123 [3:40:50<3:10:57,  7.89s/it]
 54%|█████▎    | 1671/3123 [3:40:57<3:09:57,  7.85s/it]
 54%|█████▎    | 1672/3123 [3:41:05<3:09:08,  7.82s/it]
 54%|█████▎    | 1673/3123 [3:41:13<3:08:36,  7.80s/it]
 54%|█████▎    | 1674/3123 [3:41:21<3:10:17,  7.88s/it]
 54%|█████▎    | 1675/3123 [3:41:29<3:09:17,  7.84s/it]
 54%|█████▎    | 1676/3123 [3:41:36<3:08:38,  7.82s/it]
 54%|█████▎    | 1677/3123 [3:41:44<3:10:09,  7.89s/it]
 54%|█████▎    | 1678/3123 [3:41:52<3:09:05,  7.85s/it]
 54%|█████▍    | 1679/3123 [3:42:00<3:08:21,  7.83s/it]
 54%|█████▍    | 1680/3123 [3:42:08<3:10:03,  7.90s/it]
                                                       
{'loss': '0.001498', 'grad_norm': '0.08594', 'learning_rate': '0.0001', 'ppl': '1.002', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '83.88', 'tokens/total': 27535360, 'tokens/trainable': 8627475, 'epoch': '1.613'}
+
 54%|█████▍    | 1680/3123 [3:42:08<3:10:03,  7.90s/it]
 54%|█████▍    | 1681/3123 [3:42:16<3:08:57,  7.86s/it]
 54%|█████▍    | 1682/3123 [3:42:23<3:08:12,  7.84s/it]
 54%|█████▍    | 1683/3123 [3:42:31<3:07:39,  7.82s/it]
 54%|█████▍    | 1684/3123 [3:42:39<3:09:10,  7.89s/it]
 54%|█████▍    | 1685/3123 [3:42:47<3:08:08,  7.85s/it]
 54%|█████▍    | 1686/3123 [3:42:55<3:07:23,  7.82s/it]
 54%|█████▍    | 1687/3123 [3:43:03<3:08:57,  7.90s/it]
 54%|█████▍    | 1688/3123 [3:43:11<3:07:55,  7.86s/it]
 54%|█████▍    | 1689/3123 [3:43:18<3:07:05,  7.83s/it]
 54%|█████▍    | 1690/3123 [3:43:26<3:06:28,  7.81s/it]
                                                       
{'loss': '0.00127', 'grad_norm': '0.04224', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '79.16', 'tokens/total': 27699200, 'tokens/trainable': 8678654, 'epoch': '1.623'}
+
 54%|█████▍    | 1690/3123 [3:43:26<3:06:28,  7.81s/it]
 54%|█████▍    | 1691/3123 [3:43:34<3:08:03,  7.88s/it]
 54%|█████▍    | 1692/3123 [3:43:42<3:07:08,  7.85s/it]
 54%|█████▍    | 1693/3123 [3:43:50<3:06:30,  7.83s/it]
 54%|█████▍    | 1694/3123 [3:43:58<3:08:01,  7.89s/it]
 54%|█████▍    | 1695/3123 [3:44:06<3:06:55,  7.85s/it]
 54%|█████▍    | 1696/3123 [3:44:13<3:06:08,  7.83s/it]
 54%|█████▍    | 1697/3123 [3:44:21<3:05:35,  7.81s/it]
 54%|█████▍    | 1698/3123 [3:44:29<3:07:07,  7.88s/it]
 54%|█████▍    | 1699/3123 [3:44:37<3:06:09,  7.84s/it]
 54%|█████▍    | 1700/3123 [3:44:45<3:05:28,  7.82s/it]
                                                       
{'loss': '0.001338', 'grad_norm': '0.1099', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '82.06', 'tokens/total': 27863040, 'tokens/trainable': 8729739, 'epoch': '1.633'}
+
 54%|█████▍    | 1700/3123 [3:44:45<3:05:28,  7.82s/it]
 54%|█████▍    | 1701/3123 [3:44:53<3:07:00,  7.89s/it]
 54%|█████▍    | 1702/3123 [3:45:01<3:05:59,  7.85s/it]
 55%|█████▍    | 1703/3123 [3:45:08<3:05:12,  7.83s/it]
 55%|█████▍    | 1704/3123 [3:45:16<3:04:41,  7.81s/it]
 55%|█████▍    | 1705/3123 [3:45:24<3:06:10,  7.88s/it]
 55%|█████▍    | 1706/3123 [3:45:32<3:05:15,  7.84s/it]
 55%|█████▍    | 1707/3123 [3:45:40<3:04:33,  7.82s/it]
 55%|█████▍    | 1708/3123 [3:45:48<3:06:06,  7.89s/it]
 55%|█████▍    | 1709/3123 [3:45:55<3:05:02,  7.85s/it]
 55%|█████▍    | 1710/3123 [3:46:03<3:04:19,  7.83s/it]
                                                       
{'loss': '0.001281', 'grad_norm': '0.06299', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '82.28', 'tokens/total': 28026880, 'tokens/trainable': 8781378, 'epoch': '1.642'}
+
 55%|█████▍    | 1710/3123 [3:46:04<3:04:19,  7.83s/it]
 55%|█████▍    | 1711/3123 [3:46:11<3:03:47,  7.81s/it]
 55%|█████▍    | 1712/3123 [3:46:19<3:05:22,  7.88s/it]
 55%|█████▍    | 1713/3123 [3:46:27<3:04:25,  7.85s/it]
 55%|█████▍    | 1714/3123 [3:46:35<3:03:41,  7.82s/it]
 55%|█████▍    | 1715/3123 [3:46:43<3:05:15,  7.89s/it]
 55%|█████▍    | 1716/3123 [3:46:50<3:04:12,  7.86s/it]
 55%|█████▍    | 1717/3123 [3:46:58<3:03:25,  7.83s/it]
 55%|█████▌    | 1718/3123 [3:47:06<3:02:52,  7.81s/it]
 55%|█████▌    | 1719/3123 [3:47:14<3:04:23,  7.88s/it]
 55%|█████▌    | 1720/3123 [3:47:22<3:03:28,  7.85s/it]
                                                       
{'loss': '0.0007919', 'grad_norm': '0.03442', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '83.09', 'tokens/total': 28190720, 'tokens/trainable': 8832775, 'epoch': '1.652'}
+
 55%|█████▌    | 1720/3123 [3:47:22<3:03:28,  7.85s/it]
 55%|█████▌    | 1721/3123 [3:47:29<3:02:48,  7.82s/it]
 55%|█████▌    | 1722/3123 [3:47:38<3:04:22,  7.90s/it]
 55%|█████▌    | 1723/3123 [3:47:45<3:03:17,  7.86s/it]
 55%|█████▌    | 1724/3123 [3:47:53<3:02:30,  7.83s/it]
 55%|█████▌    | 1725/3123 [3:48:01<3:01:53,  7.81s/it]
 55%|█████▌    | 1726/3123 [3:48:09<3:03:25,  7.88s/it]
 55%|█████▌    | 1727/3123 [3:48:17<3:02:30,  7.84s/it]
 55%|█████▌    | 1728/3123 [3:48:24<3:01:50,  7.82s/it]
 55%|█████▌    | 1729/3123 [3:48:32<3:03:16,  7.89s/it]
 55%|█████▌    | 1730/3123 [3:48:40<3:02:14,  7.85s/it]
                                                       
{'loss': '0.001336', 'grad_norm': '0.05322', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '74.48', 'tokens/total': 28354560, 'tokens/trainable': 8884196, 'epoch': '1.661'}
+
 55%|█████▌    | 1730/3123 [3:48:41<3:02:14,  7.85s/it]
 55%|█████▌    | 1731/3123 [3:48:48<3:01:31,  7.82s/it]
 55%|█████▌    | 1732/3123 [3:48:56<3:00:57,  7.81s/it]
 55%|█████▌    | 1733/3123 [3:49:04<3:02:32,  7.88s/it]
 56%|█████▌    | 1734/3123 [3:49:12<3:01:35,  7.84s/it]
 56%|█████▌    | 1735/3123 [3:49:19<3:00:51,  7.82s/it]
 56%|█████▌    | 1736/3123 [3:49:27<3:02:23,  7.89s/it]
 56%|█████▌    | 1737/3123 [3:49:35<3:01:21,  7.85s/it]
 56%|█████▌    | 1738/3123 [3:49:43<3:00:36,  7.82s/it]
 56%|█████▌    | 1739/3123 [3:49:51<3:00:03,  7.81s/it]
 56%|█████▌    | 1740/3123 [3:49:59<3:01:33,  7.88s/it]
                                                       
{'loss': '0.001094', 'grad_norm': '0.0498', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '88.41', 'tokens/total': 28518400, 'tokens/trainable': 8935603, 'epoch': '1.671'}
+
 56%|█████▌    | 1740/3123 [3:49:59<3:01:33,  7.88s/it]
 56%|█████▌    | 1741/3123 [3:50:06<3:00:36,  7.84s/it]
 56%|█████▌    | 1742/3123 [3:50:14<2:59:59,  7.82s/it]
 56%|█████▌    | 1743/3123 [3:50:22<3:01:36,  7.90s/it]
 56%|█████▌    | 1744/3123 [3:50:30<3:00:29,  7.85s/it]
 56%|█████▌    | 1745/3123 [3:50:38<2:59:42,  7.83s/it]
 56%|█████▌    | 1746/3123 [3:50:46<3:01:05,  7.89s/it]
 56%|█████▌    | 1747/3123 [3:50:54<3:00:04,  7.85s/it]
 56%|█████▌    | 1748/3123 [3:51:01<2:59:18,  7.82s/it]
 56%|█████▌    | 1749/3123 [3:51:09<2:58:45,  7.81s/it]
 56%|█████▌    | 1750/3123 [3:51:17<3:00:22,  7.88s/it]
                                                       
{'loss': '0.0009528', 'grad_norm': '0.02161', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '79.45', 'tokens/total': 28682240, 'tokens/trainable': 8987083, 'epoch': '1.681'}
+
 56%|█████▌    | 1750/3123 [3:51:18<3:00:22,  7.88s/it]
 56%|█████▌    | 1751/3123 [3:51:25<2:59:26,  7.85s/it]
 56%|█████▌    | 1752/3123 [3:51:33<2:58:42,  7.82s/it]
 56%|█████▌    | 1753/3123 [3:51:41<3:00:04,  7.89s/it]
 56%|█████▌    | 1754/3123 [3:51:49<2:59:05,  7.85s/it]
 56%|█████▌    | 1755/3123 [3:51:56<2:58:21,  7.82s/it]
 56%|█████▌    | 1756/3123 [3:52:04<2:57:48,  7.80s/it]
 56%|█████▋    | 1757/3123 [3:52:12<2:59:22,  7.88s/it]
 56%|█████▋    | 1758/3123 [3:52:20<2:58:22,  7.84s/it]
 56%|█████▋    | 1759/3123 [3:52:28<2:57:41,  7.82s/it]
 56%|█████▋    | 1760/3123 [3:52:36<2:59:09,  7.89s/it]
                                                       
{'loss': '0.000604', 'grad_norm': '0.07617', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '84.21', 'tokens/total': 28846080, 'tokens/trainable': 9038209, 'epoch': '1.69'}
+
 56%|█████▋    | 1760/3123 [3:52:36<2:59:09,  7.89s/it]
 56%|█████▋    | 1761/3123 [3:52:43<2:58:09,  7.85s/it]
 56%|█████▋    | 1762/3123 [3:52:51<2:57:24,  7.82s/it]
 56%|█████▋    | 1763/3123 [3:52:59<2:56:52,  7.80s/it]
 56%|███���█▋    | 1764/3123 [3:53:07<2:58:30,  7.88s/it]
 57%|█████▋    | 1765/3123 [3:53:15<2:57:34,  7.85s/it]
 57%|█████▋    | 1766/3123 [3:53:23<2:56:48,  7.82s/it]
 57%|█████▋    | 1767/3123 [3:53:31<2:58:11,  7.88s/it]
 57%|█████▋    | 1768/3123 [3:53:38<2:57:13,  7.85s/it]
 57%|█████▋    | 1769/3123 [3:53:46<2:56:32,  7.82s/it]
 57%|█████▋    | 1770/3123 [3:53:54<2:55:59,  7.80s/it]
                                                       
{'loss': '0.0006968', 'grad_norm': '0.03442', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '84.79', 'tokens/total': 29009920, 'tokens/trainable': 9088795, 'epoch': '1.7'}
+
 57%|█████▋    | 1770/3123 [3:53:54<2:55:59,  7.80s/it]
 57%|█████▋    | 1771/3123 [3:54:02<2:57:30,  7.88s/it]
 57%|█████▋    | 1772/3123 [3:54:10<2:56:34,  7.84s/it]
 57%|█████▋    | 1773/3123 [3:54:17<2:55:51,  7.82s/it]
 57%|█████▋    | 1774/3123 [3:54:25<2:57:13,  7.88s/it]
 57%|█████▋    | 1775/3123 [3:54:33<2:56:14,  7.84s/it]
 57%|█████▋    | 1776/3123 [3:54:41<2:55:37,  7.82s/it]
 57%|█████▋    | 1777/3123 [3:54:49<2:55:07,  7.81s/it]
 57%|█████▋    | 1778/3123 [3:54:57<2:56:38,  7.88s/it]
 57%|█████▋    | 1779/3123 [3:55:05<2:55:42,  7.84s/it]
 57%|█████▋    | 1780/3123 [3:55:12<2:55:01,  7.82s/it]
                                                       
{'loss': '0.0008366', 'grad_norm': '0.01636', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '91.49', 'tokens/total': 29173760, 'tokens/trainable': 9140155, 'epoch': '1.709'}
+
 57%|█████▋    | 1780/3123 [3:55:13<2:55:01,  7.82s/it]
 57%|█████▋    | 1781/3123 [3:55:20<2:56:24,  7.89s/it]
 57%|█████▋    | 1782/3123 [3:55:28<2:55:26,  7.85s/it]
 57%|█████▋    | 1783/3123 [3:55:36<2:54:42,  7.82s/it]
 57%|█████▋    | 1784/3123 [3:55:44<2:54:08,  7.80s/it]
 57%|█████▋    | 1785/3123 [3:55:52<2:55:46,  7.88s/it]
 57%|█████▋    | 1786/3123 [3:55:59<2:54:50,  7.85s/it]
 57%|█████▋    | 1787/3123 [3:56:07<2:54:07,  7.82s/it]
 57%|█████▋    | 1788/3123 [3:56:15<2:55:29,  7.89s/it]
 57%|█████▋    | 1789/3123 [3:56:23<2:54:31,  7.85s/it]
 57%|█████▋    | 1790/3123 [3:56:31<2:53:52,  7.83s/it]
                                                       
{'loss': '0.000542', 'grad_norm': '0.08105', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '91.11', 'tokens/total': 29337600, 'tokens/trainable': 9191457, 'epoch': '1.719'}
+
 57%|█████▋    | 1790/3123 [3:56:31<2:53:52,  7.83s/it]
 57%|█████▋    | 1791/3123 [3:56:39<2:53:20,  7.81s/it]
 57%|█████▋    | 1792/3123 [3:56:47<2:54:55,  7.89s/it]
 57%|█████▋    | 1793/3123 [3:56:54<2:53:55,  7.85s/it]
 57%|█████▋    | 1794/3123 [3:57:02<2:53:13,  7.82s/it]
 57%|█████▋    | 1795/3123 [3:57:10<2:54:34,  7.89s/it]
 58%|█████▊    | 1796/3123 [3:57:18<2:53:35,  7.85s/it]
 58%|█████▊    | 1797/3123 [3:57:26<2:52:51,  7.82s/it]
 58%|█████▊    | 1798/3123 [3:57:33<2:52:20,  7.80s/it]
 58%|█████▊    | 1799/3123 [3:57:42<2:53:59,  7.88s/it]
 58%|█████▊    | 1800/3123 [3:57:49<2:53:04,  7.85s/it]
                                                       
{'loss': '0.0008765', 'grad_norm': '0.1055', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '96.94', 'tokens/total': 29501440, 'tokens/trainable': 9242381, 'epoch': '1.729'}
+
 58%|█████▊    | 1800/3123 [3:57:50<2:53:04,  7.85s/it]
 58%|█████▊    | 1801/3123 [3:57:57<2:52:24,  7.83s/it]
 58%|█████▊    | 1802/3123 [3:58:05<2:53:44,  7.89s/it]
 58%|█████▊    | 1803/3123 [3:58:13<2:52:47,  7.85s/it]
 58%|█████▊    | 1804/3123 [3:58:21<2:51:59,  7.82s/it]
 58%|█████▊    | 1805/3123 [3:58:28<2:51:29,  7.81s/it]
 58%|█████▊    | 1806/3123 [3:58:36<2:53:00,  7.88s/it]
 58%|█████▊    | 1807/3123 [3:58:44<2:52:05,  7.85s/it]
 58%|█████▊    | 1808/3123 [3:58:52<2:51:23,  7.82s/it]
 58%|█████▊    | 1809/3123 [3:59:00<2:52:39,  7.88s/it]
 58%|█████▊    | 1810/3123 [3:59:08<2:51:41,  7.85s/it]
                                                       
{'loss': '0.001664', 'grad_norm': '0.3652', 'learning_rate': '0.0001', 'ppl': '1.002', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '77.72', 'tokens/total': 29665280, 'tokens/trainable': 9294105, 'epoch': '1.738'}
+
 58%|█████▊    | 1810/3123 [3:59:08<2:51:41,  7.85s/it]
 58%|█████▊    | 1811/3123 [3:59:16<2:51:03,  7.82s/it]
 58%|█████▊    | 1812/3123 [3:59:24<2:52:38,  7.90s/it]
 58%|█████▊    | 1813/3123 [3:59:31<2:51:36,  7.86s/it]
 58%|█████▊    | 1814/3123 [3:59:39<2:50:49,  7.83s/it]
 58%|█████▊    | 1815/3123 [3:59:47<2:50:16,  7.81s/it]
 58%|█████▊    | 1816/3123 [3:59:55<2:51:40,  7.88s/it]
 58%|█████▊    | 1817/3123 [4:00:03<2:50:47,  7.85s/it]
 58%|█████▊    | 1818/3123 [4:00:11<2:50:02,  7.82s/it]
 58%|█████▊    | 1819/3123 [4:00:19<2:51:27,  7.89s/it]
 58%|█████▊    | 1820/3123 [4:00:26<2:50:28,  7.85s/it]
                                                       
{'loss': '0.002325', 'grad_norm': '0.05786', 'learning_rate': '0.0001', 'ppl': '1.002', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '91.39', 'tokens/total': 29829120, 'tokens/trainable': 9345913, 'epoch': '1.748'}
+
 58%|█████▊    | 1820/3123 [4:00:27<2:50:28,  7.85s/it]
 58%|█████▊    | 1821/3123 [4:00:34<2:49:45,  7.82s/it]
 58%|█████▊    | 1822/3123 [4:00:42<2:49:13,  7.80s/it]
 58%|█████▊    | 1823/3123 [4:00:50<2:50:43,  7.88s/it]
 58%|█████▊    | 1824/3123 [4:00:58<2:49:50,  7.84s/it]
 58%|█████▊    | 1825/3123 [4:01:05<2:49:11,  7.82s/it]
 58%|█████▊    | 1826/3123 [4:01:13<2:50:36,  7.89s/it]
 59%|█████▊    | 1827/3123 [4:01:21<2:49:37,  7.85s/it]
 59%|█████▊    | 1828/3123 [4:01:29<2:48:54,  7.83s/it]
 59%|█████▊    | 1829/3123 [4:01:37<2:48:21,  7.81s/it]
 59%|█████▊    | 1830/3123 [4:01:45<2:49:49,  7.88s/it]
                                                       
{'loss': '0.004123', 'grad_norm': '0.1953', 'learning_rate': '0.0001', 'ppl': '1.004', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '80.94', 'tokens/total': 29992960, 'tokens/trainable': 9397553, 'epoch': '1.757'}
+
 59%|█████▊    | 1830/3123 [4:01:45<2:49:49,  7.88s/it]
 59%|█████▊    | 1831/3123 [4:01:53<2:48:59,  7.85s/it]
 59%|█████▊    | 1832/3123 [4:02:00<2:48:20,  7.82s/it]
 59%|█████▊    | 1833/3123 [4:02:08<2:49:51,  7.90s/it]
 59%|█████▊    | 1834/3123 [4:02:16<2:48:52,  7.86s/it]
 59%|█████▉    | 1835/3123 [4:02:24<2:48:06,  7.83s/it]
 59%|█████▉    | 1836/3123 [4:02:32<2:47:34,  7.81s/it]
 59%|█████▉    | 1837/3123 [4:02:40<2:49:00,  7.89s/it]
 59%|█████▉    | 1838/3123 [4:02:48<2:48:06,  7.85s/it]
 59%|█████▉    | 1839/3123 [4:02:55<2:47:28,  7.83s/it]
 59%|█████▉    | 1840/3123 [4:03:03<2:48:53,  7.90s/it]
                                                       
{'loss': '0.00476', 'grad_norm': '0.1875', 'learning_rate': '0.0001', 'ppl': '1.005', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '91.24', 'tokens/total': 30156800, 'tokens/trainable': 9448264, 'epoch': '1.767'}
+
 59%|█████▉    | 1840/3123 [4:03:04<2:48:53,  7.90s/it]
 59%|█████▉    | 1841/3123 [4:03:11<2:47:56,  7.86s/it]
 59%|█████▉    | 1842/3123 [4:03:19<2:47:12,  7.83s/it]
 59%|█████▉    | 1843/3123 [4:03:27<2:46:38,  7.81s/it]
 59%|█████▉    | 1844/3123 [4:03:35<2:48:07,  7.89s/it]
 59%|█████▉    | 1845/3123 [4:03:43<2:47:10,  7.85s/it]
 59%|█████▉    | 1846/3123 [4:03:50<2:46:28,  7.82s/it]
 59%|█████▉    | 1847/3123 [4:03:58<2:47:54,  7.90s/it]
 59%|█████▉    | 1848/3123 [4:04:06<2:47:01,  7.86s/it]
 59%|█████▉    | 1849/3123 [4:04:14<2:46:18,  7.83s/it]
 59%|█████▉    | 1850/3123 [4:04:22<2:45:43,  7.81s/it]
                                                       
{'loss': '0.003011', 'grad_norm': '0.1235', 'learning_rate': '0.0001', 'ppl': '1.003', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '88.67', 'tokens/total': 30320640, 'tokens/trainable': 9499714, 'epoch': '1.777'}
+
 59%|█████▉    | 1850/3123 [4:04:22<2:45:43,  7.81s/it]
 59%|█████▉    | 1851/3123 [4:04:30<2:47:12,  7.89s/it]
 59%|█████▉    | 1852/3123 [4:04:37<2:46:20,  7.85s/it]
 59%|█████▉    | 1853/3123 [4:04:45<2:45:40,  7.83s/it]
 59%|█████▉    | 1854/3123 [4:04:53<2:47:01,  7.90s/it]
 59%|█████▉    | 1855/3123 [4:05:01<2:46:03,  7.86s/it]
 59%|█████▉    | 1856/3123 [4:05:09<2:45:23,  7.83s/it]
 59%|█████▉    | 1857/3123 [4:05:17<2:44:51,  7.81s/it]
 59%|█████▉    | 1858/3123 [4:05:25<2:46:17,  7.89s/it]
 60%|█████▉    | 1859/3123 [4:05:32<2:45:27,  7.85s/it]
 60%|█████▉    | 1860/3123 [4:05:40<2:44:47,  7.83s/it]
                                                       
{'loss': '0.002122', 'grad_norm': '0.08936', 'learning_rate': '0.0001', 'ppl': '1.002', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '86.96', 'tokens/total': 30484480, 'tokens/trainable': 9551484, 'epoch': '1.786'}
+
 60%|█████▉    | 1860/3123 [4:05:41<2:44:47,  7.83s/it]
 60%|█████▉    | 1861/3123 [4:05:48<2:46:12,  7.90s/it]
 60%|█████▉    | 1862/3123 [4:05:56<2:45:10,  7.86s/it]
 60%|█████▉    | 1863/3123 [4:06:04<2:44:26,  7.83s/it]
 60%|█████▉    | 1864/3123 [4:06:12<2:43:51,  7.81s/it]
 60%|█████▉    | 1865/3123 [4:06:20<2:45:17,  7.88s/it]
 60%|█████▉    | 1866/3123 [4:06:27<2:44:24,  7.85s/it]
 60%|█████▉    | 1867/3123 [4:06:35<2:43:46,  7.82s/it]
 60%|█████▉    | 1868/3123 [4:06:43<2:45:06,  7.89s/it]
 60%|█████▉    | 1869/3123 [4:06:51<2:44:07,  7.85s/it]
 60%|█████▉    | 1870/3123 [4:06:59<2:43:27,  7.83s/it]
                                                       
{'loss': '0.001932', 'grad_norm': '0.05957', 'learning_rate': '0.0001', 'ppl': '1.002', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '87.13', 'tokens/total': 30648320, 'tokens/trainable': 9603376, 'epoch': '1.796'}
+
 60%|█████▉    | 1870/3123 [4:06:59<2:43:27,  7.83s/it]
 60%|█████▉    | 1871/3123 [4:07:07<2:44:45,  7.90s/it]
 60%|█████▉    | 1872/3123 [4:07:15<2:43:49,  7.86s/it]
 60%|█████▉    | 1873/3123 [4:07:22<2:43:08,  7.83s/it]
 60%|██████    | 1874/3123 [4:07:30<2:42:31,  7.81s/it]
 60%|██████    | 1875/3123 [4:07:38<2:44:00,  7.89s/it]
 60%|██████    | 1876/3123 [4:07:46<2:43:07,  7.85s/it]
 60%|██████    | 1877/3123 [4:07:54<2:42:28,  7.82s/it]
 60%|██████    | 1878/3123 [4:08:02<2:43:45,  7.89s/it]
 60%|██████    | 1879/3123 [4:08:10<2:42:49,  7.85s/it]
 60%|██████    | 1880/3123 [4:08:17<2:42:08,  7.83s/it]
                                                       
{'loss': '0.002131', 'grad_norm': '0.1572', 'learning_rate': '0.0001', 'ppl': '1.002', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '82.45', 'tokens/total': 30812160, 'tokens/trainable': 9654126, 'epoch': '1.805'}
+
 60%|██████    | 1880/3123 [4:08:18<2:42:08,  7.83s/it]
 60%|██████    | 1881/3123 [4:08:25<2:41:38,  7.81s/it]
 60%|██████    | 1882/3123 [4:08:33<2:43:08,  7.89s/it]
 60%|██████    | 1883/3123 [4:08:41<2:42:16,  7.85s/it]
 60%|██████    | 1884/3123 [4:08:49<2:41:36,  7.83s/it]
 60%|██████    | 1885/3123 [4:08:57<2:42:50,  7.89s/it]
 60%|██████    | 1886/3123 [4:09:04<2:41:54,  7.85s/it]
 60%|██████    | 1887/3123 [4:09:12<2:41:12,  7.83s/it]
 60%|██████    | 1888/3123 [4:09:20<2:40:39,  7.81s/it]
 60%|██████    | 1889/3123 [4:09:28<2:42:04,  7.88s/it]
 61%|██████    | 1890/3123 [4:09:36<2:41:13,  7.85s/it]
                                                       
{'loss': '0.001395', 'grad_norm': '0.1055', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '97.11', 'tokens/total': 30976000, 'tokens/trainable': 9704260, 'epoch': '1.815'}
+
 61%|██████    | 1890/3123 [4:09:36<2:41:13,  7.85s/it]
 61%|██████    | 1891/3123 [4:09:44<2:40:36,  7.82s/it]
 61%|██████    | 1892/3123 [4:09:52<2:41:50,  7.89s/it]
 61%|██████    | 1893/3123 [4:09:59<2:40:58,  7.85s/it]
 61%|██████    | 1894/3123 [4:10:07<2:40:18,  7.83s/it]
 61%|██████    | 1895/3123 [4:10:15<2:39:47,  7.81s/it]
 61%|██████    | 1896/3123 [4:10:23<2:41:11,  7.88s/it]
 61%|██████    | 1897/3123 [4:10:31<2:40:19,  7.85s/it]
 61%|██████    | 1898/3123 [4:10:38<2:39:40,  7.82s/it]
 61%|██████    | 1899/3123 [4:10:47<2:40:53,  7.89s/it]
 61%|██████    | 1900/3123 [4:10:54<2:39:58,  7.85s/it]
                                                       
{'loss': '0.001356', 'grad_norm': '0.06592', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '80.51', 'tokens/total': 31139840, 'tokens/trainable': 9755570, 'epoch': '1.825'}
+
 61%|██████    | 1900/3123 [4:10:55<2:39:58,  7.85s/it]
 61%|██████    | 1901/3123 [4:11:02<2:39:19,  7.82s/it]
 61%|██████    | 1902/3123 [4:11:10<2:38:47,  7.80s/it]
 61%|██████    | 1903/3123 [4:11:18<2:40:12,  7.88s/it]
 61%|██████    | 1904/3123 [4:11:26<2:39:22,  7.84s/it]
 61%|██████    | 1905/3123 [4:11:33<2:38:45,  7.82s/it]
 61%|██████    | 1906/3123 [4:11:41<2:40:02,  7.89s/it]
 61%|██████    | 1907/3123 [4:11:49<2:39:08,  7.85s/it]
 61%|██████    | 1908/3123 [4:11:57<2:38:27,  7.83s/it]
 61%|██████    | 1909/3123 [4:12:05<2:37:59,  7.81s/it]
 61%|██████    | 1910/3123 [4:12:13<2:39:24,  7.88s/it]
                                                       
{'loss': '0.001438', 'grad_norm': '0.09961', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '83.67', 'tokens/total': 31303680, 'tokens/trainable': 9806775, 'epoch': '1.834'}
+
 61%|██████    | 1910/3123 [4:12:13<2:39:24,  7.88s/it]
 61%|██████    | 1911/3123 [4:12:21<2:38:32,  7.85s/it]
 61%|██████    | 1912/3123 [4:12:28<2:37:52,  7.82s/it]
 61%|██████▏   | 1913/3123 [4:12:36<2:39:04,  7.89s/it]
 61%|██████▏   | 1914/3123 [4:12:44<2:38:08,  7.85s/it]
 61%|██████▏   | 1915/3123 [4:12:52<2:37:31,  7.82s/it]
 61%|██████▏   | 1916/3123 [4:13:00<2:37:00,  7.81s/it]
 61%|██████▏   | 1917/3123 [4:13:08<2:38:30,  7.89s/it]
 61%|██████▏   | 1918/3123 [4:13:15<2:37:37,  7.85s/it]
 61%|██████▏   | 1919/3123 [4:13:23<2:36:58,  7.82s/it]
 61%|██████▏   | 1920/3123 [4:13:31<2:38:15,  7.89s/it]
                                                       
{'loss': '0.001355', 'grad_norm': '0.09082', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '86.32', 'tokens/total': 31467520, 'tokens/trainable': 9858270, 'epoch': '1.844'}
+
 61%|██████▏   | 1920/3123 [4:13:32<2:38:15,  7.89s/it]
 62%|██████▏   | 1921/3123 [4:13:39<2:37:18,  7.85s/it]
 62%|██████▏   | 1922/3123 [4:13:47<2:36:38,  7.83s/it]
 62%|██████▏   | 1923/3123 [4:13:55<2:36:08,  7.81s/it]
 62%|██████▏   | 1924/3123 [4:14:03<2:37:33,  7.88s/it]
 62%|██████▏   | 1925/3123 [4:14:10<2:36:41,  7.85s/it]
 62%|██████▏   | 1926/3123 [4:14:18<2:36:04,  7.82s/it]
 62%|██████▏   | 1927/3123 [4:14:26<2:37:19,  7.89s/it]
 62%|██████▏   | 1928/3123 [4:14:34<2:36:25,  7.85s/it]
 62%|██████▏   | 1929/3123 [4:14:42<2:35:45,  7.83s/it]
 62%|██████▏   | 1930/3123 [4:14:50<2:35:14,  7.81s/it]
                                                       
{'loss': '0.001307', 'grad_norm': '0.02588', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '86.24', 'tokens/total': 31631360, 'tokens/trainable': 9909516, 'epoch': '1.853'}
+
 62%|██████▏   | 1930/3123 [4:14:50<2:35:14,  7.81s/it]
 62%|██████▏   | 1931/3123 [4:14:58<2:36:42,  7.89s/it]
 62%|██████▏   | 1932/3123 [4:15:05<2:35:49,  7.85s/it]
 62%|██████▏   | 1933/3123 [4:15:13<2:35:12,  7.83s/it]
 62%|██████▏   | 1934/3123 [4:15:21<2:36:24,  7.89s/it]
 62%|██████▏   | 1935/3123 [4:15:29<2:35:30,  7.85s/it]
 62%|██████▏   | 1936/3123 [4:15:37<2:34:50,  7.83s/it]
 62%|██████▏   | 1937/3123 [4:15:44<2:34:20,  7.81s/it]
 62%|██████▏   | 1938/3123 [4:15:53<2:35:43,  7.88s/it]
 62%|██████▏   | 1939/3123 [4:16:00<2:34:53,  7.85s/it]
 62%|██████▏   | 1940/3123 [4:16:08<2:34:14,  7.82s/it]
                                                       
{'loss': '0.001158', 'grad_norm': '0.08447', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '83.45', 'tokens/total': 31795200, 'tokens/trainable': 9960863, 'epoch': '1.863'}
+
 62%|██████▏   | 1940/3123 [4:16:08<2:34:14,  7.82s/it]
 62%|██████▏   | 1941/3123 [4:16:16<2:35:29,  7.89s/it]
 62%|██████▏   | 1942/3123 [4:16:24<2:34:38,  7.86s/it]
 62%|██████▏   | 1943/3123 [4:16:32<2:33:56,  7.83s/it]
 62%|██████▏   | 1944/3123 [4:16:40<2:35:13,  7.90s/it]
 62%|██████▏   | 1945/3123 [4:16:47<2:34:16,  7.86s/it]
 62%|██████▏   | 1946/3123 [4:16:55<2:33:37,  7.83s/it]
 62%|██████▏   | 1947/3123 [4:17:03<2:33:05,  7.81s/it]
 62%|██████▏   | 1948/3123 [4:17:11<2:34:23,  7.88s/it]
 62%|██████▏   | 1949/3123 [4:17:19<2:33:34,  7.85s/it]
 62%|██████▏   | 1950/3123 [4:17:27<2:32:55,  7.82s/it]
                                                       
{'loss': '0.001168', 'grad_norm': '0.02637', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '77.94', 'tokens/total': 31959040, 'tokens/trainable': 10013368, 'epoch': '1.873'}
+
 62%|██████▏   | 1950/3123 [4:17:27<2:32:55,  7.82s/it]
 62%|██████▏   | 1951/3123 [4:17:35<2:34:19,  7.90s/it]
 63%|██████▎   | 1952/3123 [4:17:42<2:33:22,  7.86s/it]
 63%|██████▎   | 1953/3123 [4:17:50<2:32:40,  7.83s/it]
 63%|██████▎   | 1954/3123 [4:17:58<2:32:11,  7.81s/it]
 63%|██████▎   | 1955/3123 [4:18:06<2:33:25,  7.88s/it]
 63%|██████▎   | 1956/3123 [4:18:14<2:32:37,  7.85s/it]
 63%|██████▎   | 1957/3123 [4:18:22<2:31:56,  7.82s/it]
 63%|██████▎   | 1958/3123 [4:18:30<2:33:18,  7.90s/it]
 63%|██████▎   | 1959/3123 [4:18:37<2:32:23,  7.86s/it]
 63%|██████▎   | 1960/3123 [4:18:45<2:31:42,  7.83s/it]
                                                       
{'loss': '0.0007935', 'grad_norm': '0.05908', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '90.83', 'tokens/total': 32122880, 'tokens/trainable': 10064774, 'epoch': '1.882'}
+
 63%|██████▎   | 1960/3123 [4:18:45<2:31:42,  7.83s/it]
 63%|██████▎   | 1961/3123 [4:18:53<2:31:10,  7.81s/it]
 63%|██████▎   | 1962/3123 [4:19:01<2:32:29,  7.88s/it]
 63%|██████▎   | 1963/3123 [4:19:09<2:31:44,  7.85s/it]
 63%|██████▎   | 1964/3123 [4:19:17<2:31:10,  7.83s/it]
 63%|██████▎   | 1965/3123 [4:19:25<2:32:28,  7.90s/it]
 63%|██████▎   | 1966/3123 [4:19:32<2:31:37,  7.86s/it]
 63%|██████▎   | 1967/3123 [4:19:40<2:30:59,  7.84s/it]
 63%|██████▎   | 1968/3123 [4:19:48<2:30:31,  7.82s/it]
 63%|██████▎   | 1969/3123 [4:19:56<2:31:40,  7.89s/it]
 63%|██████▎   | 1970/3123 [4:20:04<2:30:51,  7.85s/it]
                                                       
{'loss': '0.0008187', 'grad_norm': '0.05542', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '81.75', 'tokens/total': 32286720, 'tokens/trainable': 10116172, 'epoch': '1.892'}
+
 63%|██████▎   | 1970/3123 [4:20:04<2:30:51,  7.85s/it]
 63%|██████▎   | 1971/3123 [4:20:11<2:30:13,  7.82s/it]
 63%|██████▎   | 1972/3123 [4:20:20<2:31:31,  7.90s/it]
 63%|██████▎   | 1973/3123 [4:20:27<2:30:36,  7.86s/it]
 63%|██████▎   | 1974/3123 [4:20:35<2:29:56,  7.83s/it]
 63%|██████▎   | 1975/3123 [4:20:43<2:31:02,  7.89s/it]
 63%|██████▎   | 1976/3123 [4:20:51<2:30:05,  7.85s/it]
 63%|██████▎   | 1977/3123 [4:20:59<2:29:26,  7.82s/it]
 63%|██████▎   | 1978/3123 [4:21:06<2:28:58,  7.81s/it]
 63%|██████▎   | 1979/3123 [4:21:14<2:30:15,  7.88s/it]
 63%|██████▎   | 1980/3123 [4:21:22<2:29:24,  7.84s/it]
                                                       
{'loss': '0.000727', 'grad_norm': '0.0918', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '83.38', 'tokens/total': 32450560, 'tokens/trainable': 10167347, 'epoch': '1.901'}
+
 63%|██████▎   | 1980/3123 [4:21:23<2:29:24,  7.84s/it]
 63%|██████▎   | 1981/3123 [4:21:30<2:28:50,  7.82s/it]
 63%|██████▎   | 1982/3123 [4:21:38<2:30:09,  7.90s/it]
 63%|██████▎   | 1983/3123 [4:21:46<2:29:16,  7.86s/it]
 64%|██████▎   | 1984/3123 [4:21:54<2:28:40,  7.83s/it]
 64%|██████▎   | 1985/3123 [4:22:01<2:28:09,  7.81s/it]
 64%|██████▎   | 1986/3123 [4:22:09<2:29:29,  7.89s/it]
 64%|██████▎   | 1987/3123 [4:22:17<2:28:40,  7.85s/it]
 64%|██████▎   | 1988/3123 [4:22:25<2:28:01,  7.82s/it]
 64%|██████▎   | 1989/3123 [4:22:33<2:29:08,  7.89s/it]
 64%|██████▎   | 1990/3123 [4:22:41<2:28:17,  7.85s/it]
                                                       
{'loss': '0.001102', 'grad_norm': '0.0918', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '95.21', 'tokens/total': 32614400, 'tokens/trainable': 10218140, 'epoch': '1.911'}
+
 64%|██████▎   | 1990/3123 [4:22:41<2:28:17,  7.85s/it]
 64%|██████▍   | 1991/3123 [4:22:49<2:27:39,  7.83s/it]
 64%|██████▍   | 1992/3123 [4:22:56<2:27:10,  7.81s/it]
 64%|██████▍   | 1993/3123 [4:23:04<2:28:28,  7.88s/it]
 64%|██████▍   | 1994/3123 [4:23:12<2:27:41,  7.85s/it]
 64%|██████▍   | 1995/3123 [4:23:20<2:27:06,  7.83s/it]
 64%|██████▍   | 1996/3123 [4:23:28<2:28:11,  7.89s/it]
 64%|██████▍   | 1997/3123 [4:23:36<2:27:21,  7.85s/it]
 64%|██████▍   | 1998/3123 [4:23:43<2:26:46,  7.83s/it]
 64%|██████▍   | 1999/3123 [4:23:51<2:26:14,  7.81s/it]
 64%|██████▍   | 2000/3123 [4:23:59<2:27:35,  7.89s/it]
                                                       
{'loss': '0.0009611', 'grad_norm': '0.1396', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '69.05', 'tokens/total': 32778240, 'tokens/trainable': 10269179, 'epoch': '1.921'}
+
 64%|██████▍   | 2000/3123 [4:24:00<2:27:35,  7.89s/it]
 64%|██████▍   | 2001/3123 [4:24:07<2:26:48,  7.85s/it]
 64%|██████▍   | 2002/3123 [4:24:15<2:26:13,  7.83s/it]
 64%|██████▍   | 2003/3123 [4:24:23<2:27:24,  7.90s/it]
 64%|██████▍   | 2004/3123 [4:24:31<2:26:35,  7.86s/it]
 64%|██████▍   | 2005/3123 [4:24:38<2:25:55,  7.83s/it]
 64%|██████▍   | 2006/3123 [4:24:46<2:27:07,  7.90s/it]
 64%|██████▍   | 2007/3123 [4:24:54<2:26:15,  7.86s/it]
 64%|██████▍   | 2008/3123 [4:25:02<2:25:34,  7.83s/it]
 64%|██████▍   | 2009/3123 [4:25:10<2:25:05,  7.81s/it]
 64%|██████▍   | 2010/3123 [4:25:18<2:26:18,  7.89s/it]
                                                       
{'loss': '0.0008244', 'grad_norm': '0.02478', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '83.36', 'tokens/total': 32942080, 'tokens/trainable': 10320155, 'epoch': '1.93'}
+
 64%|██████▍   | 2010/3123 [4:25:18<2:26:18,  7.89s/it]
 64%|██████▍   | 2011/3123 [4:25:26<2:25:29,  7.85s/it]
 64%|██████▍   | 2012/3123 [4:25:33<2:24:54,  7.83s/it]
 64%|██████▍   | 2013/3123 [4:25:41<2:26:09,  7.90s/it]
 64%|██████▍   | 2014/3123 [4:25:49<2:25:16,  7.86s/it]
 65%|██████▍   | 2015/3123 [4:25:57<2:24:37,  7.83s/it]
 65%|██████▍   | 2016/3123 [4:26:05<2:24:08,  7.81s/it]
 65%|██████▍   | 2017/3123 [4:26:13<2:25:25,  7.89s/it]
 65%|██████▍   | 2018/3123 [4:26:21<2:24:38,  7.85s/it]
 65%|██████▍   | 2019/3123 [4:26:28<2:24:02,  7.83s/it]
 65%|██████▍   | 2020/3123 [4:26:36<2:25:15,  7.90s/it]
                                                       
{'loss': '0.0006628', 'grad_norm': '0.06396', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '68.11', 'tokens/total': 33105920, 'tokens/trainable': 10371059, 'epoch': '1.94'}
+
 65%|██████▍   | 2020/3123 [4:26:37<2:25:15,  7.90s/it]
 65%|██████▍   | 2021/3123 [4:26:44<2:24:23,  7.86s/it]
 65%|██████▍   | 2022/3123 [4:26:52<2:23:40,  7.83s/it]
 65%|██████▍   | 2023/3123 [4:27:00<2:23:10,  7.81s/it]
 65%|██████▍   | 2024/3123 [4:27:08<2:24:24,  7.88s/it]
 65%|██████▍   | 2025/3123 [4:27:16<2:23:33,  7.84s/it]
 65%|██████▍   | 2026/3123 [4:27:23<2:22:57,  7.82s/it]
 65%|██████▍   | 2027/3123 [4:27:31<2:24:12,  7.89s/it]
 65%|██████▍   | 2028/3123 [4:27:39<2:23:21,  7.85s/it]
 65%|██████▍   | 2029/3123 [4:27:47<2:22:42,  7.83s/it]
 65%|██████▌   | 2030/3123 [4:27:55<2:22:13,  7.81s/it]
                                                       
{'loss': '0.0009559', 'grad_norm': '0.02917', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '88.82', 'tokens/total': 33269760, 'tokens/trainable': 10422089, 'epoch': '1.949'}
+
 65%|██████▌   | 2030/3123 [4:27:55<2:22:13,  7.81s/it]
 65%|██████▌   | 2031/3123 [4:28:03<2:23:30,  7.89s/it]
 65%|██████▌   | 2032/3123 [4:28:10<2:22:41,  7.85s/it]
 65%|██████▌   | 2033/3123 [4:28:18<2:22:07,  7.82s/it]
 65%|██████▌   | 2034/3123 [4:28:26<2:23:16,  7.89s/it]
 65%|██████▌   | 2035/3123 [4:28:34<2:22:23,  7.85s/it]
 65%|██████▌   | 2036/3123 [4:28:42<2:21:47,  7.83s/it]
 65%|██████▌   | 2037/3123 [4:28:50<2:21:17,  7.81s/it]
 65%|██████▌   | 2038/3123 [4:28:58<2:22:32,  7.88s/it]
 65%|██████▌   | 2039/3123 [4:29:05<2:21:43,  7.84s/it]
 65%|██████▌   | 2040/3123 [4:29:13<2:21:09,  7.82s/it]
                                                       
{'loss': '0.0006137', 'grad_norm': '0.0835', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '78.44', 'tokens/total': 33433600, 'tokens/trainable': 10473462, 'epoch': '1.959'}
+
 65%|██████▌   | 2040/3123 [4:29:13<2:21:09,  7.82s/it]
 65%|██████▌   | 2041/3123 [4:29:21<2:22:22,  7.90s/it]
 65%|██████▌   | 2042/3123 [4:29:29<2:21:30,  7.85s/it]
 65%|██████▌   | 2043/3123 [4:29:37<2:20:50,  7.82s/it]
 65%|██████▌   | 2044/3123 [4:29:45<2:21:52,  7.89s/it]
 65%|██████▌   | 2045/3123 [4:29:53<2:21:01,  7.85s/it]
 66%|██████▌   | 2046/3123 [4:30:00<2:20:22,  7.82s/it]
 66%|██████▌   | 2047/3123 [4:30:08<2:19:55,  7.80s/it]
 66%|██████▌   | 2048/3123 [4:30:16<2:21:12,  7.88s/it]
 66%|██████▌   | 2049/3123 [4:30:24<2:20:24,  7.84s/it]
 66%|██████▌   | 2050/3123 [4:30:32<2:19:48,  7.82s/it]
                                                       
{'loss': '0.0007996', 'grad_norm': '0.291', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '84.56', 'tokens/total': 33597440, 'tokens/trainable': 10524388, 'epoch': '1.969'}
+
 66%|██████▌   | 2050/3123 [4:30:32<2:19:48,  7.82s/it]
 66%|██████▌   | 2051/3123 [4:30:40<2:20:51,  7.88s/it]
 66%|██████▌   | 2052/3123 [4:30:47<2:20:01,  7.84s/it]
 66%|██████▌   | 2053/3123 [4:30:55<2:19:26,  7.82s/it]
 66%|██████▌   | 2054/3123 [4:31:03<2:18:58,  7.80s/it]
 66%|██████▌   | 2055/3123 [4:31:11<2:20:09,  7.87s/it]
 66%|██████▌   | 2056/3123 [4:31:19<2:19:22,  7.84s/it]
 66%|██████▌   | 2057/3123 [4:31:27<2:18:49,  7.81s/it]
 66%|██████▌   | 2058/3123 [4:31:35<2:19:57,  7.88s/it]
 66%|██████▌   | 2059/3123 [4:31:42<2:19:10,  7.85s/it]
 66%|██████▌   | 2060/3123 [4:31:50<2:18:35,  7.82s/it]
                                                       
{'loss': '0.001284', 'grad_norm': '0.01318', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '86.39', 'tokens/total': 33761280, 'tokens/trainable': 10575622, 'epoch': '1.978'}
+
 66%|██████▌   | 2060/3123 [4:31:50<2:18:35,  7.82s/it]
 66%|██████▌   | 2061/3123 [4:31:58<2:18:08,  7.81s/it]
 66%|██████▌   | 2062/3123 [4:32:06<2:19:23,  7.88s/it]
 66%|██████▌   | 2063/3123 [4:32:14<2:18:36,  7.85s/it]
 66%|██████▌   | 2064/3123 [4:32:21<2:18:01,  7.82s/it]
 66%|██████▌   | 2065/3123 [4:32:29<2:19:09,  7.89s/it]
 66%|██████▌   | 2066/3123 [4:32:37<2:18:20,  7.85s/it]
 66%|██████▌   | 2067/3123 [4:32:45<2:17:42,  7.82s/it]
 66%|██████▌   | 2068/3123 [4:32:53<2:17:15,  7.81s/it]
 66%|██████▋   | 2069/3123 [4:33:01<2:18:28,  7.88s/it]
 66%|██████▋   | 2070/3123 [4:33:09<2:17:41,  7.85s/it]
                                                       
{'loss': '0.001102', 'grad_norm': '0.01135', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '95.7', 'tokens/total': 33925120, 'tokens/trainable': 10627503, 'epoch': '1.988'}
+
 66%|██████▋   | 2070/3123 [4:33:09<2:17:41,  7.85s/it]
 66%|██████▋   | 2071/3123 [4:33:16<2:17:06,  7.82s/it]
 66%|██████▋   | 2072/3123 [4:33:24<2:18:12,  7.89s/it]
 66%|██████▋   | 2073/3123 [4:33:32<2:17:23,  7.85s/it]
 66%|██████▋   | 2074/3123 [4:33:40<2:16:47,  7.82s/it]
 66%|██████▋   | 2075/3123 [4:33:48<2:16:16,  7.80s/it]
 66%|██████▋   | 2076/3123 [4:33:56<2:17:28,  7.88s/it]
 67%|██████▋   | 2077/3123 [4:34:04<2:16:45,  7.84s/it]
 67%|██████▋   | 2078/3123 [4:34:11<2:16:11,  7.82s/it]
 67%|██████▋   | 2079/3123 [4:34:19<2:17:15,  7.89s/it]
 67%|██████▋   | 2080/3123 [4:34:27<2:16:27,  7.85s/it]
                                                       
{'loss': '0.0009905', 'grad_norm': '0.1631', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '85.09', 'tokens/total': 34088960, 'tokens/trainable': 10679027, 'epoch': '1.997'}
+
 67%|██████▋   | 2080/3123 [4:34:27<2:16:27,  7.85s/it]
 67%|██████▋   | 2081/3123 [4:34:35<2:15:55,  7.83s/it]
 67%|██████▋   | 2082/3123 [4:34:43<2:15:26,  7.81s/it][2026-03-08 20:31:57,088] [INFO] [axolotl.core.trainers.base._save:721] [PID:1659682] Saving model checkpoint to /home/jiaruil5/math_rl/mix_teachers/r3lit_rl/mix_teachers/checkpoints/math_operations/primitive_atomic_full_sft_50k_lr1e4_t20260308/checkpoint-2082
+
+
Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]
+
Writing model shards: 100%|██████████| 1/1 [00:22<00:00, 22.70s/it]
Writing model shards: 100%|██████████| 1/1 [00:22<00:00, 22.70s/it]Traceback (most recent call last):
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/multiprocessing/util.py", line 300, in _run_finalizers
+    finalizer()
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/multiprocessing/util.py", line 224, in __call__
+    res = self._callback(*self._args, **self._kwargs)
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/multiprocessing/util.py", line 133, in _remove_temp_dir
+    rmtree(tempdir)
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/shutil.py", line 725, in rmtree
+    _rmtree_safe_fd(fd, path, onerror)
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/shutil.py", line 681, in _rmtree_safe_fd
+    onerror(os.unlink, fullname, sys.exc_info())
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/shutil.py", line 679, in _rmtree_safe_fd
+    os.unlink(entry.name, dir_fd=topfd)
+OSError: [Errno 16] Device or resource busy: '.nfs00000000000dabb60001250d'
+
+
 67%|██████▋   | 2083/3123 [4:35:55<7:52:50, 27.28s/it]
 67%|██████▋   | 2084/3123 [4:36:05<6:22:36, 22.09s/it][2026-03-08 20:33:19,772] [INFO] [axolotl.core.trainers.base.evaluate:400] [PID:1659682] Running evaluation step...
+[2026-03-08 20:33:20,766] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:1659682] generate_batches time: 0.4665355682373047
+[2026-03-08 20:33:21,236] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:1659682] generate_batches time: 0.46981167793273926
+[2026-03-08 20:33:21,710] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:1659682] generate_batches time: 0.47421741485595703
+[2026-03-08 20:33:22,181] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:1659682] generate_batches time: 0.4700648784637451
+[2026-03-08 20:33:22,181] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:1659682] gather_len_batches: [34]
+
+
  0%|          | 0/34 [00:00<?, ?it/s]
+
  6%|▌         | 2/34 [00:00<00:03,  8.58it/s]
+
  9%|▉         | 3/34 [00:00<00:05,  6.01it/s]
+
 12%|█▏        | 4/34 [00:00<00:05,  5.20it/s]
+
 15%|█▍        | 5/34 [00:00<00:06,  4.82it/s]
+
 18%|█▊        | 6/34 [00:01<00:06,  4.61it/s]
+
 21%|██        | 7/34 [00:01<00:06,  4.48it/s]
+
 24%|██▎       | 8/34 [00:01<00:05,  4.40it/s]
+
 26%|██▋       | 9/34 [00:01<00:06,  3.80it/s]
+
 29%|██▉       | 10/34 [00:02<00:06,  3.98it/s]
+
 32%|███▏      | 11/34 [00:02<00:05,  4.05it/s]
+
 35%|███▌      | 12/34 [00:02<00:05,  4.10it/s]
+
 38%|███▊      | 13/34 [00:02<00:05,  4.14it/s]
+
 41%|████      | 14/34 [00:03<00:04,  4.18it/s]
+
 44%|████▍     | 15/34 [00:03<00:04,  4.20it/s]
+
 47%|████▋     | 16/34 [00:03<00:04,  4.21it/s]
+
 50%|█████     | 17/34 [00:03<00:04,  3.98it/s]
+
 53%|█████▎    | 18/34 [00:04<00:03,  4.09it/s]
+
 56%|█████▌    | 19/34 [00:04<00:03,  4.14it/s]
+
 59%|█████▉    | 20/34 [00:04<00:03,  4.17it/s]
+
 62%|██████▏   | 21/34 [00:04<00:03,  4.19it/s]
+
 65%|██████▍   | 22/34 [00:05<00:02,  4.20it/s]
+
 68%|██████▊   | 23/34 [00:05<00:02,  4.22it/s]
+
 71%|███████   | 24/34 [00:05<00:02,  4.23it/s]
+
 74%|███████▎  | 25/34 [00:05<00:02,  4.01it/s]
+
 76%|███████▋  | 26/34 [00:06<00:01,  4.13it/s]
+
 79%|███████▉  | 27/34 [00:06<00:01,  4.16it/s]
+
 82%|████████▏ | 28/34 [00:06<00:01,  4.18it/s]
+
 85%|████████▌ | 29/34 [00:06<00:01,  4.19it/s]
+
 88%|████████▊ | 30/34 [00:07<00:00,  4.20it/s]
+
 91%|█████████ | 31/34 [00:07<00:00,  4.21it/s]
+
 94%|█████████▍| 32/34 [00:07<00:00,  4.22it/s]
+
 97%|█████████▋| 33/34 [00:07<00:00,  4.10it/s]Traceback (most recent call last):
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/multiprocessing/util.py", line 300, in _run_finalizers
+    finalizer()
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/multiprocessing/util.py", line 224, in __call__
+    res = self._callback(*self._args, **self._kwargs)
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/multiprocessing/util.py", line 133, in _remove_temp_dir
+    rmtree(tempdir)
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/shutil.py", line 725, in rmtree
+    _rmtree_safe_fd(fd, path, onerror)
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/shutil.py", line 681, in _rmtree_safe_fd
+    onerror(os.unlink, fullname, sys.exc_info())
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/shutil.py", line 679, in _rmtree_safe_fd
+    os.unlink(entry.name, dir_fd=topfd)
+OSError: [Errno 16] Device or resource busy: '.nfs00000000000db21c0001250e'
+
+
100%|██████████| 34/34 [00:08<00:00,  3.94it/s]
                                                       
+
                                               
{'eval_loss': '0.0004862', 'eval_runtime': '8.924', 'eval_samples_per_second': '22.41', 'eval_steps_per_second': '22.41', 'eval_ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'epoch': '2.001', 'tokens/train_per_sec_per_gpu': '94.03'}
+
 67%|██████▋   | 2084/3123 [4:36:17<6:22:36, 22.09s/it]
+
100%|██████████| 34/34 [00:08<00:00,  3.94it/s]
+
                                               
 67%|██████▋   | 2085/3123 [4:36:24<6:05:02, 21.10s/it]
 67%|██████▋   | 2086/3123 [4:36:32<4:55:14, 17.08s/it]
 67%|██████▋   | 2087/3123 [4:36:40<4:06:30, 14.28s/it]
 67%|██████▋   | 2088/3123 [4:36:48<3:34:07, 12.41s/it]
 67%|██████▋   | 2089/3123 [4:36:55<3:09:47, 11.01s/it]
 67%|██████▋   | 2090/3123 [4:37:03<2:52:46, 10.04s/it]
                                                       
{'loss': '0.0009286', 'grad_norm': '0.123', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.96', 'memory/max_allocated (GiB)': '33.96', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '79.63', 'tokens/total': 34248704, 'tokens/trainable': 10729151, 'epoch': '2.007'}
+
 67%|██████▋   | 2090/3123 [4:37:03<2:52:46, 10.04s/it]
 67%|██████▋   | 2091/3123 [4:37:11<2:42:19,  9.44s/it]
 67%|██████▋   | 2092/3123 [4:37:19<2:33:28,  8.93s/it]
 67%|██████▋   | 2093/3123 [4:37:27<2:27:17,  8.58s/it]
 67%|██████▋   | 2094/3123 [4:37:34<2:22:56,  8.33s/it]
 67%|██████▋   | 2095/3123 [4:37:42<2:21:24,  8.25s/it]
 67%|██████▋   | 2096/3123 [4:37:50<2:18:46,  8.11s/it]
 67%|██████▋   | 2097/3123 [4:37:58<2:16:52,  8.00s/it]
 67%|██████▋   | 2098/3123 [4:38:06<2:17:02,  8.02s/it]
 67%|██████▋   | 2099/3123 [4:38:14<2:15:35,  7.94s/it]
 67%|██████▋   | 2100/3123 [4:38:22<2:14:33,  7.89s/it]
                                                       
{'loss': '0.000985', 'grad_norm': '0.01562', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '76.93', 'tokens/total': 34412544, 'tokens/trainable': 10780412, 'epoch': '2.016'}
+
 67%|██████▋   | 2100/3123 [4:38:22<2:14:33,  7.89s/it]
 67%|██████▋   | 2101/3123 [4:38:29<2:13:46,  7.85s/it]
 67%|██████▋   | 2102/3123 [4:38:37<2:14:45,  7.92s/it]
 67%|██████▋   | 2103/3123 [4:38:45<2:13:49,  7.87s/it]
 67%|██████▋   | 2104/3123 [4:38:53<2:13:10,  7.84s/it]
 67%|██████▋   | 2105/3123 [4:39:01<2:14:08,  7.91s/it]
 67%|██████▋   | 2106/3123 [4:39:09<2:13:17,  7.86s/it]
 67%|██████▋   | 2107/3123 [4:39:17<2:12:41,  7.84s/it]
 67%|██████▋   | 2108/3123 [4:39:24<2:12:14,  7.82s/it]
 68%|██████▊   | 2109/3123 [4:39:32<2:13:22,  7.89s/it]
 68%|██████▊   | 2110/3123 [4:39:40<2:12:34,  7.85s/it]
                                                       
{'loss': '0.0007462', 'grad_norm': '0.08594', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '85.46', 'tokens/total': 34576384, 'tokens/trainable': 10832231, 'epoch': '2.026'}
+
 68%|██████▊   | 2110/3123 [4:39:40<2:12:34,  7.85s/it]
 68%|██████▊   | 2111/3123 [4:39:48<2:12:00,  7.83s/it]
 68%|██████▊   | 2112/3123 [4:39:56<2:13:05,  7.90s/it]
 68%|██████▊   | 2113/3123 [4:40:04<2:12:15,  7.86s/it]
 68%|██████▊   | 2114/3123 [4:40:12<2:11:40,  7.83s/it]
 68%|██████▊   | 2115/3123 [4:40:19<2:11:13,  7.81s/it]
 68%|██████▊   | 2116/3123 [4:40:27<2:12:22,  7.89s/it]
 68%|██████▊   | 2117/3123 [4:40:35<2:11:38,  7.85s/it]
 68%|██████▊   | 2118/3123 [4:40:43<2:11:05,  7.83s/it]
 68%|██████▊   | 2119/3123 [4:40:51<2:12:05,  7.89s/it]
 68%|██████▊   | 2120/3123 [4:40:59<2:11:18,  7.86s/it]
                                                       
{'loss': '0.002171', 'grad_norm': '0.1216', 'learning_rate': '0.0001', 'ppl': '1.002', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '88.09', 'tokens/total': 34740224, 'tokens/trainable': 10883673, 'epoch': '2.036'}
+
 68%|██████▊   | 2120/3123 [4:40:59<2:11:18,  7.86s/it]
 68%|██████▊   | 2121/3123 [4:41:06<2:10:46,  7.83s/it]
 68%|██████▊   | 2122/3123 [4:41:14<2:10:19,  7.81s/it]
 68%|██████▊   | 2123/3123 [4:41:22<2:11:33,  7.89s/it]
 68%|██████▊   | 2124/3123 [4:41:30<2:10:48,  7.86s/it]
 68%|██████▊   | 2125/3123 [4:41:38<2:10:13,  7.83s/it]
 68%|██████▊   | 2126/3123 [4:41:46<2:11:13,  7.90s/it]
 68%|██████▊   | 2127/3123 [4:41:54<2:10:27,  7.86s/it]
 68%|██████▊   | 2128/3123 [4:42:01<2:09:52,  7.83s/it]
 68%|██████▊   | 2129/3123 [4:42:09<2:09:29,  7.82s/it]
 68%|██████▊   | 2130/3123 [4:42:17<2:10:38,  7.89s/it]
                                                       
{'loss': '0.002085', 'grad_norm': '0.1484', 'learning_rate': '0.0001', 'ppl': '1.002', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '76.97', 'tokens/total': 34904064, 'tokens/trainable': 10935305, 'epoch': '2.045'}
+
 68%|██████▊   | 2130/3123 [4:42:18<2:10:38,  7.89s/it]
 68%|██████▊   | 2131/3123 [4:42:25<2:09:56,  7.86s/it]
 68%|██████▊   | 2132/3123 [4:42:33<2:09:19,  7.83s/it]
 68%|██████▊   | 2133/3123 [4:42:41<2:10:28,  7.91s/it]
 68%|██████▊   | 2134/3123 [4:42:49<2:09:39,  7.87s/it]
 68%|██████▊   | 2135/3123 [4:42:56<2:09:03,  7.84s/it]
 68%|██████▊   | 2136/3123 [4:43:04<2:08:37,  7.82s/it]
 68%|██████▊   | 2137/3123 [4:43:12<2:09:45,  7.90s/it]
 68%|██████▊   | 2138/3123 [4:43:20<2:09:00,  7.86s/it]
 68%|██████▊   | 2139/3123 [4:43:28<2:08:26,  7.83s/it]
 69%|██████▊   | 2140/3123 [4:43:36<2:09:23,  7.90s/it]
                                                       
{'loss': '0.00155', 'grad_norm': '0.1494', 'learning_rate': '0.0001', 'ppl': '1.002', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '79.16', 'tokens/total': 35067904, 'tokens/trainable': 10986544, 'epoch': '2.055'}
+
 69%|██████▊   | 2140/3123 [4:43:36<2:09:23,  7.90s/it]
 69%|██████▊   | 2141/3123 [4:43:44<2:08:34,  7.86s/it]
 69%|██████▊   | 2142/3123 [4:43:51<2:07:58,  7.83s/it]
 69%|██████▊   | 2143/3123 [4:44:00<2:09:04,  7.90s/it]
 69%|██████▊   | 2144/3123 [4:44:07<2:08:17,  7.86s/it]
 69%|██████▊   | 2145/3123 [4:44:15<2:07:42,  7.83s/it]
 69%|██████▊   | 2146/3123 [4:44:23<2:07:12,  7.81s/it]
 69%|██████▊   | 2147/3123 [4:44:31<2:08:14,  7.88s/it]
 69%|██████▉   | 2148/3123 [4:44:39<2:07:27,  7.84s/it]
 69%|██████▉   | 2149/3123 [4:44:46<2:06:56,  7.82s/it]
 69%|██████▉   | 2150/3123 [4:44:54<2:07:59,  7.89s/it]
                                                       
{'loss': '0.001194', 'grad_norm': '0.1162', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '76.72', 'tokens/total': 35231744, 'tokens/trainable': 11037389, 'epoch': '2.064'}
+
 69%|██████▉   | 2150/3123 [4:44:55<2:07:59,  7.89s/it]
 69%|██████▉   | 2151/3123 [4:45:02<2:07:16,  7.86s/it]
 69%|██████▉   | 2152/3123 [4:45:10<2:06:41,  7.83s/it]
 69%|██████▉   | 2153/3123 [4:45:18<2:06:14,  7.81s/it]
 69%|██████▉   | 2154/3123 [4:45:26<2:07:16,  7.88s/it]
 69%|██████▉   | 2155/3123 [4:45:34<2:06:32,  7.84s/it]
 69%|██████▉   | 2156/3123 [4:45:41<2:05:58,  7.82s/it]
 69%|██████▉   | 2157/3123 [4:45:49<2:06:58,  7.89s/it]
 69%|██████▉   | 2158/3123 [4:45:57<2:06:13,  7.85s/it]
 69%|██████▉   | 2159/3123 [4:46:05<2:05:41,  7.82s/it]
 69%|██████▉   | 2160/3123 [4:46:13<2:05:14,  7.80s/it]
                                                       
{'loss': '0.00172', 'grad_norm': '0.1104', 'learning_rate': '0.0001', 'ppl': '1.002', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '91.15', 'tokens/total': 35395584, 'tokens/trainable': 11088661, 'epoch': '2.074'}
+
 69%|██████▉   | 2160/3123 [4:46:13<2:05:14,  7.80s/it]
 69%|██████▉   | 2161/3123 [4:46:21<2:06:20,  7.88s/it]
 69%|██████▉   | 2162/3123 [4:46:28<2:05:37,  7.84s/it]
 69%|██████▉   | 2163/3123 [4:46:36<2:05:05,  7.82s/it]
 69%|██████▉   | 2164/3123 [4:46:44<2:06:01,  7.88s/it]
 69%|██████▉   | 2165/3123 [4:46:52<2:05:19,  7.85s/it]
 69%|██████▉   | 2166/3123 [4:47:00<2:04:45,  7.82s/it]
 69%|██████▉   | 2167/3123 [4:47:08<2:04:22,  7.81s/it]
 69%|██████▉   | 2168/3123 [4:47:16<2:05:29,  7.88s/it]
 69%|██████▉   | 2169/3123 [4:47:23<2:04:47,  7.85s/it]
 69%|██████▉   | 2170/3123 [4:47:31<2:04:14,  7.82s/it]
                                                       
{'loss': '0.0009884', 'grad_norm': '0.1514', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '87.06', 'tokens/total': 35559424, 'tokens/trainable': 11140966, 'epoch': '2.084'}
+
 69%|██████▉   | 2170/3123 [4:47:31<2:04:14,  7.82s/it]
 70%|██████▉   | 2171/3123 [4:47:39<2:05:10,  7.89s/it]
 70%|██████▉   | 2172/3123 [4:47:47<2:04:27,  7.85s/it]
 70%|██████▉   | 2173/3123 [4:47:55<2:03:53,  7.82s/it]
 70%|██████▉   | 2174/3123 [4:48:02<2:03:27,  7.81s/it]
 70%|██████▉   | 2175/3123 [4:48:11<2:04:31,  7.88s/it]
 70%|██████▉   | 2176/3123 [4:48:18<2:03:48,  7.84s/it]
 70%|██████▉   | 2177/3123 [4:48:26<2:03:16,  7.82s/it]
 70%|██████▉   | 2178/3123 [4:48:34<2:04:14,  7.89s/it]
 70%|██████▉   | 2179/3123 [4:48:42<2:03:33,  7.85s/it]
 70%|██████▉   | 2180/3123 [4:48:50<2:03:00,  7.83s/it]
                                                       
{'loss': '0.001412', 'grad_norm': '0.1299', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '81.92', 'tokens/total': 35723264, 'tokens/trainable': 11192705, 'epoch': '2.093'}
+
 70%|██████▉   | 2180/3123 [4:48:50<2:03:00,  7.83s/it]
 70%|██████▉   | 2181/3123 [4:48:57<2:02:36,  7.81s/it]
 70%|██████▉   | 2182/3123 [4:49:05<2:03:40,  7.89s/it]
 70%|██████▉   | 2183/3123 [4:49:13<2:02:57,  7.85s/it]
 70%|██████▉   | 2184/3123 [4:49:21<2:02:24,  7.82s/it]
 70%|██████▉   | 2185/3123 [4:49:29<2:03:20,  7.89s/it]
 70%|██████▉   | 2186/3123 [4:49:37<2:02:36,  7.85s/it]
 70%|███████   | 2187/3123 [4:49:45<2:02:04,  7.83s/it]
 70%|███████   | 2188/3123 [4:49:52<2:01:38,  7.81s/it]
 70%|███████   | 2189/3123 [4:50:00<2:02:40,  7.88s/it]
 70%|███████   | 2190/3123 [4:50:08<2:01:59,  7.85s/it]
                                                       
{'loss': '0.0007082', 'grad_norm': '0.03564', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '79.94', 'tokens/total': 35887104, 'tokens/trainable': 11244150, 'epoch': '2.103'}
+
 70%|███████   | 2190/3123 [4:50:08<2:01:59,  7.85s/it]
 70%|███████   | 2191/3123 [4:50:16<2:01:30,  7.82s/it]
 70%|███████   | 2192/3123 [4:50:24<2:02:24,  7.89s/it]
 70%|███████   | 2193/3123 [4:50:32<2:01:41,  7.85s/it]
 70%|███████   | 2194/3123 [4:50:39<2:01:06,  7.82s/it]
 70%|███████   | 2195/3123 [4:50:47<2:00:43,  7.81s/it]
 70%|███████   | 2196/3123 [4:50:55<2:01:48,  7.88s/it]
 70%|███████   | 2197/3123 [4:51:03<2:01:05,  7.85s/it]
 70%|███████   | 2198/3123 [4:51:11<2:00:33,  7.82s/it]
 70%|███████   | 2199/3123 [4:51:19<2:01:30,  7.89s/it]
 70%|███████   | 2200/3123 [4:51:27<2:00:47,  7.85s/it]
                                                       
{'loss': '0.0005755', 'grad_norm': '0.05713', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '89.56', 'tokens/total': 36050944, 'tokens/trainable': 11295480, 'epoch': '2.112'}
+
 70%|███████   | 2200/3123 [4:51:27<2:00:47,  7.85s/it]
 70%|███████   | 2201/3123 [4:51:34<2:00:17,  7.83s/it]
 71%|███████   | 2202/3123 [4:51:42<1:59:48,  7.81s/it]
 71%|███████   | 2203/3123 [4:51:50<2:00:52,  7.88s/it]
 71%|███████   | 2204/3123 [4:51:58<2:00:09,  7.85s/it]
 71%|███████   | 2205/3123 [4:52:06<1:59:40,  7.82s/it]
 71%|███████   | 2206/3123 [4:52:14<2:00:44,  7.90s/it]
 71%|███████   | 2207/3123 [4:52:22<1:59:58,  7.86s/it]
 71%|███████   | 2208/3123 [4:52:29<1:59:23,  7.83s/it]
 71%|███████   | 2209/3123 [4:52:37<1:58:57,  7.81s/it]
 71%|███████   | 2210/3123 [4:52:45<1:59:56,  7.88s/it]
                                                       
{'loss': '0.000491', 'grad_norm': '0.03662', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '81.05', 'tokens/total': 36214784, 'tokens/trainable': 11346806, 'epoch': '2.122'}
+
 71%|███████   | 2210/3123 [4:52:45<1:59:56,  7.88s/it]
 71%|███████   | 2211/3123 [4:52:53<1:59:18,  7.85s/it]
 71%|███████   | 2212/3123 [4:53:01<1:58:47,  7.82s/it]
 71%|███████   | 2213/3123 [4:53:09<1:59:46,  7.90s/it]
 71%|███████   | 2214/3123 [4:53:17<1:59:02,  7.86s/it]
 71%|███████   | 2215/3123 [4:53:24<1:58:30,  7.83s/it]
 71%|███████   | 2216/3123 [4:53:32<1:58:04,  7.81s/it]
 71%|███████   | 2217/3123 [4:53:40<1:59:04,  7.89s/it]
 71%|███████   | 2218/3123 [4:53:48<1:58:22,  7.85s/it]
 71%|███████   | 2219/3123 [4:53:56<1:57:49,  7.82s/it]
 71%|███████   | 2220/3123 [4:54:04<1:58:41,  7.89s/it]
                                                       
{'loss': '0.0004828', 'grad_norm': '0.06689', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '88.07', 'tokens/total': 36378624, 'tokens/trainable': 11398184, 'epoch': '2.132'}
+
 71%|███████   | 2220/3123 [4:54:04<1:58:41,  7.89s/it]
 71%|███████   | 2221/3123 [4:54:11<1:58:00,  7.85s/it]
 71%|███████   | 2222/3123 [4:54:19<1:57:29,  7.82s/it]
 71%|███████   | 2223/3123 [4:54:27<1:57:04,  7.81s/it]
 71%|███████   | 2224/3123 [4:54:35<1:58:07,  7.88s/it]
 71%|███████   | 2225/3123 [4:54:43<1:57:25,  7.85s/it]
 71%|███████▏  | 2226/3123 [4:54:51<1:56:55,  7.82s/it]
 71%|███████▏  | 2227/3123 [4:54:59<1:57:50,  7.89s/it]
 71%|███████▏  | 2228/3123 [4:55:06<1:57:09,  7.85s/it]
 71%|███████▏  | 2229/3123 [4:55:14<1:56:38,  7.83s/it]
 71%|███████▏  | 2230/3123 [4:55:22<1:56:12,  7.81s/it]
                                                       
{'loss': '0.0004925', 'grad_norm': '0.01337', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '91.5', 'tokens/total': 36542464, 'tokens/trainable': 11448827, 'epoch': '2.141'}
+
 71%|███████▏  | 2230/3123 [4:55:22<1:56:12,  7.81s/it]
 71%|███████▏  | 2231/3123 [4:55:30<1:57:08,  7.88s/it]
 71%|███████▏  | 2232/3123 [4:55:38<1:56:28,  7.84s/it]
 72%|███████▏  | 2233/3123 [4:55:46<1:56:01,  7.82s/it]
 72%|███████▏  | 2234/3123 [4:55:54<1:56:58,  7.90s/it]
 72%|███████▏  | 2235/3123 [4:56:01<1:56:13,  7.85s/it]
 72%|███████▏  | 2236/3123 [4:56:09<1:55:41,  7.83s/it]
 72%|███████▏  | 2237/3123 [4:56:17<1:56:33,  7.89s/it]
 72%|███████▏  | 2238/3123 [4:56:25<1:55:51,  7.85s/it]
 72%|███████▏  | 2239/3123 [4:56:33<1:55:18,  7.83s/it]
 72%|███████▏  | 2240/3123 [4:56:40<1:54:55,  7.81s/it]
                                                       
{'loss': '0.0003488', 'grad_norm': '0.04077', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '98.63', 'tokens/total': 36706304, 'tokens/trainable': 11500688, 'epoch': '2.151'}
+
 72%|███████▏  | 2240/3123 [4:56:41<1:54:55,  7.81s/it]
 72%|███████▏  | 2241/3123 [4:56:48<1:55:48,  7.88s/it]
 72%|███████▏  | 2242/3123 [4:56:56<1:55:09,  7.84s/it]
 72%|███████▏  | 2243/3123 [4:57:04<1:54:40,  7.82s/it]
 72%|███████▏  | 2244/3123 [4:57:12<1:55:36,  7.89s/it]
 72%|███████▏  | 2245/3123 [4:57:20<1:54:53,  7.85s/it]
 72%|███████▏  | 2246/3123 [4:57:28<1:54:21,  7.82s/it]
 72%|███████▏  | 2247/3123 [4:57:35<1:53:57,  7.81s/it]
 72%|███████▏  | 2248/3123 [4:57:43<1:54:57,  7.88s/it]
 72%|███████▏  | 2249/3123 [4:57:51<1:54:15,  7.84s/it]
 72%|███████▏  | 2250/3123 [4:57:59<1:53:46,  7.82s/it]
                                                       
{'loss': '0.000749', 'grad_norm': '0.02246', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '76.44', 'tokens/total': 36870144, 'tokens/trainable': 11551102, 'epoch': '2.16'}
+
 72%|███████▏  | 2250/3123 [4:57:59<1:53:46,  7.82s/it]
 72%|███████▏  | 2251/3123 [4:58:07<1:54:37,  7.89s/it]
 72%|███████▏  | 2252/3123 [4:58:15<1:53:56,  7.85s/it]
 72%|███████▏  | 2253/3123 [4:58:22<1:53:27,  7.82s/it]
 72%|███████▏  | 2254/3123 [4:58:30<1:53:05,  7.81s/it]
 72%|███████▏  | 2255/3123 [4:58:38<1:54:04,  7.89s/it]
 72%|███████▏  | 2256/3123 [4:58:46<1:53:28,  7.85s/it]
 72%|███████▏  | 2257/3123 [4:58:54<1:52:58,  7.83s/it]
 72%|███████▏  | 2258/3123 [4:59:02<1:53:45,  7.89s/it]
 72%|███████▏  | 2259/3123 [4:59:10<1:53:06,  7.86s/it]
 72%|███████▏  | 2260/3123 [4:59:17<1:52:36,  7.83s/it]
                                                       
{'loss': '0.0006475', 'grad_norm': '0.04224', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '74.26', 'tokens/total': 37033984, 'tokens/trainable': 11602924, 'epoch': '2.17'}
+
 72%|███████▏  | 2260/3123 [4:59:18<1:52:36,  7.83s/it]
 72%|███████▏  | 2261/3123 [4:59:25<1:52:14,  7.81s/it]
 72%|███████▏  | 2262/3123 [4:59:33<1:53:11,  7.89s/it]
 72%|███████▏  | 2263/3123 [4:59:41<1:52:34,  7.85s/it]
 72%|███████▏  | 2264/3123 [4:59:49<1:52:03,  7.83s/it]
 73%|███████▎  | 2265/3123 [4:59:57<1:52:58,  7.90s/it]
 73%|███████▎  | 2266/3123 [5:00:05<1:52:16,  7.86s/it]
 73%|███████▎  | 2267/3123 [5:00:12<1:51:48,  7.84s/it]
 73%|███████▎  | 2268/3123 [5:00:21<1:52:41,  7.91s/it]
 73%|███████▎  | 2269/3123 [5:00:28<1:51:55,  7.86s/it]
 73%|███████▎  | 2270/3123 [5:00:36<1:51:22,  7.83s/it]
                                                       
{'loss': '0.0004128', 'grad_norm': '0.07031', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '83.81', 'tokens/total': 37197824, 'tokens/trainable': 11654167, 'epoch': '2.18'}
+
 73%|███████▎  | 2270/3123 [5:00:36<1:51:22,  7.83s/it]
 73%|███████▎  | 2271/3123 [5:00:44<1:50:57,  7.81s/it]
 73%|███████▎  | 2272/3123 [5:00:52<1:51:54,  7.89s/it]
 73%|███████▎  | 2273/3123 [5:01:00<1:51:15,  7.85s/it]
 73%|███████▎  | 2274/3123 [5:01:07<1:50:46,  7.83s/it]
 73%|███████▎  | 2275/3123 [5:01:15<1:51:40,  7.90s/it]
 73%|███████▎  | 2276/3123 [5:01:23<1:50:59,  7.86s/it]
 73%|███████▎  | 2277/3123 [5:01:31<1:50:29,  7.84s/it]
 73%|███████▎  | 2278/3123 [5:01:39<1:50:05,  7.82s/it]
 73%|███████▎  | 2279/3123 [5:01:47<1:50:59,  7.89s/it]
 73%|███████▎  | 2280/3123 [5:01:55<1:50:21,  7.85s/it]
                                                       
{'loss': '0.0002948', 'grad_norm': '0.07959', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '80.26', 'tokens/total': 37361664, 'tokens/trainable': 11705226, 'epoch': '2.189'}
+
 73%|███████▎  | 2280/3123 [5:01:55<1:50:21,  7.85s/it]
 73%|███████▎  | 2281/3123 [5:02:02<1:49:53,  7.83s/it]
 73%|███████▎  | 2282/3123 [5:02:10<1:50:46,  7.90s/it]
 73%|███████▎  | 2283/3123 [5:02:18<1:50:05,  7.86s/it]
 73%|███████▎  | 2284/3123 [5:02:26<1:49:35,  7.84s/it]
 73%|███████▎  | 2285/3123 [5:02:34<1:49:10,  7.82s/it]
 73%|███████▎  | 2286/3123 [5:02:42<1:50:07,  7.89s/it]
 73%|███████▎  | 2287/3123 [5:02:50<1:49:28,  7.86s/it]
 73%|███████▎  | 2288/3123 [5:02:57<1:49:00,  7.83s/it]
 73%|███████▎  | 2289/3123 [5:03:05<1:49:51,  7.90s/it]
 73%|███████▎  | 2290/3123 [5:03:13<1:49:09,  7.86s/it]
                                                       
{'loss': '0.0002662', 'grad_norm': '0.05493', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '83.92', 'tokens/total': 37525504, 'tokens/trainable': 11755550, 'epoch': '2.199'}
+
 73%|███████▎  | 2290/3123 [5:03:14<1:49:09,  7.86s/it]
 73%|███████▎  | 2291/3123 [5:03:21<1:48:40,  7.84s/it]
 73%|███████▎  | 2292/3123 [5:03:29<1:48:18,  7.82s/it]
 73%|███████▎  | 2293/3123 [5:03:37<1:49:22,  7.91s/it]
 73%|███████▎  | 2294/3123 [5:03:45<1:48:41,  7.87s/it]
 73%|███████▎  | 2295/3123 [5:03:52<1:48:11,  7.84s/it]
 74%|███████▎  | 2296/3123 [5:04:01<1:49:01,  7.91s/it]
 74%|███████▎  | 2297/3123 [5:04:08<1:48:22,  7.87s/it]
 74%|███████▎  | 2298/3123 [5:04:16<1:47:51,  7.84s/it]
 74%|███████▎  | 2299/3123 [5:04:24<1:47:24,  7.82s/it]
 74%|███████▎  | 2300/3123 [5:04:32<1:48:22,  7.90s/it]
                                                       
{'loss': '0.0002991', 'grad_norm': '0.03662', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '93.97', 'tokens/total': 37689344, 'tokens/trainable': 11807401, 'epoch': '2.208'}
+
 74%|███████▎  | 2300/3123 [5:04:32<1:48:22,  7.90s/it]
 74%|███████▎  | 2301/3123 [5:04:40<1:47:42,  7.86s/it]
 74%|███████▎  | 2302/3123 [5:04:48<1:47:09,  7.83s/it]
 74%|███████▎  | 2303/3123 [5:04:56<1:48:02,  7.91s/it]
 74%|███████▍  | 2304/3123 [5:05:03<1:47:20,  7.86s/it]
 74%|███████▍  | 2305/3123 [5:05:11<1:46:49,  7.84s/it]
 74%|███████▍  | 2306/3123 [5:05:19<1:46:25,  7.82s/it]
 74%|███████▍  | 2307/3123 [5:05:27<1:47:13,  7.88s/it]
 74%|███████▍  | 2308/3123 [5:05:35<1:46:36,  7.85s/it]
 74%|███████▍  | 2309/3123 [5:05:42<1:46:08,  7.82s/it]
 74%|███████▍  | 2310/3123 [5:05:51<1:46:56,  7.89s/it]
                                                       
{'loss': '0.0002654', 'grad_norm': '0.04126', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '72.15', 'tokens/total': 37853184, 'tokens/trainable': 11858361, 'epoch': '2.218'}
+
 74%|███████▍  | 2310/3123 [5:05:51<1:46:56,  7.89s/it]
 74%|███████▍  | 2311/3123 [5:05:58<1:46:18,  7.85s/it]
 74%|███████▍  | 2312/3123 [5:06:06<1:45:48,  7.83s/it]
 74%|███████▍  | 2313/3123 [5:06:14<1:45:24,  7.81s/it]
 74%|███████▍  | 2314/3123 [5:06:22<1:46:20,  7.89s/it]
 74%|███████▍  | 2315/3123 [5:06:30<1:45:42,  7.85s/it]
 74%|███████▍  | 2316/3123 [5:06:37<1:45:11,  7.82s/it]
 74%|███████▍  | 2317/3123 [5:06:45<1:45:58,  7.89s/it]
 74%|███████▍  | 2318/3123 [5:06:53<1:45:18,  7.85s/it]
 74%|███████▍  | 2319/3123 [5:07:01<1:44:49,  7.82s/it]
 74%|███████▍  | 2320/3123 [5:07:09<1:44:27,  7.81s/it]
                                                       
{'loss': '0.0003397', 'grad_norm': '0.02881', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '85.8', 'tokens/total': 38017024, 'tokens/trainable': 11910568, 'epoch': '2.228'}
+
 74%|███████▍  | 2320/3123 [5:07:09<1:44:27,  7.81s/it]
 74%|███████▍  | 2321/3123 [5:07:17<1:45:21,  7.88s/it]
 74%|███████▍  | 2322/3123 [5:07:25<1:44:43,  7.84s/it]
 74%|███████▍  | 2323/3123 [5:07:32<1:44:16,  7.82s/it]
 74%|███████▍  | 2324/3123 [5:07:40<1:45:07,  7.89s/it]
 74%|███████▍  | 2325/3123 [5:07:48<1:44:26,  7.85s/it]
 74%|███████▍  | 2326/3123 [5:07:56<1:43:55,  7.82s/it]
 75%|███████▍  | 2327/3123 [5:08:04<1:44:42,  7.89s/it]
 75%|███████▍  | 2328/3123 [5:08:12<1:44:03,  7.85s/it]
 75%|███████▍  | 2329/3123 [5:08:19<1:43:31,  7.82s/it]
 75%|███████▍  | 2330/3123 [5:08:27<1:43:08,  7.80s/it]
                                                       
{'loss': '0.0004237', 'grad_norm': '0.1328', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '91.26', 'tokens/total': 38180864, 'tokens/trainable': 11961958, 'epoch': '2.237'}
+
 75%|███████▍  | 2330/3123 [5:08:28<1:43:08,  7.80s/it]
 75%|███████▍  | 2331/3123 [5:08:35<1:44:03,  7.88s/it]
 75%|███████▍  | 2332/3123 [5:08:43<1:43:25,  7.85s/it]
 75%|███████▍  | 2333/3123 [5:08:51<1:42:56,  7.82s/it]
 75%|███████▍  | 2334/3123 [5:08:59<1:43:44,  7.89s/it]
 75%|███████▍  | 2335/3123 [5:09:07<1:43:04,  7.85s/it]
 75%|███████▍  | 2336/3123 [5:09:14<1:42:34,  7.82s/it]
 75%|███████▍  | 2337/3123 [5:09:22<1:42:12,  7.80s/it]
 75%|███████▍  | 2338/3123 [5:09:30<1:43:01,  7.87s/it]
 75%|███████▍  | 2339/3123 [5:09:38<1:42:25,  7.84s/it]
 75%|███████▍  | 2340/3123 [5:09:46<1:41:57,  7.81s/it]
                                                       
{'loss': '0.0002397', 'grad_norm': '0.008728', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '82.19', 'tokens/total': 38344704, 'tokens/trainable': 12013115, 'epoch': '2.247'}
+
 75%|███████▍  | 2340/3123 [5:09:46<1:41:57,  7.81s/it]
 75%|███████▍  | 2341/3123 [5:09:54<1:42:49,  7.89s/it]
 75%|███████▍  | 2342/3123 [5:10:01<1:42:10,  7.85s/it]
 75%|███████▌  | 2343/3123 [5:10:09<1:41:39,  7.82s/it]
 75%|███████▌  | 2344/3123 [5:10:17<1:41:16,  7.80s/it]
 75%|███████▌  | 2345/3123 [5:10:25<1:42:08,  7.88s/it]
 75%|███████▌  | 2346/3123 [5:10:33<1:41:33,  7.84s/it]
 75%|███████▌  | 2347/3123 [5:10:41<1:41:06,  7.82s/it]
 75%|███████▌  | 2348/3123 [5:10:49<1:41:54,  7.89s/it]
 75%|███████▌  | 2349/3123 [5:10:56<1:41:15,  7.85s/it]
 75%|███████▌  | 2350/3123 [5:11:04<1:40:45,  7.82s/it]
                                                       
{'loss': '0.0004939', 'grad_norm': '0.1172', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '83.66', 'tokens/total': 38508544, 'tokens/trainable': 12064956, 'epoch': '2.256'}
+
 75%|███████▌  | 2350/3123 [5:11:04<1:40:45,  7.82s/it]
 75%|███████▌  | 2351/3123 [5:11:12<1:40:22,  7.80s/it]
 75%|███████▌  | 2352/3123 [5:11:20<1:41:08,  7.87s/it]
 75%|███████▌  | 2353/3123 [5:11:28<1:40:34,  7.84s/it]
 75%|███████▌  | 2354/3123 [5:11:35<1:40:08,  7.81s/it]
 75%|███████▌  | 2355/3123 [5:11:43<1:40:56,  7.89s/it]
 75%|███████▌  | 2356/3123 [5:11:51<1:40:18,  7.85s/it]
 75%|███████▌  | 2357/3123 [5:11:59<1:39:51,  7.82s/it]
 76%|███████▌  | 2358/3123 [5:12:07<1:40:35,  7.89s/it]
 76%|███████▌  | 2359/3123 [5:12:15<1:39:58,  7.85s/it]
 76%|███████▌  | 2360/3123 [5:12:23<1:39:29,  7.82s/it]
                                                       
{'loss': '0.0003551', 'grad_norm': '0.008423', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '76.48', 'tokens/total': 38672384, 'tokens/trainable': 12116179, 'epoch': '2.266'}
+
 76%|███████▌  | 2360/3123 [5:12:23<1:39:29,  7.82s/it]
 76%|███████▌  | 2361/3123 [5:12:30<1:39:05,  7.80s/it]
 76%|███████▌  | 2362/3123 [5:12:38<1:39:55,  7.88s/it]
 76%|███████▌  | 2363/3123 [5:12:46<1:39:19,  7.84s/it]
 76%|███████▌  | 2364/3123 [5:12:54<1:38:52,  7.82s/it]
 76%|███████▌  | 2365/3123 [5:13:02<1:39:35,  7.88s/it]
 76%|███████▌  | 2366/3123 [5:13:10<1:38:58,  7.84s/it]
 76%|███████▌  | 2367/3123 [5:13:17<1:38:29,  7.82s/it]
 76%|███████▌  | 2368/3123 [5:13:25<1:38:09,  7.80s/it]
 76%|███████▌  | 2369/3123 [5:13:33<1:38:58,  7.88s/it]
 76%|███████▌  | 2370/3123 [5:13:41<1:38:23,  7.84s/it]
                                                       
{'loss': '0.000306', 'grad_norm': '0.01038', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '98.31', 'tokens/total': 38836224, 'tokens/trainable': 12167853, 'epoch': '2.275'}
+
 76%|███████▌  | 2370/3123 [5:13:41<1:38:23,  7.84s/it]
 76%|███████▌  | 2371/3123 [5:13:49<1:37:57,  7.82s/it]
 76%|███████▌  | 2372/3123 [5:13:57<1:38:40,  7.88s/it]
 76%|███████▌  | 2373/3123 [5:14:05<1:38:03,  7.84s/it]
 76%|███████▌  | 2374/3123 [5:14:12<1:37:36,  7.82s/it]
 76%|███████▌  | 2375/3123 [5:14:20<1:37:14,  7.80s/it]
 76%|███████▌  | 2376/3123 [5:14:28<1:38:06,  7.88s/it]
 76%|███████▌  | 2377/3123 [5:14:36<1:37:30,  7.84s/it]
 76%|███████▌  | 2378/3123 [5:14:44<1:37:05,  7.82s/it]
 76%|███████▌  | 2379/3123 [5:14:52<1:37:49,  7.89s/it]
 76%|███████▌  | 2380/3123 [5:14:59<1:37:11,  7.85s/it]
                                                       
{'loss': '0.000176', 'grad_norm': '0.02515', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '82.28', 'tokens/total': 39000064, 'tokens/trainable': 12219215, 'epoch': '2.285'}
+
 76%|███████▌  | 2380/3123 [5:15:00<1:37:11,  7.85s/it]
 76%|███████▌  | 2381/3123 [5:15:07<1:36:46,  7.83s/it]
 76%|███████▋  | 2382/3123 [5:15:15<1:36:24,  7.81s/it]
 76%|███████▋  | 2383/3123 [5:15:23<1:37:14,  7.88s/it]
 76%|███████▋  | 2384/3123 [5:15:31<1:36:39,  7.85s/it]
 76%|███████▋  | 2385/3123 [5:15:39<1:36:10,  7.82s/it]
 76%|███████▋  | 2386/3123 [5:15:47<1:36:53,  7.89s/it]
 76%|███████▋  | 2387/3123 [5:15:54<1:36:17,  7.85s/it]
 76%|███████▋  | 2388/3123 [5:16:02<1:35:48,  7.82s/it]
 76%|███████▋  | 2389/3123 [5:16:10<1:35:26,  7.80s/it]
 77%|███████▋  | 2390/3123 [5:16:18<1:36:16,  7.88s/it]
                                                       
{'loss': '0.0002824', 'grad_norm': '0.02368', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '79.69', 'tokens/total': 39163904, 'tokens/trainable': 12270536, 'epoch': '2.295'}
+
 77%|███████▋  | 2390/3123 [5:16:18<1:36:16,  7.88s/it]
 77%|███████▋  | 2391/3123 [5:16:26<1:35:42,  7.85s/it]
 77%|███████▋  | 2392/3123 [5:16:34<1:35:15,  7.82s/it]
 77%|███████▋  | 2393/3123 [5:16:42<1:35:55,  7.88s/it]
 77%|███████▋  | 2394/3123 [5:16:49<1:35:20,  7.85s/it]
 77%|███████▋  | 2395/3123 [5:16:57<1:34:54,  7.82s/it]
 77%|███████▋  | 2396/3123 [5:17:05<1:34:32,  7.80s/it]
 77%|███████▋  | 2397/3123 [5:17:13<1:35:22,  7.88s/it]
 77%|███████▋  | 2398/3123 [5:17:21<1:34:46,  7.84s/it]
 77%|███████▋  | 2399/3123 [5:17:28<1:34:20,  7.82s/it]
 77%|███████▋  | 2400/3123 [5:17:36<1:35:03,  7.89s/it]
                                                       
{'loss': '0.0001884', 'grad_norm': '0.04858', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '92.93', 'tokens/total': 39327744, 'tokens/trainable': 12321708, 'epoch': '2.304'}
+
 77%|███████▋  | 2400/3123 [5:17:37<1:35:03,  7.89s/it]
 77%|███████▋  | 2401/3123 [5:17:44<1:34:28,  7.85s/it]
 77%|███████▋  | 2402/3123 [5:17:52<1:34:00,  7.82s/it]
 77%|███████▋  | 2403/3123 [5:18:00<1:34:46,  7.90s/it]
 77%|███████▋  | 2404/3123 [5:18:08<1:34:08,  7.86s/it]
 77%|███████▋  | 2405/3123 [5:18:16<1:33:40,  7.83s/it]
 77%|███████▋  | 2406/3123 [5:18:23<1:33:19,  7.81s/it]
 77%|███████▋  | 2407/3123 [5:18:31<1:34:03,  7.88s/it]
 77%|███████▋  | 2408/3123 [5:18:39<1:33:30,  7.85s/it]
 77%|███████▋  | 2409/3123 [5:18:47<1:33:04,  7.82s/it]
 77%|███████▋  | 2410/3123 [5:18:55<1:33:49,  7.90s/it]
                                                       
{'loss': '0.0002685', 'grad_norm': '0.0271', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '89.42', 'tokens/total': 39491584, 'tokens/trainable': 12373442, 'epoch': '2.314'}
+
 77%|███████▋  | 2410/3123 [5:18:55<1:33:49,  7.90s/it]
 77%|███████▋  | 2411/3123 [5:19:03<1:33:12,  7.85s/it]
 77%|███████▋  | 2412/3123 [5:19:11<1:32:44,  7.83s/it]
 77%|███████▋  | 2413/3123 [5:19:18<1:32:20,  7.80s/it]
 77%|███████▋  | 2414/3123 [5:19:26<1:33:06,  7.88s/it]
 77%|███████▋  | 2415/3123 [5:19:34<1:32:34,  7.84s/it]
 77%|███████▋  | 2416/3123 [5:19:42<1:32:10,  7.82s/it]
 77%|███████▋  | 2417/3123 [5:19:50<1:32:51,  7.89s/it]
 77%|███████▋  | 2418/3123 [5:19:58<1:32:17,  7.85s/it]
 77%|███████▋  | 2419/3123 [5:20:05<1:31:50,  7.83s/it]
 77%|███████▋  | 2420/3123 [5:20:13<1:31:29,  7.81s/it]
                                                       
{'loss': '0.000126', 'grad_norm': '0.007233', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '89.97', 'tokens/total': 39655424, 'tokens/trainable': 12424711, 'epoch': '2.323'}
+
 77%|███████▋  | 2420/3123 [5:20:13<1:31:29,  7.81s/it]
 78%|███████▊  | 2421/3123 [5:20:21<1:32:12,  7.88s/it]
 78%|███████▊  | 2422/3123 [5:20:29<1:31:39,  7.85s/it]
 78%|███████▊  | 2423/3123 [5:20:37<1:31:15,  7.82s/it]
 78%|███████▊  | 2424/3123 [5:20:45<1:32:01,  7.90s/it]
 78%|███████▊  | 2425/3123 [5:20:53<1:31:22,  7.85s/it]
 78%|███████▊  | 2426/3123 [5:21:00<1:30:53,  7.82s/it]
 78%|███████▊  | 2427/3123 [5:21:08<1:30:35,  7.81s/it]
 78%|███████▊  | 2428/3123 [5:21:16<1:31:17,  7.88s/it]
 78%|███████▊  | 2429/3123 [5:21:24<1:30:46,  7.85s/it]
 78%|███████▊  | 2430/3123 [5:21:32<1:30:20,  7.82s/it]
                                                       
{'loss': '0.0002283', 'grad_norm': '0.009399', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '84.25', 'tokens/total': 39819264, 'tokens/trainable': 12475845, 'epoch': '2.333'}
+
 78%|███████▊  | 2430/3123 [5:21:32<1:30:20,  7.82s/it]
 78%|███████▊  | 2431/3123 [5:21:40<1:31:02,  7.89s/it]
 78%|███████▊  | 2432/3123 [5:21:48<1:30:26,  7.85s/it]
 78%|███████▊  | 2433/3123 [5:21:55<1:30:01,  7.83s/it]
 78%|███████▊  | 2434/3123 [5:22:03<1:30:40,  7.90s/it]
 78%|███████▊  | 2435/3123 [5:22:11<1:30:07,  7.86s/it]
 78%|███████▊  | 2436/3123 [5:22:19<1:29:42,  7.84s/it]
 78%|███████▊  | 2437/3123 [5:22:27<1:29:22,  7.82s/it]
 78%|███████▊  | 2438/3123 [5:22:35<1:30:09,  7.90s/it]
 78%|███████▊  | 2439/3123 [5:22:43<1:29:35,  7.86s/it]
 78%|███████▊  | 2440/3123 [5:22:50<1:29:06,  7.83s/it]
                                                       
{'loss': '0.000331', 'grad_norm': '0.0141', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '84.82', 'tokens/total': 39983104, 'tokens/trainable': 12527044, 'epoch': '2.343'}
+
 78%|███████▊  | 2440/3123 [5:22:51<1:29:06,  7.83s/it]
 78%|███████▊  | 2441/3123 [5:22:58<1:29:45,  7.90s/it]
 78%|███████▊  | 2442/3123 [5:23:06<1:29:10,  7.86s/it]
 78%|███████▊  | 2443/3123 [5:23:14<1:28:45,  7.83s/it]
 78%|███████▊  | 2444/3123 [5:23:22<1:28:25,  7.81s/it]
 78%|███████▊  | 2445/3123 [5:23:30<1:29:07,  7.89s/it]
 78%|███████▊  | 2446/3123 [5:23:37<1:28:33,  7.85s/it]
 78%|███████▊  | 2447/3123 [5:23:45<1:28:08,  7.82s/it]
 78%|███████▊  | 2448/3123 [5:23:53<1:28:46,  7.89s/it]
 78%|███████▊  | 2449/3123 [5:24:01<1:28:11,  7.85s/it]
 78%|███████▊  | 2450/3123 [5:24:09<1:27:47,  7.83s/it]
                                                       
{'loss': '0.0002154', 'grad_norm': '0.02173', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '91.1', 'tokens/total': 40146944, 'tokens/trainable': 12578584, 'epoch': '2.352'}
+
 78%|███████▊  | 2450/3123 [5:24:09<1:27:47,  7.83s/it]
 78%|███████▊  | 2451/3123 [5:24:17<1:27:27,  7.81s/it]
 79%|███████▊  | 2452/3123 [5:24:25<1:28:08,  7.88s/it]
 79%|███████▊  | 2453/3123 [5:24:32<1:27:37,  7.85s/it]
 79%|███████▊  | 2454/3123 [5:24:40<1:27:12,  7.82s/it]
 79%|███████▊  | 2455/3123 [5:24:48<1:27:46,  7.88s/it]
 79%|███████▊  | 2456/3123 [5:24:56<1:27:15,  7.85s/it]
 79%|███████▊  | 2457/3123 [5:25:04<1:26:51,  7.83s/it]
 79%|███████▊  | 2458/3123 [5:25:12<1:26:33,  7.81s/it]
 79%|███████▊  | 2459/3123 [5:25:20<1:27:15,  7.88s/it]
 79%|███████▉  | 2460/3123 [5:25:27<1:26:42,  7.85s/it]
                                                       
{'loss': '0.0003789', 'grad_norm': '0.003067', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '83.09', 'tokens/total': 40310784, 'tokens/trainable': 12630281, 'epoch': '2.362'}
+
 79%|███████▉  | 2460/3123 [5:25:28<1:26:42,  7.85s/it]
 79%|███████▉  | 2461/3123 [5:25:35<1:26:17,  7.82s/it]
 79%|███████▉  | 2462/3123 [5:25:43<1:26:54,  7.89s/it]
 79%|███████▉  | 2463/3123 [5:25:51<1:26:20,  7.85s/it]
 79%|███████▉  | 2464/3123 [5:25:59<1:25:53,  7.82s/it]
 79%|███████▉  | 2465/3123 [5:26:06<1:25:34,  7.80s/it]
 79%|███████▉  | 2466/3123 [5:26:14<1:26:14,  7.88s/it]
 79%|███████▉  | 2467/3123 [5:26:22<1:25:42,  7.84s/it]
 79%|███████▉  | 2468/3123 [5:26:30<1:25:18,  7.82s/it]
 79%|███████▉  | 2469/3123 [5:26:38<1:25:56,  7.88s/it]
 79%|███████▉  | 2470/3123 [5:26:46<1:25:24,  7.85s/it]
                                                       
{'loss': '0.0003918', 'grad_norm': '0.006439', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '88.9', 'tokens/total': 40474624, 'tokens/trainable': 12681543, 'epoch': '2.371'}
+
 79%|███████▉  | 2470/3123 [5:26:46<1:25:24,  7.85s/it]
 79%|███████▉  | 2471/3123 [5:26:54<1:25:00,  7.82s/it]
 79%|███████▉  | 2472/3123 [5:27:01<1:24:40,  7.80s/it]
 79%|███████▉  | 2473/3123 [5:27:09<1:25:20,  7.88s/it]
 79%|███████▉  | 2474/3123 [5:27:17<1:24:48,  7.84s/it]
 79%|███████▉  | 2475/3123 [5:27:25<1:24:24,  7.82s/it]
 79%|███████▉  | 2476/3123 [5:27:33<1:24:58,  7.88s/it]
 79%|███████▉  | 2477/3123 [5:27:41<1:24:24,  7.84s/it]
 79%|███████▉  | 2478/3123 [5:27:48<1:23:59,  7.81s/it]
 79%|███████▉  | 2479/3123 [5:27:56<1:23:42,  7.80s/it]
 79%|███████▉  | 2480/3123 [5:28:04<1:24:23,  7.87s/it]
                                                       
{'loss': '0.0004559', 'grad_norm': '0.001526', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '86.64', 'tokens/total': 40638464, 'tokens/trainable': 12733182, 'epoch': '2.381'}
+
 79%|███████▉  | 2480/3123 [5:28:05<1:24:23,  7.87s/it]
 79%|███████▉  | 2481/3123 [5:28:12<1:23:50,  7.84s/it]
 79%|███████▉  | 2482/3123 [5:28:20<1:23:25,  7.81s/it]
 80%|███████▉  | 2483/3123 [5:28:28<1:24:01,  7.88s/it]
 80%|███████▉  | 2484/3123 [5:28:35<1:23:29,  7.84s/it]
 80%|███████▉  | 2485/3123 [5:28:43<1:23:04,  7.81s/it]
 80%|███████▉  | 2486/3123 [5:28:51<1:23:40,  7.88s/it]
 80%|███████▉  | 2487/3123 [5:28:59<1:23:07,  7.84s/it]
 80%|███████▉  | 2488/3123 [5:29:07<1:22:42,  7.81s/it]
 80%|███████▉  | 2489/3123 [5:29:15<1:22:21,  7.79s/it]
 80%|███████▉  | 2490/3123 [5:29:23<1:23:00,  7.87s/it]
                                                       
{'loss': '0.0004104', 'grad_norm': '0.0152', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '82.63', 'tokens/total': 40802304, 'tokens/trainable': 12784379, 'epoch': '2.391'}
+
 80%|███████▉  | 2490/3123 [5:29:23<1:23:00,  7.87s/it]
 80%|███████▉  | 2491/3123 [5:29:30<1:22:30,  7.83s/it]
 80%|███████▉  | 2492/3123 [5:29:38<1:22:07,  7.81s/it]
 80%|███████▉  | 2493/3123 [5:29:46<1:22:45,  7.88s/it]
 80%|███████▉  | 2494/3123 [5:29:54<1:22:13,  7.84s/it]
 80%|███████▉  | 2495/3123 [5:30:02<1:21:47,  7.81s/it]
 80%|███████▉  | 2496/3123 [5:30:09<1:21:27,  7.80s/it]
 80%|███████▉  | 2497/3123 [5:30:17<1:22:04,  7.87s/it]
 80%|███████▉  | 2498/3123 [5:30:25<1:21:34,  7.83s/it]
 80%|████████  | 2499/3123 [5:30:33<1:21:11,  7.81s/it]
 80%|████████  | 2500/3123 [5:30:41<1:21:48,  7.88s/it]
                                                       
{'loss': '0.0003842', 'grad_norm': '0.04102', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '85.4', 'tokens/total': 40966144, 'tokens/trainable': 12836202, 'epoch': '2.4'}
+
 80%|████████  | 2500/3123 [5:30:41<1:21:48,  7.88s/it]
 80%|████████  | 2501/3123 [5:30:49<1:21:16,  7.84s/it]
 80%|████████  | 2502/3123 [5:30:56<1:20:53,  7.81s/it]
 80%|████████  | 2503/3123 [5:31:04<1:20:32,  7.79s/it]
 80%|████████  | 2504/3123 [5:31:12<1:21:09,  7.87s/it]
 80%|████████  | 2505/3123 [5:31:20<1:20:39,  7.83s/it]
 80%|████████  | 2506/3123 [5:31:28<1:20:16,  7.81s/it]
 80%|████████  | 2507/3123 [5:31:36<1:20:51,  7.88s/it]
 80%|████████  | 2508/3123 [5:31:44<1:20:18,  7.84s/it]
 80%|████████  | 2509/3123 [5:31:51<1:19:53,  7.81s/it]
 80%|████████  | 2510/3123 [5:31:59<1:19:34,  7.79s/it]
                                                       
{'loss': '0.0003895', 'grad_norm': '0.06641', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '94.12', 'tokens/total': 41129984, 'tokens/trainable': 12887883, 'epoch': '2.41'}
+
 80%|████████  | 2510/3123 [5:31:59<1:19:34,  7.79s/it]
 80%|████████  | 2511/3123 [5:32:07<1:20:11,  7.86s/it]
 80%|████████  | 2512/3123 [5:32:15<1:19:42,  7.83s/it]
 80%|████████  | 2513/3123 [5:32:23<1:19:20,  7.80s/it]
 80%|████████  | 2514/3123 [5:32:31<1:19:54,  7.87s/it]
 81%|████████  | 2515/3123 [5:32:38<1:19:23,  7.84s/it]
 81%|████████  | 2516/3123 [5:32:46<1:19:00,  7.81s/it]
 81%|████████  | 2517/3123 [5:32:54<1:18:41,  7.79s/it]
 81%|████████  | 2518/3123 [5:33:02<1:19:15,  7.86s/it]
 81%|████████  | 2519/3123 [5:33:10<1:18:47,  7.83s/it]
 81%|████████  | 2520/3123 [5:33:17<1:18:25,  7.80s/it]
                                                       
{'loss': '0.0003926', 'grad_norm': '0.05078', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '82.27', 'tokens/total': 41293824, 'tokens/trainable': 12938847, 'epoch': '2.419'}
+
 81%|████████  | 2520/3123 [5:33:18<1:18:25,  7.80s/it]
 81%|████████  | 2521/3123 [5:33:25<1:19:01,  7.88s/it]
 81%|████████  | 2522/3123 [5:33:33<1:18:30,  7.84s/it]
 81%|████████  | 2523/3123 [5:33:41<1:18:06,  7.81s/it]
 81%|████████  | 2524/3123 [5:33:49<1:18:37,  7.88s/it]
 81%|████████  | 2525/3123 [5:33:57<1:18:06,  7.84s/it]
 81%|████████  | 2526/3123 [5:34:04<1:17:41,  7.81s/it]
 81%|████████  | 2527/3123 [5:34:12<1:17:23,  7.79s/it]
 81%|████████  | 2528/3123 [5:34:20<1:17:59,  7.86s/it]
 81%|████████  | 2529/3123 [5:34:28<1:17:31,  7.83s/it]
 81%|████████  | 2530/3123 [5:34:36<1:17:09,  7.81s/it]
                                                       
{'loss': '0.0001311', 'grad_norm': '0.005524', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '95.79', 'tokens/total': 41457664, 'tokens/trainable': 12990352, 'epoch': '2.429'}
+
 81%|████████  | 2530/3123 [5:34:36<1:17:09,  7.81s/it]
 81%|████████  | 2531/3123 [5:34:44<1:17:43,  7.88s/it]
 81%|████████  | 2532/3123 [5:34:51<1:17:11,  7.84s/it]
 81%|████████  | 2533/3123 [5:34:59<1:16:47,  7.81s/it]
 81%|████████  | 2534/3123 [5:35:07<1:16:28,  7.79s/it]
 81%|████████  | 2535/3123 [5:35:15<1:17:03,  7.86s/it]
 81%|████████  | 2536/3123 [5:35:23<1:16:36,  7.83s/it]
 81%|████████  | 2537/3123 [5:35:30<1:16:13,  7.80s/it]
 81%|████████▏ | 2538/3123 [5:35:39<1:16:48,  7.88s/it]
 81%|████████▏ | 2539/3123 [5:35:46<1:16:19,  7.84s/it]
 81%|████████▏ | 2540/3123 [5:35:54<1:15:56,  7.81s/it]
                                                       
{'loss': '0.0002963', 'grad_norm': '0.2109', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '82.32', 'tokens/total': 41621504, 'tokens/trainable': 13041445, 'epoch': '2.439'}
+
 81%|████████▏ | 2540/3123 [5:35:54<1:15:56,  7.81s/it]
 81%|████████▏ | 2541/3123 [5:36:02<1:15:38,  7.80s/it]
 81%|████████▏ | 2542/3123 [5:36:10<1:16:13,  7.87s/it]
 81%|████████▏ | 2543/3123 [5:36:18<1:15:44,  7.83s/it]
 81%|████████▏ | 2544/3123 [5:36:25<1:15:22,  7.81s/it]
 81%|████████▏ | 2545/3123 [5:36:33<1:15:51,  7.87s/it]
 82%|████████▏ | 2546/3123 [5:36:41<1:15:22,  7.84s/it]
 82%|████████▏ | 2547/3123 [5:36:49<1:14:59,  7.81s/it]
 82%|████████▏ | 2548/3123 [5:36:57<1:14:41,  7.79s/it]
 82%|████████▏ | 2549/3123 [5:37:05<1:15:14,  7.86s/it]
 82%|████████▏ | 2550/3123 [5:37:12<1:14:47,  7.83s/it]
                                                       
{'loss': '0.0002713', 'grad_norm': '0.01562', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '97.04', 'tokens/total': 41785344, 'tokens/trainable': 13092732, 'epoch': '2.448'}
+
 82%|████████▏ | 2550/3123 [5:37:13<1:14:47,  7.83s/it]
 82%|████████▏ | 2551/3123 [5:37:20<1:14:25,  7.81s/it]
 82%|████████▏ | 2552/3123 [5:37:28<1:14:58,  7.88s/it]
 82%|████████▏ | 2553/3123 [5:37:36<1:14:27,  7.84s/it]
 82%|████████▏ | 2554/3123 [5:37:44<1:14:05,  7.81s/it]
 82%|████████▏ | 2555/3123 [5:37:51<1:13:46,  7.79s/it]
 82%|████████▏ | 2556/3123 [5:37:59<1:14:21,  7.87s/it]
 82%|████████▏ | 2557/3123 [5:38:07<1:13:54,  7.83s/it]
 82%|████████▏ | 2558/3123 [5:38:15<1:13:31,  7.81s/it]
 82%|████████▏ | 2559/3123 [5:38:23<1:14:01,  7.88s/it]
 82%|████████▏ | 2560/3123 [5:38:31<1:13:32,  7.84s/it]
                                                       
{'loss': '0.0003738', 'grad_norm': '0.07666', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '90.33', 'tokens/total': 41949184, 'tokens/trainable': 13143724, 'epoch': '2.458'}
+
 82%|████████▏ | 2560/3123 [5:38:31<1:13:32,  7.84s/it]
 82%|████████▏ | 2561/3123 [5:38:39<1:13:10,  7.81s/it]
 82%|████████▏ | 2562/3123 [5:38:47<1:13:40,  7.88s/it]
 82%|████████▏ | 2563/3123 [5:38:54<1:13:10,  7.84s/it]
 82%|████████▏ | 2564/3123 [5:39:02<1:12:47,  7.81s/it]
 82%|████████▏ | 2565/3123 [5:39:10<1:12:29,  7.79s/it]
 82%|████████▏ | 2566/3123 [5:39:18<1:13:00,  7.86s/it]
 82%|████████▏ | 2567/3123 [5:39:26<1:12:33,  7.83s/it]
 82%|████████▏ | 2568/3123 [5:39:33<1:12:13,  7.81s/it]
 82%|████████▏ | 2569/3123 [5:39:41<1:12:43,  7.88s/it]
 82%|████████▏ | 2570/3123 [5:39:49<1:12:14,  7.84s/it]
                                                       
{'loss': '0.0004245', 'grad_norm': '0.09961', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '88.65', 'tokens/total': 42113024, 'tokens/trainable': 13195474, 'epoch': '2.467'}
+
 82%|████████▏ | 2570/3123 [5:39:49<1:12:14,  7.84s/it]
 82%|████████▏ | 2571/3123 [5:39:57<1:11:52,  7.81s/it]
 82%|████████▏ | 2572/3123 [5:40:05<1:11:35,  7.80s/it]
 82%|████████▏ | 2573/3123 [5:40:13<1:12:06,  7.87s/it]
 82%|████████▏ | 2574/3123 [5:40:20<1:11:40,  7.83s/it]
 82%|████████▏ | 2575/3123 [5:40:28<1:11:18,  7.81s/it]
 82%|████████▏ | 2576/3123 [5:40:36<1:11:47,  7.87s/it]
 83%|████████▎ | 2577/3123 [5:40:44<1:11:18,  7.84s/it]
 83%|████████▎ | 2578/3123 [5:40:52<1:10:56,  7.81s/it]
 83%|████████▎ | 2579/3123 [5:40:59<1:10:38,  7.79s/it]
 83%|████████▎ | 2580/3123 [5:41:07<1:11:06,  7.86s/it]
                                                       
{'loss': '0.0007923', 'grad_norm': '0.2051', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '81.86', 'tokens/total': 42276864, 'tokens/trainable': 13246900, 'epoch': '2.477'}
+
 83%|████████▎ | 2580/3123 [5:41:08<1:11:06,  7.86s/it]
 83%|████████▎ | 2581/3123 [5:41:15<1:10:42,  7.83s/it]
 83%|████████▎ | 2582/3123 [5:41:23<1:10:20,  7.80s/it]
 83%|████████▎ | 2583/3123 [5:41:31<1:10:51,  7.87s/it]
 83%|████████▎ | 2584/3123 [5:41:39<1:10:23,  7.84s/it]
 83%|████████▎ | 2585/3123 [5:41:46<1:10:01,  7.81s/it]
 83%|████████▎ | 2586/3123 [5:41:54<1:09:43,  7.79s/it]
 83%|████████▎ | 2587/3123 [5:42:02<1:10:14,  7.86s/it]
 83%|████████▎ | 2588/3123 [5:42:10<1:09:48,  7.83s/it]
 83%|████████▎ | 2589/3123 [5:42:18<1:09:28,  7.81s/it]
 83%|████████▎ | 2590/3123 [5:42:26<1:09:57,  7.88s/it]
                                                       
{'loss': '0.0005574', 'grad_norm': '0.1006', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '86.28', 'tokens/total': 42440704, 'tokens/trainable': 13298400, 'epoch': '2.487'}
+
 83%|████████▎ | 2590/3123 [5:42:26<1:09:57,  7.88s/it]
 83%|████████▎ | 2591/3123 [5:42:34<1:09:29,  7.84s/it]
 83%|████████▎ | 2592/3123 [5:42:41<1:09:06,  7.81s/it]
 83%|████████▎ | 2593/3123 [5:42:49<1:08:48,  7.79s/it]
 83%|████████▎ | 2594/3123 [5:42:57<1:09:17,  7.86s/it]
 83%|████████▎ | 2595/3123 [5:43:05<1:08:52,  7.83s/it]
 83%|████████▎ | 2596/3123 [5:43:13<1:08:32,  7.80s/it]
 83%|████████▎ | 2597/3123 [5:43:21<1:09:00,  7.87s/it]
 83%|████████▎ | 2598/3123 [5:43:28<1:08:32,  7.83s/it]
 83%|████████▎ | 2599/3123 [5:43:36<1:08:11,  7.81s/it]
 83%|████████▎ | 2600/3123 [5:43:44<1:07:53,  7.79s/it]
                                                       
{'loss': '0.001263', 'grad_norm': '0.07227', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '96.03', 'tokens/total': 42604544, 'tokens/trainable': 13350114, 'epoch': '2.496'}
+
 83%|████████▎ | 2600/3123 [5:43:44<1:07:53,  7.79s/it]
 83%|████████▎ | 2601/3123 [5:43:52<1:08:22,  7.86s/it]
 83%|████████▎ | 2602/3123 [5:44:00<1:07:57,  7.83s/it]
 83%|████████▎ | 2603/3123 [5:44:07<1:07:38,  7.80s/it]
 83%|████████▎ | 2604/3123 [5:44:15<1:08:07,  7.88s/it]
 83%|████████▎ | 2605/3123 [5:44:23<1:07:38,  7.83s/it][2026-03-08 21:41:37,596] [INFO] [axolotl.core.trainers.base.evaluate:400] [PID:1659682] Running evaluation step...
+[2026-03-08 21:41:38,482] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:1659682] generate_batches time: 0.43762636184692383
+[2026-03-08 21:41:38,930] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:1659682] generate_batches time: 0.4480259418487549
+[2026-03-08 21:41:39,365] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:1659682] generate_batches time: 0.4345855712890625
+[2026-03-08 21:41:39,814] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:1659682] generate_batches time: 0.44884610176086426
+[2026-03-08 21:41:39,814] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:1659682] gather_len_batches: [34]
+
+
  0%|          | 0/34 [00:00<?, ?it/s]
+
  6%|▌         | 2/34 [00:00<00:03,  8.54it/s]
+
  9%|▉         | 3/34 [00:00<00:05,  5.98it/s]
+
 12%|█▏        | 4/34 [00:00<00:05,  5.16it/s]
+
 15%|█▍        | 5/34 [00:00<00:06,  4.78it/s]
+
 18%|█▊        | 6/34 [00:01<00:06,  4.57it/s]
+
 21%|██        | 7/34 [00:01<00:06,  4.45it/s]
+
 24%|██▎       | 8/34 [00:01<00:05,  4.37it/s]
+
 26%|██▋       | 9/34 [00:01<00:06,  3.93it/s]
+
 29%|██▉       | 10/34 [00:02<00:05,  4.08it/s]
+
 32%|███▏      | 11/34 [00:02<00:05,  4.12it/s]
+
 35%|███▌      | 12/34 [00:02<00:05,  4.14it/s]
+
 38%|███▊      | 13/34 [00:02<00:05,  4.16it/s]
+
 41%|████      | 14/34 [00:03<00:04,  4.17it/s]
+
 44%|████▍     | 15/34 [00:03<00:04,  4.19it/s]
+
 47%|████▋     | 16/34 [00:03<00:04,  4.19it/s]
+
 50%|█████     | 17/34 [00:04<00:05,  3.35it/s]
+
 53%|█████▎    | 18/34 [00:04<00:04,  3.65it/s]
+
 56%|█████▌    | 19/34 [00:04<00:03,  3.80it/s]
+
 59%|█████▉    | 20/34 [00:04<00:03,  3.92it/s]
+
 62%|██████▏   | 21/34 [00:04<00:03,  4.00it/s]
+
 65%|██████▍   | 22/34 [00:05<00:02,  4.06it/s]
+
 68%|██████▊   | 23/34 [00:05<00:02,  4.10it/s]
+
 71%|███████   | 24/34 [00:05<00:02,  4.14it/s]
+
 74%|███████▎  | 25/34 [00:06<00:03,  2.91it/s]
+
 76%|███████▋  | 26/34 [00:06<00:02,  3.23it/s]
+
 79%|███████▉  | 27/34 [00:06<00:02,  3.48it/s]
+
 82%|████████▏ | 28/34 [00:06<00:01,  3.67it/s]
+
 85%|████████▌ | 29/34 [00:07<00:01,  3.81it/s]
+
 88%|████████▊ | 30/34 [00:07<00:01,  3.92it/s]
+
 91%|█████████ | 31/34 [00:07<00:00,  4.01it/s]
+
 94%|█████████▍| 32/34 [00:07<00:00,  4.07it/s]
+
 97%|█████████▋| 33/34 [00:08<00:00,  3.98it/s]Traceback (most recent call last):
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/multiprocessing/util.py", line 300, in _run_finalizers
+    finalizer()
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/multiprocessing/util.py", line 224, in __call__
+    res = self._callback(*self._args, **self._kwargs)
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/multiprocessing/util.py", line 133, in _remove_temp_dir
+    rmtree(tempdir)
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/shutil.py", line 725, in rmtree
+    _rmtree_safe_fd(fd, path, onerror)
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/shutil.py", line 681, in _rmtree_safe_fd
+    onerror(os.unlink, fullname, sys.exc_info())
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/shutil.py", line 679, in _rmtree_safe_fd
+    os.unlink(entry.name, dir_fd=topfd)
+OSError: [Errno 16] Device or resource busy: '.nfs00000000000db29000012514'
+
+
100%|██████████| 34/34 [00:08<00:00,  3.89it/s]
                                                       
+
                                               
{'eval_loss': '0.000673', 'eval_runtime': '9.369', 'eval_samples_per_second': '21.35', 'eval_steps_per_second': '21.35', 'eval_ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'epoch': '2.501', 'tokens/train_per_sec_per_gpu': '83.82'}
+
 83%|████████▎ | 2605/3123 [5:44:35<1:07:38,  7.83s/it]
+
100%|██████████| 34/34 [00:08<00:00,  3.89it/s]
+
                                               
 83%|████████▎ | 2606/3123 [5:44:42<1:36:35, 11.21s/it]
 83%|████████▎ | 2607/3123 [5:44:50<1:27:28, 10.17s/it]
 84%|████████▎ | 2608/3123 [5:44:58<1:21:03,  9.44s/it]
 84%|████████▎ | 2609/3123 [5:45:06<1:17:25,  9.04s/it]
 84%|████████▎ | 2610/3123 [5:45:14<1:13:57,  8.65s/it]
                                                       
{'loss': '0.0009072', 'grad_norm': '0.09277', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.96', 'memory/max_allocated (GiB)': '33.96', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '89.57', 'tokens/total': 42768384, 'tokens/trainable': 13401904, 'epoch': '2.506'}
+
 84%|████████▎ | 2610/3123 [5:45:14<1:13:57,  8.65s/it]
 84%|████████▎ | 2611/3123 [5:45:21<1:11:29,  8.38s/it]
 84%|████████▎ | 2612/3123 [5:45:29<1:09:43,  8.19s/it]
 84%|████████▎ | 2613/3123 [5:45:37<1:09:14,  8.15s/it]
 84%|████████▎ | 2614/3123 [5:45:45<1:08:06,  8.03s/it]
 84%|████████▎ | 2615/3123 [5:45:53<1:07:17,  7.95s/it]
 84%|████████▍ | 2616/3123 [5:46:01<1:07:22,  7.97s/it]
 84%|████████▍ | 2617/3123 [5:46:08<1:06:41,  7.91s/it]
 84%|████████▍ | 2618/3123 [5:46:16<1:06:10,  7.86s/it]
 84%|████████▍ | 2619/3123 [5:46:24<1:05:46,  7.83s/it]
 84%|████████▍ | 2620/3123 [5:46:32<1:06:10,  7.89s/it]
                                                       
{'loss': '0.001005', 'grad_norm': '0.02405', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '87.13', 'tokens/total': 42932224, 'tokens/trainable': 13453338, 'epoch': '2.515'}
+
 84%|████████▍ | 2620/3123 [5:46:32<1:06:10,  7.89s/it]
 84%|████████▍ | 2621/3123 [5:46:40<1:05:42,  7.85s/it]
 84%|████████▍ | 2622/3123 [5:46:47<1:05:19,  7.82s/it]
 84%|████████▍ | 2623/3123 [5:46:56<1:05:48,  7.90s/it]
 84%|████████▍ | 2624/3123 [5:47:03<1:05:20,  7.86s/it]
 84%|████████▍ | 2625/3123 [5:47:11<1:04:57,  7.83s/it]
 84%|████████▍ | 2626/3123 [5:47:19<1:04:40,  7.81s/it]
 84%|████████▍ | 2627/3123 [5:47:27<1:05:07,  7.88s/it]
 84%|████████▍ | 2628/3123 [5:47:35<1:04:41,  7.84s/it]
 84%|████████▍ | 2629/3123 [5:47:42<1:04:21,  7.82s/it]
 84%|████████▍ | 2630/3123 [5:47:50<1:04:47,  7.88s/it]
                                                       
{'loss': '0.0007242', 'grad_norm': '0.02087', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '74.71', 'tokens/total': 43096064, 'tokens/trainable': 13504169, 'epoch': '2.525'}
+
 84%|████████▍ | 2630/3123 [5:47:51<1:04:47,  7.88s/it]
 84%|████████▍ | 2631/3123 [5:47:58<1:04:20,  7.85s/it]
 84%|████████▍ | 2632/3123 [5:48:06<1:03:59,  7.82s/it]
 84%|████████▍ | 2633/3123 [5:48:14<1:03:40,  7.80s/it]
 84%|████████▍ | 2634/3123 [5:48:22<1:04:07,  7.87s/it]
 84%|████████▍ | 2635/3123 [5:48:29<1:03:43,  7.83s/it]
 84%|████████▍ | 2636/3123 [5:48:37<1:03:25,  7.81s/it]
 84%|████████▍ | 2637/3123 [5:48:45<1:03:51,  7.88s/it]
 84%|████████▍ | 2638/3123 [5:48:53<1:03:24,  7.85s/it]
 85%|████████▍ | 2639/3123 [5:49:01<1:03:04,  7.82s/it]
 85%|████████▍ | 2640/3123 [5:49:09<1:02:47,  7.80s/it]
                                                       
{'loss': '0.0006305', 'grad_norm': '0.01373', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '87', 'tokens/total': 43259904, 'tokens/trainable': 13555651, 'epoch': '2.535'}
+
 85%|████████▍ | 2640/3123 [5:49:09<1:02:47,  7.80s/it]
 85%|████████▍ | 2641/3123 [5:49:17<1:03:15,  7.87s/it]
 85%|████████▍ | 2642/3123 [5:49:24<1:02:51,  7.84s/it]
 85%|████████▍ | 2643/3123 [5:49:32<1:02:30,  7.81s/it]
 85%|████████▍ | 2644/3123 [5:49:40<1:02:56,  7.88s/it]
 85%|████████▍ | 2645/3123 [5:49:48<1:02:28,  7.84s/it]
 85%|████████▍ | 2646/3123 [5:49:56<1:02:08,  7.82s/it]
 85%|█��██████▍ | 2647/3123 [5:50:04<1:02:32,  7.88s/it]
 85%|████████▍ | 2648/3123 [5:50:11<1:02:07,  7.85s/it]
 85%|████████▍ | 2649/3123 [5:50:19<1:01:46,  7.82s/it]
 85%|████████▍ | 2650/3123 [5:50:27<1:01:31,  7.80s/it]
                                                       
{'loss': '0.0003988', 'grad_norm': '0.04028', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '79.48', 'tokens/total': 43423744, 'tokens/trainable': 13606876, 'epoch': '2.544'}
+
 85%|████████▍ | 2650/3123 [5:50:27<1:01:31,  7.80s/it]
 85%|████████▍ | 2651/3123 [5:50:35<1:02:01,  7.88s/it]
 85%|████████▍ | 2652/3123 [5:50:43<1:01:35,  7.85s/it]
 85%|████████▍ | 2653/3123 [5:50:51<1:01:16,  7.82s/it]
 85%|████████▍ | 2654/3123 [5:50:59<1:01:41,  7.89s/it]
 85%|████████▌ | 2655/3123 [5:51:06<1:01:13,  7.85s/it]
 85%|████████▌ | 2656/3123 [5:51:14<1:00:52,  7.82s/it]
 85%|████████▌ | 2657/3123 [5:51:22<1:00:34,  7.80s/it]
 85%|████████▌ | 2658/3123 [5:51:30<1:01:03,  7.88s/it]
 85%|████████▌ | 2659/3123 [5:51:38<1:00:37,  7.84s/it]
 85%|████████▌ | 2660/3123 [5:51:45<1:00:16,  7.81s/it]
                                                       
{'loss': '0.0002031', 'grad_norm': '0.004333', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '79.86', 'tokens/total': 43587584, 'tokens/trainable': 13659148, 'epoch': '2.554'}
+
 85%|████████▌ | 2660/3123 [5:51:46<1:00:16,  7.81s/it]
 85%|████████▌ | 2661/3123 [5:51:53<1:00:41,  7.88s/it]
 85%|████████▌ | 2662/3123 [5:52:01<1:00:15,  7.84s/it]
 85%|████████▌ | 2663/3123 [5:52:09<59:54,  7.81s/it]  
 85%|████████▌ | 2664/3123 [5:52:17<59:37,  7.79s/it]
 85%|████████▌ | 2665/3123 [5:52:25<1:00:02,  7.87s/it]
 85%|████████▌ | 2666/3123 [5:52:33<59:37,  7.83s/it]  
 85%|████████▌ | 2667/3123 [5:52:40<59:18,  7.80s/it]
 85%|████████▌ | 2668/3123 [5:52:48<59:42,  7.87s/it]
 85%|████████▌ | 2669/3123 [5:52:56<59:17,  7.84s/it]
 85%|████████▌ | 2670/3123 [5:53:04<58:58,  7.81s/it]
                                                     
{'loss': '0.0004935', 'grad_norm': '0.1709', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '89.03', 'tokens/total': 43751424, 'tokens/trainable': 13710309, 'epoch': '2.563'}
+
 85%|████████▌ | 2670/3123 [5:53:04<58:58,  7.81s/it]
 86%|████████▌ | 2671/3123 [5:53:12<58:43,  7.80s/it]
 86%|████████▌ | 2672/3123 [5:53:20<59:09,  7.87s/it]
 86%|████████▌ | 2673/3123 [5:53:27<58:46,  7.84s/it]
 86%|████████▌ | 2674/3123 [5:53:35<58:27,  7.81s/it]
 86%|████████▌ | 2675/3123 [5:53:43<58:52,  7.88s/it]
 86%|████████▌ | 2676/3123 [5:53:51<58:25,  7.84s/it]
 86%|████████▌ | 2677/3123 [5:53:59<58:05,  7.82s/it]
 86%|████████▌ | 2678/3123 [5:54:06<57:50,  7.80s/it]
 86%|████████▌ | 2679/3123 [5:54:14<58:15,  7.87s/it]
 86%|████████▌ | 2680/3123 [5:54:22<57:51,  7.84s/it]
                                                     
{'loss': '0.0003111', 'grad_norm': '0.2061', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '88.78', 'tokens/total': 43915264, 'tokens/trainable': 13762067, 'epoch': '2.573'}
+
 86%|████████▌ | 2680/3123 [5:54:23<57:51,  7.84s/it]
 86%|████████▌ | 2681/3123 [5:54:30<57:33,  7.81s/it]
 86%|████████▌ | 2682/3123 [5:54:38<57:58,  7.89s/it]
 86%|████████▌ | 2683/3123 [5:54:46<57:32,  7.85s/it]
 86%|████████▌ | 2684/3123 [5:54:54<57:12,  7.82s/it]
 86%|████████▌ | 2685/3123 [5:55:01<56:55,  7.80s/it]
 86%|████████▌ | 2686/3123 [5:55:09<57:19,  7.87s/it]
 86%|████████▌ | 2687/3123 [5:55:17<56:55,  7.83s/it]
 86%|████████▌ | 2688/3123 [5:55:25<56:37,  7.81s/it]
 86%|████████▌ | 2689/3123 [5:55:33<57:00,  7.88s/it]
 86%|████████▌ | 2690/3123 [5:55:41<56:35,  7.84s/it]
                                                     
{'loss': '0.0002585', 'grad_norm': '0.01428', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '81.95', 'tokens/total': 44079104, 'tokens/trainable': 13812823, 'epoch': '2.583'}
+
 86%|████████▌ | 2690/3123 [5:55:41<56:35,  7.84s/it]
 86%|████████▌ | 2691/3123 [5:55:48<56:15,  7.81s/it]
 86%|████████▌ | 2692/3123 [5:55:56<56:00,  7.80s/it]
 86%|████████▌ | 2693/3123 [5:56:04<56:22,  7.87s/it]
 86%|████████▋ | 2694/3123 [5:56:12<56:00,  7.83s/it]
 86%|████████▋ | 2695/3123 [5:56:20<55:42,  7.81s/it]
 86%|████████▋ | 2696/3123 [5:56:28<56:06,  7.88s/it]
 86%|████████▋ | 2697/3123 [5:56:35<55:41,  7.84s/it]
 86%|████████▋ | 2698/3123 [5:56:43<55:21,  7.82s/it]
 86%|████████▋ | 2699/3123 [5:56:51<55:06,  7.80s/it]
 86%|████████▋ | 2700/3123 [5:56:59<55:30,  7.87s/it]
                                                     
{'loss': '0.0005487', 'grad_norm': '0.007599', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '77.46', 'tokens/total': 44242944, 'tokens/trainable': 13863696, 'epoch': '2.592'}
+
 86%|████████▋ | 2700/3123 [5:56:59<55:30,  7.87s/it]
 86%|████████▋ | 2701/3123 [5:57:07<55:07,  7.84s/it]
 87%|████████▋ | 2702/3123 [5:57:15<54:48,  7.81s/it]
 87%|████████▋ | 2703/3123 [5:57:23<55:07,  7.87s/it]
 87%|████████▋ | 2704/3123 [5:57:30<54:44,  7.84s/it]
 87%|████████▋ | 2705/3123 [5:57:38<54:26,  7.81s/it]
 87%|████████▋ | 2706/3123 [5:57:46<54:09,  7.79s/it]
 87%|████████▋ | 2707/3123 [5:57:54<54:34,  7.87s/it]
 87%|████████▋ | 2708/3123 [5:58:02<54:12,  7.84s/it]
 87%|████████▋ | 2709/3123 [5:58:09<53:53,  7.81s/it]
 87%|████████▋ | 2710/3123 [5:58:17<54:13,  7.88s/it]
                                                     
{'loss': '0.0002872', 'grad_norm': '0.01196', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '82.82', 'tokens/total': 44406784, 'tokens/trainable': 13914880, 'epoch': '2.602'}
+
 87%|████████▋ | 2710/3123 [5:58:18<54:13,  7.88s/it]
 87%|████████▋ | 2711/3123 [5:58:25<53:51,  7.84s/it]
 87%|████████▋ | 2712/3123 [5:58:33<53:32,  7.82s/it]
 87%|████████▋ | 2713/3123 [5:58:41<53:53,  7.89s/it]
 87%|████████▋ | 2714/3123 [5:58:49<53:28,  7.85s/it]
 87%|████████▋ | 2715/3123 [5:58:56<53:10,  7.82s/it]
 87%|████████▋ | 2716/3123 [5:59:04<52:54,  7.80s/it]
 87%|████████▋ | 2717/3123 [5:59:12<53:17,  7.88s/it]
 87%|████████▋ | 2718/3123 [5:59:20<52:54,  7.84s/it]
 87%|████████▋ | 2719/3123 [5:59:28<52:36,  7.81s/it]
 87%|████████▋ | 2720/3123 [5:59:36<52:56,  7.88s/it]
                                                     
{'loss': '0.0003391', 'grad_norm': '0.05298', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '86.75', 'tokens/total': 44570624, 'tokens/trainable': 13966294, 'epoch': '2.611'}
+
 87%|████████▋ | 2720/3123 [5:59:36<52:56,  7.88s/it]
 87%|████████▋ | 2721/3123 [5:59:44<52:34,  7.85s/it]
 87%|████████▋ | 2722/3123 [5:59:51<52:15,  7.82s/it]
 87%|████████▋ | 2723/3123 [5:59:59<51:59,  7.80s/it]
 87%|████████▋ | 2724/3123 [6:00:07<52:21,  7.87s/it]
 87%|████████▋ | 2725/3123 [6:00:15<51:59,  7.84s/it]
 87%|████████▋ | 2726/3123 [6:00:23<51:41,  7.81s/it]
 87%|████████▋ | 2727/3123 [6:00:30<51:25,  7.79s/it]
 87%|████████▋ | 2728/3123 [6:00:38<51:46,  7.86s/it]
 87%|████████▋ | 2729/3123 [6:00:46<51:24,  7.83s/it]
 87%|████████▋ | 2730/3123 [6:00:54<51:08,  7.81s/it]
                                                     
{'loss': '0.001788', 'grad_norm': '0.06396', 'learning_rate': '0.0001', 'ppl': '1.002', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '86.62', 'tokens/total': 44734464, 'tokens/trainable': 14017349, 'epoch': '2.621'}
+
 87%|████████▋ | 2730/3123 [6:00:54<51:08,  7.81s/it]
 87%|████████▋ | 2731/3123 [6:01:02<51:27,  7.88s/it]
 87%|████████▋ | 2732/3123 [6:01:10<51:04,  7.84s/it]
 88%|█���██████▊ | 2733/3123 [6:01:17<50:46,  7.81s/it]
 88%|████████▊ | 2734/3123 [6:01:25<50:32,  7.79s/it]
 88%|████████▊ | 2735/3123 [6:01:33<50:51,  7.87s/it]
 88%|████████▊ | 2736/3123 [6:01:41<50:30,  7.83s/it]
 88%|████████▊ | 2737/3123 [6:01:49<50:13,  7.81s/it]
 88%|████████▊ | 2738/3123 [6:01:57<50:32,  7.88s/it]
 88%|████████▊ | 2739/3123 [6:02:05<50:10,  7.84s/it]
 88%|████████▊ | 2740/3123 [6:02:12<49:52,  7.81s/it]
                                                     
{'loss': '0.001176', 'grad_norm': '0.1055', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '82.74', 'tokens/total': 44898304, 'tokens/trainable': 14068850, 'epoch': '2.631'}
+
 88%|████████▊ | 2740/3123 [6:02:13<49:52,  7.81s/it]
 88%|████████▊ | 2741/3123 [6:02:20<49:37,  7.79s/it]
 88%|████████▊ | 2742/3123 [6:02:28<49:56,  7.86s/it]
 88%|████████▊ | 2743/3123 [6:02:36<49:35,  7.83s/it]
 88%|████████▊ | 2744/3123 [6:02:44<49:19,  7.81s/it]
 88%|████████▊ | 2745/3123 [6:02:52<49:38,  7.88s/it]
 88%|████████▊ | 2746/3123 [6:02:59<49:14,  7.84s/it]
 88%|████████▊ | 2747/3123 [6:03:07<48:56,  7.81s/it]
 88%|████████▊ | 2748/3123 [6:03:15<48:42,  7.79s/it]
 88%|████████▊ | 2749/3123 [6:03:23<49:02,  7.87s/it]
 88%|████████▊ | 2750/3123 [6:03:31<48:41,  7.83s/it]
                                                     
{'loss': '0.0007541', 'grad_norm': '0.1602', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '78.49', 'tokens/total': 45062144, 'tokens/trainable': 14119860, 'epoch': '2.64'}
+
 88%|████████▊ | 2750/3123 [6:03:31<48:41,  7.83s/it]
 88%|████████▊ | 2751/3123 [6:03:38<48:24,  7.81s/it]
 88%|████████▊ | 2752/3123 [6:03:46<48:41,  7.87s/it]
 88%|████████▊ | 2753/3123 [6:03:54<48:19,  7.84s/it]
 88%|████████▊ | 2754/3123 [6:04:02<48:02,  7.81s/it]
 88%|████████▊ | 2755/3123 [6:04:10<48:20,  7.88s/it]
 88%|████████▊ | 2756/3123 [6:04:18<47:58,  7.84s/it]
 88%|████████▊ | 2757/3123 [6:04:26<47:40,  7.82s/it]
 88%|████████▊ | 2758/3123 [6:04:33<47:25,  7.80s/it]
 88%|████████▊ | 2759/3123 [6:04:41<47:42,  7.87s/it]
 88%|████████▊ | 2760/3123 [6:04:49<47:22,  7.83s/it]
                                                     
{'loss': '0.0005128', 'grad_norm': '0.0437', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '81.58', 'tokens/total': 45225984, 'tokens/trainable': 14170507, 'epoch': '2.65'}
+
 88%|████████▊ | 2760/3123 [6:04:49<47:22,  7.83s/it]
 88%|████████▊ | 2761/3123 [6:04:57<47:05,  7.81s/it]
 88%|████████▊ | 2762/3123 [6:05:05<47:24,  7.88s/it]
 88%|████████▊ | 2763/3123 [6:05:13<47:01,  7.84s/it]
 89%|████████▊ | 2764/3123 [6:05:20<46:45,  7.81s/it]
 89%|████████▊ | 2765/3123 [6:05:28<46:31,  7.80s/it]
 89%|████████▊ | 2766/3123 [6:05:36<46:51,  7.87s/it]
 89%|████████▊ | 2767/3123 [6:05:44<46:30,  7.84s/it]
 89%|████████▊ | 2768/3123 [6:05:52<46:14,  7.82s/it]
 89%|████████▊ | 2769/3123 [6:06:00<46:31,  7.88s/it]
 89%|████████▊ | 2770/3123 [6:06:07<46:10,  7.85s/it]
                                                     
{'loss': '0.000405', 'grad_norm': '0.02393', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '86.13', 'tokens/total': 45389824, 'tokens/trainable': 14222596, 'epoch': '2.659'}
+
 89%|████████▊ | 2770/3123 [6:06:08<46:10,  7.85s/it]
 89%|████████▊ | 2771/3123 [6:06:15<45:52,  7.82s/it]
 89%|████████▉ | 2772/3123 [6:06:23<45:38,  7.80s/it]
 89%|████████▉ | 2773/3123 [6:06:31<45:55,  7.87s/it]
 89%|████████▉ | 2774/3123 [6:06:39<45:35,  7.84s/it]
 89%|████████▉ | 2775/3123 [6:06:47<45:19,  7.82s/it]
 89%|████████▉ | 2776/3123 [6:06:55<45:37,  7.89s/it]
 89%|████████▉ | 2777/3123 [6:07:02<45:15,  7.85s/it]
 89%|████████▉ | 2778/3123 [6:07:10<44:59,  7.82s/it]
 89%|████████▉ | 2779/3123 [6:07:18<44:45,  7.81s/it]
 89%|████████▉ | 2780/3123 [6:07:26<45:02,  7.88s/it]
                                                     
{'loss': '0.0003614', 'grad_norm': '0.04858', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '90.3', 'tokens/total': 45553664, 'tokens/trainable': 14273514, 'epoch': '2.669'}
+
 89%|████████▉ | 2780/3123 [6:07:26<45:02,  7.88s/it]
 89%|████████▉ | 2781/3123 [6:07:34<44:42,  7.84s/it]
 89%|████████▉ | 2782/3123 [6:07:41<44:26,  7.82s/it]
 89%|████████▉ | 2783/3123 [6:07:50<44:42,  7.89s/it]
 89%|████████▉ | 2784/3123 [6:07:57<44:20,  7.85s/it]
 89%|████████▉ | 2785/3123 [6:08:05<44:04,  7.82s/it]
 89%|████████▉ | 2786/3123 [6:08:13<44:21,  7.90s/it]
 89%|████████▉ | 2787/3123 [6:08:21<43:59,  7.86s/it]
 89%|████████▉ | 2788/3123 [6:08:29<43:42,  7.83s/it]
 89%|████████▉ | 2789/3123 [6:08:36<43:27,  7.81s/it]
 89%|████████▉ | 2790/3123 [6:08:44<43:42,  7.88s/it]
                                                     
{'loss': '0.0002967', 'grad_norm': '0.02478', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '74.99', 'tokens/total': 45717504, 'tokens/trainable': 14324069, 'epoch': '2.679'}
+
 89%|████████▉ | 2790/3123 [6:08:45<43:42,  7.88s/it]
 89%|████████▉ | 2791/3123 [6:08:52<43:23,  7.84s/it]
 89%|████████▉ | 2792/3123 [6:09:00<43:07,  7.82s/it]
 89%|████████▉ | 2793/3123 [6:09:08<43:23,  7.89s/it]
 89%|████████▉ | 2794/3123 [6:09:16<43:02,  7.85s/it]
 89%|████████▉ | 2795/3123 [6:09:24<42:45,  7.82s/it]
 90%|████████▉ | 2796/3123 [6:09:31<42:32,  7.80s/it]
 90%|████████▉ | 2797/3123 [6:09:39<42:47,  7.88s/it]
 90%|████████▉ | 2798/3123 [6:09:47<42:28,  7.84s/it]
 90%|████████▉ | 2799/3123 [6:09:55<42:13,  7.82s/it]
 90%|████████▉ | 2800/3123 [6:10:03<42:28,  7.89s/it]
                                                     
{'loss': '0.0004262', 'grad_norm': '0.0238', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '72.26', 'tokens/total': 45881344, 'tokens/trainable': 14374873, 'epoch': '2.688'}
+
 90%|████████▉ | 2800/3123 [6:10:03<42:28,  7.89s/it]
 90%|████████▉ | 2801/3123 [6:10:11<42:08,  7.85s/it]
 90%|████████▉ | 2802/3123 [6:10:18<41:52,  7.83s/it]
 90%|████████▉ | 2803/3123 [6:10:26<41:37,  7.81s/it]
 90%|████████▉ | 2804/3123 [6:10:34<41:52,  7.88s/it]
 90%|████████▉ | 2805/3123 [6:10:42<41:34,  7.84s/it]
 90%|████████▉ | 2806/3123 [6:10:50<41:17,  7.82s/it]
 90%|████████▉ | 2807/3123 [6:10:58<41:32,  7.89s/it]
 90%|████████▉ | 2808/3123 [6:11:06<41:12,  7.85s/it]
 90%|████████▉ | 2809/3123 [6:11:13<40:55,  7.82s/it]
 90%|████████▉ | 2810/3123 [6:11:21<40:41,  7.80s/it]
                                                     
{'loss': '0.0003718', 'grad_norm': '0.01331', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '93.98', 'tokens/total': 46045184, 'tokens/trainable': 14426134, 'epoch': '2.698'}
+
 90%|████████▉ | 2810/3123 [6:11:21<40:41,  7.80s/it]
 90%|█████████ | 2811/3123 [6:11:29<40:56,  7.87s/it]
 90%|█████████ | 2812/3123 [6:11:37<40:38,  7.84s/it]
 90%|█████████ | 2813/3123 [6:11:45<40:22,  7.82s/it]
 90%|█████████ | 2814/3123 [6:11:53<40:38,  7.89s/it]
 90%|█████████ | 2815/3123 [6:12:00<40:17,  7.85s/it]
 90%|█████████ | 2816/3123 [6:12:08<40:00,  7.82s/it]
 90%|█████████ | 2817/3123 [6:12:16<39:46,  7.80s/it]
 90%|█████████ | 2818/3123 [6:12:24<40:02,  7.88s/it]
 90%|█████████ | 2819/3123 [6:12:32<39:43,  7.84s/it]
 90%|█████████ | 2820/3123 [6:12:40<39:27,  7.81s/it]
                                                     
{'loss': '0.0002686', 'grad_norm': '0.01257', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '86.79', 'tokens/total': 46209024, 'tokens/trainable': 14477507, 'epoch': '2.707'}
+
 90%|█████████ | 2820/3123 [6:12:40<39:27,  7.81s/it]
 90%|█████████ | 2821/3123 [6:12:48<39:40,  7.88s/it]
 90%|█████████ | 2822/3123 [6:12:55<39:20,  7.84s/it]
 90%|█████████ | 2823/3123 [6:13:03<39:05,  7.82s/it]
 90%|█████████ | 2824/3123 [6:13:11<38:52,  7.80s/it]
 90%|█████████ | 2825/3123 [6:13:19<39:09,  7.88s/it]
 90%|█████████ | 2826/3123 [6:13:27<38:50,  7.85s/it]
 91%|█████████ | 2827/3123 [6:13:34<38:34,  7.82s/it]
 91%|█████████ | 2828/3123 [6:13:42<38:46,  7.89s/it]
 91%|█████████ | 2829/3123 [6:13:50<38:27,  7.85s/it]
 91%|█████████ | 2830/3123 [6:13:58<38:11,  7.82s/it]
                                                     
{'loss': '0.0002037', 'grad_norm': '0.09424', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '84.57', 'tokens/total': 46372864, 'tokens/trainable': 14529494, 'epoch': '2.717'}
+
 91%|█████████ | 2830/3123 [6:13:58<38:11,  7.82s/it]
 91%|█████████ | 2831/3123 [6:14:06<37:58,  7.80s/it]
 91%|█████████ | 2832/3123 [6:14:14<38:13,  7.88s/it]
 91%|█████████ | 2833/3123 [6:14:22<37:54,  7.84s/it]
 91%|█████████ | 2834/3123 [6:14:29<37:39,  7.82s/it]
 91%|█████████ | 2835/3123 [6:14:37<37:52,  7.89s/it]
 91%|█████████ | 2836/3123 [6:14:45<37:33,  7.85s/it]
 91%|█████████ | 2837/3123 [6:14:53<37:17,  7.82s/it]
 91%|█████████ | 2838/3123 [6:15:01<37:03,  7.80s/it]
 91%|█████████ | 2839/3123 [6:15:09<37:16,  7.88s/it]
 91%|█████████ | 2840/3123 [6:15:16<36:58,  7.84s/it]
                                                     
{'loss': '0.0002429', 'grad_norm': '0.02173', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '83.23', 'tokens/total': 46536704, 'tokens/trainable': 14581488, 'epoch': '2.727'}
+
 91%|█████████ | 2840/3123 [6:15:17<36:58,  7.84s/it]
 91%|█████████ | 2841/3123 [6:15:24<36:43,  7.81s/it]
 91%|█████████ | 2842/3123 [6:15:32<36:54,  7.88s/it]
 91%|█████████ | 2843/3123 [6:15:40<36:36,  7.84s/it]
 91%|█████████ | 2844/3123 [6:15:48<36:20,  7.82s/it]
 91%|█████████ | 2845/3123 [6:15:56<36:07,  7.80s/it]
 91%|█████████ | 2846/3123 [6:16:04<36:21,  7.88s/it]
 91%|█████████ | 2847/3123 [6:16:11<36:03,  7.84s/it]
 91%|█████████ | 2848/3123 [6:16:19<35:49,  7.81s/it]
 91%|█████████ | 2849/3123 [6:16:27<36:00,  7.88s/it]
 91%|█████████▏| 2850/3123 [6:16:35<35:42,  7.85s/it]
                                                     
{'loss': '0.0001872', 'grad_norm': '0.01624', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '83.81', 'tokens/total': 46700544, 'tokens/trainable': 14632439, 'epoch': '2.736'}
+
 91%|█████████▏| 2850/3123 [6:16:35<35:42,  7.85s/it]
 91%|█████████▏| 2851/3123 [6:16:43<35:27,  7.82s/it]
 91%|█████████▏| 2852/3123 [6:16:51<35:37,  7.89s/it]
 91%|█████████▏| 2853/3123 [6:16:58<35:18,  7.85s/it]
 91%|█████████▏| 2854/3123 [6:17:06<35:03,  7.82s/it]
 91%|█████████▏| 2855/3123 [6:17:14<34:50,  7.80s/it]
 91%|█████████▏| 2856/3123 [6:17:22<35:03,  7.88s/it]
 91%|█████████▏| 2857/3123 [6:17:30<34:45,  7.84s/it]
 92%|█████████▏| 2858/3123 [6:17:38<34:30,  7.81s/it]
 92%|█████████▏| 2859/3123 [6:17:46<34:39,  7.88s/it]
 92%|█████████▏| 2860/3123 [6:17:53<34:21,  7.84s/it]
                                                     
{'loss': '0.0002612', 'grad_norm': '0.01508', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '87.83', 'tokens/total': 46864384, 'tokens/trainable': 14683211, 'epoch': '2.746'}
+
 92%|█████████▏| 2860/3123 [6:17:54<34:21,  7.84s/it]
 92%|█████████▏| 2861/3123 [6:18:01<34:07,  7.82s/it]
 92%|█████████▏| 2862/3123 [6:18:09<33:55,  7.80s/it]
 92%|█████████▏| 2863/3123 [6:18:17<34:07,  7.87s/it]
 92%|█████████▏| 2864/3123 [6:18:25<33:49,  7.84s/it]
 92%|█████████▏| 2865/3123 [6:18:32<33:35,  7.81s/it]
 92%|█████████▏| 2866/3123 [6:18:40<33:45,  7.88s/it]
 92%|█████████▏| 2867/3123 [6:18:48<33:27,  7.84s/it]
 92%|█████████▏| 2868/3123 [6:18:56<33:13,  7.82s/it]
 92%|█████████▏| 2869/3123 [6:19:04<33:01,  7.80s/it]
 92%|█████████▏| 2870/3123 [6:19:12<33:12,  7.87s/it]
                                                     
{'loss': '0.0001987', 'grad_norm': '0.0238', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '79.86', 'tokens/total': 47028224, 'tokens/trainable': 14734325, 'epoch': '2.755'}
+
 92%|█████████▏| 2870/3123 [6:19:12<33:12,  7.87s/it]
 92%|█████████▏| 2871/3123 [6:19:19<32:54,  7.84s/it]
 92%|█████████▏| 2872/3123 [6:19:27<32:41,  7.81s/it]
 92%|█████████▏| 2873/3123 [6:19:35<32:50,  7.88s/it]
 92%|█████████▏| 2874/3123 [6:19:43<32:32,  7.84s/it]
 92%|█████████▏| 2875/3123 [6:19:51<32:18,  7.82s/it]
 92%|█████████▏| 2876/3123 [6:19:59<32:05,  7.80s/it]
 92%|█████████▏| 2877/3123 [6:20:07<32:17,  7.88s/it]
 92%|█████████▏| 2878/3123 [6:20:14<32:00,  7.84s/it]
 92%|█████████▏| 2879/3123 [6:20:22<31:46,  7.81s/it]
 92%|█████████▏| 2880/3123 [6:20:30<31:54,  7.88s/it]
                                                     
{'loss': '0.000153', 'grad_norm': '0.001907', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '87.24', 'tokens/total': 47192064, 'tokens/trainable': 14785260, 'epoch': '2.765'}
+
 92%|█████████▏| 2880/3123 [6:20:30<31:54,  7.88s/it]
 92%|█████████▏| 2881/3123 [6:20:38<31:39,  7.85s/it]
 92%|█████████▏| 2882/3123 [6:20:46<31:23,  7.81s/it]
 92%|█████████▏| 2883/3123 [6:20:53<31:11,  7.80s/it]
 92%|█████████▏| 2884/3123 [6:21:01<31:21,  7.87s/it]
 92%|█████████▏| 2885/3123 [6:21:09<31:04,  7.84s/it]
 92%|█████████▏| 2886/3123 [6:21:17<30:50,  7.81s/it]
 92%|█████████▏| 2887/3123 [6:21:25<31:00,  7.88s/it]
 92%|█████████▏| 2888/3123 [6:21:33<30:43,  7.84s/it]
 93%|█████████▎| 2889/3123 [6:21:41<30:29,  7.82s/it]
 93%|█████████▎| 2890/3123 [6:21:48<30:17,  7.80s/it]
                                                     
{'loss': '0.0002375', 'grad_norm': '0.00531', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '91.71', 'tokens/total': 47355904, 'tokens/trainable': 14836928, 'epoch': '2.775'}
+
 93%|█████████▎| 2890/3123 [6:21:49<30:17,  7.80s/it]
 93%|█████████▎| 2891/3123 [6:21:56<30:26,  7.87s/it]
 93%|█████████▎| 2892/3123 [6:22:04<30:10,  7.84s/it]
 93%|█████████▎| 2893/3123 [6:22:12<29:57,  7.81s/it]
 93%|█████████▎| 2894/3123 [6:22:20<30:05,  7.89s/it]
 93%|█████████▎| 2895/3123 [6:22:28<29:47,  7.84s/it]
 93%|█████████▎| 2896/3123 [6:22:35<29:33,  7.81s/it]
 93%|█████████▎| 2897/3123 [6:22:43<29:21,  7.80s/it]
 93%|█████████▎| 2898/3123 [6:22:51<29:30,  7.87s/it]
 93%|█████████▎| 2899/3123 [6:22:59<29:14,  7.83s/it]
 93%|█████████▎| 2900/3123 [6:23:07<29:01,  7.81s/it]
                                                     
{'loss': '0.000309', 'grad_norm': '0.04785', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '84.82', 'tokens/total': 47519744, 'tokens/trainable': 14888312, 'epoch': '2.784'}
+
 93%|█████████▎| 2900/3123 [6:23:07<29:01,  7.81s/it]
 93%|█████████▎| 2901/3123 [6:23:15<29:09,  7.88s/it]
 93%|█████████▎| 2902/3123 [6:23:22<28:53,  7.84s/it]
 93%|█████████▎| 2903/3123 [6:23:30<28:39,  7.82s/it]
 93%|█████████▎| 2904/3123 [6:23:38<28:27,  7.80s/it]
 93%|█████████▎| 2905/3123 [6:23:46<28:37,  7.88s/it]
 93%|█████████▎| 2906/3123 [6:23:54<28:21,  7.84s/it]
 93%|█████████▎| 2907/3123 [6:24:02<28:07,  7.81s/it]
 93%|█████████▎| 2908/3123 [6:24:10<28:14,  7.88s/it]
 93%|█████████▎| 2909/3123 [6:24:17<27:58,  7.84s/it]
 93%|█████████▎| 2910/3123 [6:24:25<27:44,  7.81s/it]
                                                     
{'loss': '0.0001197', 'grad_norm': '0.02246', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '84.51', 'tokens/total': 47683584, 'tokens/trainable': 14940288, 'epoch': '2.794'}
+
 93%|█████████▎| 2910/3123 [6:24:25<27:44,  7.81s/it]
 93%|█████████▎| 2911/3123 [6:24:33<27:32,  7.80s/it]
 93%|█████████▎| 2912/3123 [6:24:41<27:40,  7.87s/it]
 93%|█████████▎| 2913/3123 [6:24:49<27:25,  7.83s/it]
 93%|█████████▎| 2914/3123 [6:24:56<27:12,  7.81s/it]
 93%|█████████▎| 2915/3123 [6:25:04<27:18,  7.88s/it]
 93%|█████████▎| 2916/3123 [6:25:12<27:02,  7.84s/it]
 93%|█████████▎| 2917/3123 [6:25:20<26:49,  7.81s/it]
 93%|█████████▎| 2918/3123 [6:25:28<26:38,  7.80s/it]
 93%|█████████▎| 2919/3123 [6:25:36<26:45,  7.87s/it]
 93%|█████████▎| 2920/3123 [6:25:43<26:30,  7.84s/it]
                                                     
{'loss': '0.0002271', 'grad_norm': '0.02368', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '80.57', 'tokens/total': 47847424, 'tokens/trainable': 14991148, 'epoch': '2.803'}
+
 93%|█████████▎| 2920/3123 [6:25:44<26:30,  7.84s/it]
 94%|█████████▎| 2921/3123 [6:25:51<26:17,  7.81s/it]
 94%|█████████▎| 2922/3123 [6:25:59<26:23,  7.88s/it]
 94%|█████████▎| 2923/3123 [6:26:07<26:08,  7.84s/it]
 94%|█████████▎| 2924/3123 [6:26:15<25:55,  7.81s/it]
 94%|█████████▎| 2925/3123 [6:26:23<25:43,  7.80s/it]
 94%|█████████▎| 2926/3123 [6:26:31<25:49,  7.87s/it]
 94%|█████████▎| 2927/3123 [6:26:38<25:35,  7.83s/it]
 94%|█████████▍| 2928/3123 [6:26:46<25:23,  7.81s/it]
 94%|█████████▍| 2929/3123 [6:26:54<25:28,  7.88s/it]
 94%|█████████▍| 2930/3123 [6:27:02<25:13,  7.84s/it]
                                                     
{'loss': '0.0004371', 'grad_norm': '0.166', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '77.79', 'tokens/total': 48011264, 'tokens/trainable': 15041868, 'epoch': '2.813'}
+
 94%|█████████▍| 2930/3123 [6:27:02<25:13,  7.84s/it]
 94%|█████████▍| 2931/3123 [6:27:10<25:01,  7.82s/it]
 94%|█████████▍| 2932/3123 [6:27:17<24:49,  7.80s/it]
 94%|█████████▍| 2933/3123 [6:27:25<24:56,  7.88s/it]
 94%|█████████▍| 2934/3123 [6:27:33<24:41,  7.84s/it]
 94%|█████████▍| 2935/3123 [6:27:41<24:29,  7.82s/it]
 94%|█████████▍| 2936/3123 [6:27:49<24:33,  7.88s/it]
 94%|█████████▍| 2937/3123 [6:27:57<24:19,  7.85s/it]
 94%|█████████▍| 2938/3123 [6:28:05<24:06,  7.82s/it]
 94%|█████████▍| 2939/3123 [6:28:12<23:55,  7.80s/it]
 94%|█████████▍| 2940/3123 [6:28:20<24:00,  7.87s/it]
                                                     
{'loss': '0.0001193', 'grad_norm': '0.00354', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '79.97', 'tokens/total': 48175104, 'tokens/trainable': 15092931, 'epoch': '2.823'}
+
 94%|█████████▍| 2940/3123 [6:28:21<24:00,  7.87s/it]
 94%|█████████▍| 2941/3123 [6:28:28<23:46,  7.84s/it]
 94%|█████████▍| 2942/3123 [6:28:36<23:34,  7.82s/it]
 94%|█████████▍| 2943/3123 [6:28:44<23:40,  7.89s/it]
 94%|█████████▍| 2944/3123 [6:28:52<23:26,  7.85s/it]
 94%|█████████▍| 2945/3123 [6:28:59<23:13,  7.83s/it]
 94%|█████████▍| 2946/3123 [6:29:07<23:01,  7.81s/it]
 94%|█████████▍| 2947/3123 [6:29:15<23:06,  7.88s/it]
 94%|█████████▍| 2948/3123 [6:29:23<22:52,  7.84s/it]
 94%|█████████▍| 2949/3123 [6:29:31<22:40,  7.82s/it]
 94%|█████████▍| 2950/3123 [6:29:39<22:45,  7.90s/it]
                                                     
{'loss': '0.0004138', 'grad_norm': '0.1055', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '73.92', 'tokens/total': 48338944, 'tokens/trainable': 15144372, 'epoch': '2.832'}
+
 94%|█████████▍| 2950/3123 [6:29:39<22:45,  7.90s/it]
 94%|█████████▍| 2951/3123 [6:29:47<22:31,  7.86s/it]
 95%|█████████▍| 2952/3123 [6:29:54<22:18,  7.83s/it]
 95%|█████████▍| 2953/3123 [6:30:02<22:07,  7.81s/it]
 95%|█████████▍| 2954/3123 [6:30:10<22:11,  7.88s/it]
 95%|█████████▍| 2955/3123 [6:30:18<21:57,  7.84s/it]
 95%|█████████▍| 2956/3123 [6:30:26<21:45,  7.82s/it]
 95%|█████████▍| 2957/3123 [6:30:34<21:48,  7.89s/it]
 95%|█████████▍| 2958/3123 [6:30:41<21:34,  7.84s/it]
 95%|█████████▍| 2959/3123 [6:30:49<21:22,  7.82s/it]
 95%|█████████▍| 2960/3123 [6:30:57<21:11,  7.80s/it]
                                                     
{'loss': '0.0001791', 'grad_norm': '0.03833', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '81.16', 'tokens/total': 48502784, 'tokens/trainable': 15195361, 'epoch': '2.842'}
+
 95%|█████████▍| 2960/3123 [6:30:57<21:11,  7.80s/it]
 95%|█████████▍| 2961/3123 [6:31:05<21:16,  7.88s/it]
 95%|█████████▍| 2962/3123 [6:31:13<21:02,  7.84s/it]
 95%|█████████▍| 2963/3123 [6:31:21<20:51,  7.82s/it]
 95%|█████████▍| 2964/3123 [6:31:29<20:55,  7.90s/it]
 95%|█████████▍| 2965/3123 [6:31:36<20:41,  7.86s/it]
 95%|█████████▍| 2966/3123 [6:31:44<20:29,  7.83s/it]
 95%|█████████▌| 2967/3123 [6:31:52<20:31,  7.89s/it]
 95%|█████████▌| 2968/3123 [6:32:00<20:17,  7.85s/it]
 95%|█████████▌| 2969/3123 [6:32:08<20:04,  7.82s/it]
 95%|█████████▌| 2970/3123 [6:32:16<19:53,  7.80s/it]
                                                     
{'loss': '0.0002842', 'grad_norm': '0.01575', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '90.08', 'tokens/total': 48666624, 'tokens/trainable': 15247018, 'epoch': '2.851'}
+
 95%|█████████▌| 2970/3123 [6:32:16<19:53,  7.80s/it]
 95%|█████████▌| 2971/3123 [6:32:24<19:57,  7.88s/it]
 95%|█████████▌| 2972/3123 [6:32:31<19:44,  7.84s/it]
 95%|█████████▌| 2973/3123 [6:32:39<19:32,  7.82s/it]
 95%|█████████▌| 2974/3123 [6:32:47<19:34,  7.88s/it]
 95%|█████████▌| 2975/3123 [6:32:55<19:20,  7.84s/it]
 95%|█████████▌| 2976/3123 [6:33:03<19:09,  7.82s/it]
 95%|█████████▌| 2977/3123 [6:33:10<18:58,  7.80s/it]
 95%|█████████▌| 2978/3123 [6:33:18<19:02,  7.88s/it]
 95%|█████████▌| 2979/3123 [6:33:26<18:49,  7.84s/it]
 95%|█████████▌| 2980/3123 [6:33:34<18:37,  7.82s/it]
                                                     
{'loss': '8.735e-05', 'grad_norm': '0.0009003', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '83.09', 'tokens/total': 48830464, 'tokens/trainable': 15297761, 'epoch': '2.861'}
+
 95%|█████████▌| 2980/3123 [6:33:34<18:37,  7.82s/it]
 95%|█████████▌| 2981/3123 [6:33:42<18:39,  7.89s/it]
 95%|█████████▌| 2982/3123 [6:33:50<18:26,  7.85s/it]
 96%|█████████▌| 2983/3123 [6:33:58<18:14,  7.82s/it]
 96%|█████████▌| 2984/3123 [6:34:05<18:04,  7.80s/it]
 96%|█████████▌| 2985/3123 [6:34:13<18:06,  7.88s/it]
 96%|█████████▌| 2986/3123 [6:34:21<17:54,  7.84s/it]
 96%|█████████▌| 2987/3123 [6:34:29<17:43,  7.82s/it]
 96%|█████████▌| 2988/3123 [6:34:37<17:44,  7.89s/it]
 96%|█████████▌| 2989/3123 [6:34:45<17:31,  7.85s/it]
 96%|█████████▌| 2990/3123 [6:34:52<17:20,  7.82s/it]
                                                     
{'loss': '0.0001495', 'grad_norm': '0.05713', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '84.14', 'tokens/total': 48994304, 'tokens/trainable': 15349517, 'epoch': '2.871'}
+
 96%|█████████▌| 2990/3123 [6:34:53<17:20,  7.82s/it]
 96%|█████████▌| 2991/3123 [6:35:00<17:09,  7.80s/it]
 96%|█████████▌| 2992/3123 [6:35:08<17:11,  7.88s/it]
 96%|█████████▌| 2993/3123 [6:35:16<16:59,  7.84s/it]
 96%|█████████▌| 2994/3123 [6:35:24<16:48,  7.82s/it]
 96%|█████████▌| 2995/3123 [6:35:32<16:49,  7.89s/it]
 96%|█████████▌| 2996/3123 [6:35:40<16:36,  7.85s/it]
 96%|█████████▌| 2997/3123 [6:35:47<16:25,  7.82s/it]
 96%|█████████▌| 2998/3123 [6:35:55<16:15,  7.80s/it]
 96%|█████████▌| 2999/3123 [6:36:03<16:16,  7.87s/it]
 96%|█████████▌| 3000/3123 [6:36:11<16:04,  7.84s/it]
                                                     
{'loss': '0.0003364', 'grad_norm': '0.0918', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '90.48', 'tokens/total': 49158144, 'tokens/trainable': 15400474, 'epoch': '2.88'}
+
 96%|█████████▌| 3000/3123 [6:36:11<16:04,  7.84s/it]
 96%|█████████▌| 3001/3123 [6:36:19<15:53,  7.82s/it]
 96%|█████████▌| 3002/3123 [6:36:27<15:54,  7.89s/it]
 96%|█████████▌| 3003/3123 [6:36:34<15:41,  7.85s/it]
 96%|█████████▌| 3004/3123 [6:36:42<15:30,  7.82s/it]
 96%|█████████▌| 3005/3123 [6:36:50<15:20,  7.80s/it]
 96%|█████████▋| 3006/3123 [6:36:58<15:21,  7.88s/it]
 96%|█████████▋| 3007/3123 [6:37:06<15:09,  7.84s/it]
 96%|█████████▋| 3008/3123 [6:37:14<14:58,  7.81s/it]
 96%|█████████▋| 3009/3123 [6:37:22<14:59,  7.89s/it]
 96%|█████████▋| 3010/3123 [6:37:29<14:46,  7.85s/it]
                                                     
{'loss': '0.0002618', 'grad_norm': '0.02393', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '82.19', 'tokens/total': 49321984, 'tokens/trainable': 15451955, 'epoch': '2.89'}
+
 96%|█████████▋| 3010/3123 [6:37:30<14:46,  7.85s/it]
 96%|█████████▋| 3011/3123 [6:37:37<14:36,  7.82s/it]
 96%|█████████▋| 3012/3123 [6:37:45<14:25,  7.80s/it]
 96%|█████████▋| 3013/3123 [6:37:53<14:26,  7.88s/it]
 97%|█████████▋| 3014/3123 [6:38:01<14:14,  7.84s/it]
 97%|█████████▋| 3015/3123 [6:38:08<14:04,  7.82s/it]
 97%|█████████▋| 3016/3123 [6:38:16<14:03,  7.88s/it]
 97%|█████████▋| 3017/3123 [6:38:24<13:51,  7.85s/it]
 97%|█████████▋| 3018/3123 [6:38:32<13:40,  7.82s/it]
 97%|█████████▋| 3019/3123 [6:38:40<13:31,  7.80s/it]
 97%|█████████▋| 3020/3123 [6:38:48<13:31,  7.88s/it]
                                                     
{'loss': '0.0007522', 'grad_norm': '0.167', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '75.88', 'tokens/total': 49485824, 'tokens/trainable': 15502869, 'epoch': '2.899'}
+
 97%|█████████▋| 3020/3123 [6:38:48<13:31,  7.88s/it]
 97%|█████████▋| 3021/3123 [6:38:56<13:19,  7.84s/it]
 97%|█████████▋| 3022/3123 [6:39:03<13:09,  7.82s/it]
 97%|█████████▋| 3023/3123 [6:39:11<13:08,  7.89s/it]
 97%|█████████▋| 3024/3123 [6:39:19<12:57,  7.85s/it]
 97%|█████████▋| 3025/3123 [6:39:27<12:46,  7.82s/it]
 97%|█████████▋| 3026/3123 [6:39:35<12:45,  7.89s/it]
 97%|█████████▋| 3027/3123 [6:39:43<12:33,  7.85s/it]
 97%|█████████▋| 3028/3123 [6:39:50<12:22,  7.82s/it]
 97%|█████████▋| 3029/3123 [6:39:58<12:13,  7.80s/it]
 97%|█████████▋| 3030/3123 [6:40:06<12:12,  7.87s/it]
                                                     
{'loss': '0.0006199', 'grad_norm': '0.002472', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '94.38', 'tokens/total': 49649664, 'tokens/trainable': 15554536, 'epoch': '2.909'}
+
 97%|█████████▋| 3030/3123 [6:40:07<12:12,  7.87s/it]
 97%|█████████▋| 3031/3123 [6:40:14<12:00,  7.84s/it]
 97%|█████████▋| 3032/3123 [6:40:22<11:50,  7.81s/it]
 97%|█████████▋| 3033/3123 [6:40:30<11:49,  7.88s/it]
 97%|█████████▋| 3034/3123 [6:40:38<11:37,  7.84s/it]
 97%|█████████▋| 3035/3123 [6:40:45<11:27,  7.81s/it]
 97%|█████████▋| 3036/3123 [6:40:53<11:18,  7.80s/it]
 97%|█████████▋| 3037/3123 [6:41:01<11:16,  7.87s/it]
 97%|█████████▋| 3038/3123 [6:41:09<11:05,  7.84s/it]
 97%|█████████▋| 3039/3123 [6:41:17<10:56,  7.81s/it]
 97%|█████████▋| 3040/3123 [6:41:25<10:54,  7.88s/it]
                                                     
{'loss': '0.000421', 'grad_norm': '0.1523', 'learning_rate': '0.0001', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '88', 'tokens/total': 49813504, 'tokens/trainable': 15606226, 'epoch': '2.919'}
+
 97%|█████████▋| 3040/3123 [6:41:25<10:54,  7.88s/it]
 97%|█████████▋| 3041/3123 [6:41:32<10:43,  7.85s/it]
 97%|█████████▋| 3042/3123 [6:41:40<10:33,  7.82s/it]
 97%|█████████▋| 3043/3123 [6:41:48<10:23,  7.80s/it]
 97%|█████████▋| 3044/3123 [6:41:56<10:22,  7.88s/it]
 98%|█████████▊| 3045/3123 [6:42:04<10:11,  7.84s/it]
 98%|█████████▊| 3046/3123 [6:42:11<10:01,  7.82s/it]
 98%|█████████▊| 3047/3123 [6:42:19<09:58,  7.88s/it]
 98%|█████████▊| 3048/3123 [6:42:27<09:48,  7.84s/it]
 98%|█████████▊| 3049/3123 [6:42:35<09:38,  7.82s/it]
 98%|█████████▊| 3050/3123 [6:42:43<09:29,  7.80s/it]
                                                     
{'loss': '0.0008211', 'grad_norm': '0.127', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '78.48', 'tokens/total': 49977344, 'tokens/trainable': 15657506, 'epoch': '2.928'}
+
 98%|█████████▊| 3050/3123 [6:42:43<09:29,  7.80s/it]
 98%|█████████▊| 3051/3123 [6:42:51<09:26,  7.87s/it]
 98%|█████████▊| 3052/3123 [6:42:59<09:16,  7.83s/it]
 98%|█████████▊| 3053/3123 [6:43:06<09:06,  7.81s/it]
 98%|█████████▊| 3054/3123 [6:43:14<09:03,  7.87s/it]
 98%|█████████▊| 3055/3123 [6:43:22<08:52,  7.84s/it]
 98%|█████████▊| 3056/3123 [6:43:30<08:43,  7.81s/it]
 98%|█████████▊| 3057/3123 [6:43:38<08:34,  7.79s/it]
 98%|█████████▊| 3058/3123 [6:43:46<08:31,  7.86s/it]
 98%|█████████▊| 3059/3123 [6:43:53<08:21,  7.83s/it]
 98%|█████████▊| 3060/3123 [6:44:01<08:11,  7.80s/it]
                                                     
{'loss': '0.0009389', 'grad_norm': '0.01685', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '77.16', 'tokens/total': 50141184, 'tokens/trainable': 15707913, 'epoch': '2.938'}
+
 98%|█████████▊| 3060/3123 [6:44:01<08:11,  7.80s/it]
 98%|█████████▊| 3061/3123 [6:44:09<08:08,  7.87s/it]
 98%|█████████▊| 3062/3123 [6:44:17<07:58,  7.84s/it]
 98%|█████████▊| 3063/3123 [6:44:25<07:48,  7.81s/it]
 98%|█████████▊| 3064/3123 [6:44:32<07:39,  7.79s/it]
 98%|█████████▊| 3065/3123 [6:44:40<07:36,  7.87s/it]
 98%|█████████▊| 3066/3123 [6:44:48<07:26,  7.83s/it]
 98%|█████████▊| 3067/3123 [6:44:56<07:17,  7.81s/it]
 98%|█████████▊| 3068/3123 [6:45:04<07:13,  7.88s/it]
 98%|█████████▊| 3069/3123 [6:45:12<07:03,  7.84s/it]
 98%|█████████▊| 3070/3123 [6:45:19<06:54,  7.81s/it]
                                                     
{'loss': '0.001051', 'grad_norm': '0.1094', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '87.27', 'tokens/total': 50305024, 'tokens/trainable': 15759273, 'epoch': '2.947'}
+
 98%|█████████▊| 3070/3123 [6:45:20<06:54,  7.81s/it]
 98%|█████████▊| 3071/3123 [6:45:28<06:50,  7.88s/it]
 98%|█████████▊| 3072/3123 [6:45:35<06:40,  7.84s/it]
 98%|█████████▊| 3073/3123 [6:45:43<06:30,  7.82s/it]
 98%|█████████▊| 3074/3123 [6:45:51<06:21,  7.79s/it]
 98%|█████████▊| 3075/3123 [6:45:59<06:17,  7.86s/it]
 98%|█████████▊| 3076/3123 [6:46:07<06:08,  7.83s/it]
 99%|█████████▊| 3077/3123 [6:46:14<05:59,  7.81s/it]
 99%|█████████▊| 3078/3123 [6:46:22<05:54,  7.88s/it]
 99%|█████████▊| 3079/3123 [6:46:30<05:44,  7.84s/it]
 99%|█████████▊| 3080/3123 [6:46:38<05:36,  7.82s/it]
                                                     
{'loss': '0.0006491', 'grad_norm': '0.1221', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '89.09', 'tokens/total': 50468864, 'tokens/trainable': 15810911, 'epoch': '2.957'}
+
 99%|█████████▊| 3080/3123 [6:46:38<05:36,  7.82s/it]
 99%|█████████▊| 3081/3123 [6:46:46<05:27,  7.80s/it]
 99%|█████████▊| 3082/3123 [6:46:54<05:22,  7.87s/it]
 99%|█████████▊| 3083/3123 [6:47:01<05:13,  7.83s/it]
 99%|█████████▉| 3084/3123 [6:47:09<05:04,  7.81s/it]
 99%|█████████▉| 3085/3123 [6:47:17<04:59,  7.89s/it]
 99%|█████████▉| 3086/3123 [6:47:25<04:50,  7.85s/it]
 99%|█████████▉| 3087/3123 [6:47:33<04:41,  7.82s/it]
 99%|█████████▉| 3088/3123 [6:47:40<04:32,  7.80s/it]
 99%|█████████▉| 3089/3123 [6:47:49<04:27,  7.87s/it]
 99%|█████████▉| 3090/3123 [6:47:56<04:18,  7.84s/it]
                                                     
{'loss': '0.0006357', 'grad_norm': '0.05225', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '88.06', 'tokens/total': 50632704, 'tokens/trainable': 15862299, 'epoch': '2.967'}
+
 99%|█████████▉| 3090/3123 [6:47:57<04:18,  7.84s/it]
 99%|█████████▉| 3091/3123 [6:48:04<04:10,  7.81s/it]
 99%|█████████▉| 3092/3123 [6:48:12<04:04,  7.88s/it]
 99%|█████████▉| 3093/3123 [6:48:20<03:55,  7.84s/it]
 99%|█████████▉| 3094/3123 [6:48:28<03:46,  7.81s/it]
 99%|█████████▉| 3095/3123 [6:48:35<03:38,  7.80s/it]
 99%|█████████▉| 3096/3123 [6:48:43<03:32,  7.87s/it]
 99%|█████████▉| 3097/3123 [6:48:51<03:23,  7.84s/it]
 99%|█████████▉| 3098/3123 [6:48:59<03:15,  7.82s/it]
 99%|█████████▉| 3099/3123 [6:49:07<03:09,  7.88s/it]
 99%|█████████▉| 3100/3123 [6:49:15<03:00,  7.84s/it]
                                                     
{'loss': '0.0005909', 'grad_norm': '0.1006', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '91.16', 'tokens/total': 50796544, 'tokens/trainable': 15914018, 'epoch': '2.976'}
+
 99%|█████████▉| 3100/3123 [6:49:15<03:00,  7.84s/it]
 99%|█████████▉| 3101/3123 [6:49:22<02:51,  7.82s/it]
 99%|█████████▉| 3102/3123 [6:49:30<02:43,  7.80s/it]
 99%|█████████▉| 3103/3123 [6:49:38<02:37,  7.88s/it]
 99%|█████████▉| 3104/3123 [6:49:46<02:28,  7.84s/it]
 99%|█████████▉| 3105/3123 [6:49:54<02:20,  7.82s/it]
 99%|█████████▉| 3106/3123 [6:50:02<02:14,  7.89s/it]
 99%|█████████▉| 3107/3123 [6:50:10<02:05,  7.85s/it]
100%|█████████▉| 3108/3123 [6:50:17<01:57,  7.82s/it]
100%|█████████▉| 3109/3123 [6:50:25<01:49,  7.80s/it]
100%|█████████▉| 3110/3123 [6:50:33<01:42,  7.87s/it]
                                                     
{'loss': '0.0005775', 'grad_norm': '0.1279', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '81.27', 'tokens/total': 50960384, 'tokens/trainable': 15965278, 'epoch': '2.986'}
+
100%|█████████▉| 3110/3123 [6:50:33<01:42,  7.87s/it]
100%|█████████▉| 3111/3123 [6:50:41<01:34,  7.84s/it]
100%|█████████▉| 3112/3123 [6:50:49<01:25,  7.81s/it]
100%|█████████▉| 3113/3123 [6:50:57<01:18,  7.89s/it]
100%|█████████▉| 3114/3123 [6:51:04<01:10,  7.85s/it]
100%|█████████▉| 3115/3123 [6:51:12<01:02,  7.82s/it]
100%|█████████▉| 3116/3123 [6:51:20<00:54,  7.80s/it]
100%|█████████▉| 3117/3123 [6:51:28<00:47,  7.88s/it]
100%|█████████▉| 3118/3123 [6:51:36<00:39,  7.84s/it]
100%|█████████▉| 3119/3123 [6:51:44<00:31,  7.82s/it]
100%|█████████▉| 3120/3123 [6:51:52<00:23,  7.89s/it]
                                                     
{'loss': '0.0006674', 'grad_norm': '0.09033', 'learning_rate': '0.0001', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '90.89', 'tokens/total': 51124224, 'tokens/trainable': 16016191, 'epoch': '2.995'}
+
100%|█████████▉| 3120/3123 [6:51:52<00:23,  7.89s/it]
100%|█████████▉| 3121/3123 [6:51:59<00:15,  7.85s/it]
100%|█████████▉| 3122/3123 [6:52:07<00:07,  7.82s/it]
100%|██████████| 3123/3123 [6:52:15<00:00,  7.89s/it][2026-03-08 22:49:29,646] [INFO] [axolotl.core.trainers.base._save:721] [PID:1659682] Saving model checkpoint to /home/jiaruil5/math_rl/mix_teachers/r3lit_rl/mix_teachers/checkpoints/math_operations/primitive_atomic_full_sft_50k_lr1e4_t20260308/checkpoint-3123
+
+
Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]
+
Writing model shards: 100%|██████████| 1/1 [00:25<00:00, 25.97s/it]
Writing model shards: 100%|██████████| 1/1 [00:25<00:00, 25.97s/it]
+
                                                     
{'train_runtime': '2.483e+04', 'train_samples_per_second': '1.006', 'train_steps_per_second': '0.126', 'train_loss': '0.00887', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.97', 'epoch': '2.998', 'tokens/train_per_sec_per_gpu': '91.62'}
+
100%|██████████| 3123/3123 [6:53:30<00:00,  7.89s/it]
100%|██████████| 3123/3123 [6:53:30<00:00,  7.94s/it]Traceback (most recent call last):
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/multiprocessing/util.py", line 300, in _run_finalizers
+    finalizer()
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/multiprocessing/util.py", line 224, in __call__
+    res = self._callback(*self._args, **self._kwargs)
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/multiprocessing/util.py", line 133, in _remove_temp_dir
+    rmtree(tempdir)
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/shutil.py", line 725, in rmtree
+    _rmtree_safe_fd(fd, path, onerror)
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/shutil.py", line 681, in _rmtree_safe_fd
+    onerror(os.unlink, fullname, sys.exc_info())
+  File "/data/user_data/jiaruil5/miniconda3/envs/axolotl/lib/python3.10/shutil.py", line 679, in _rmtree_safe_fd
+    os.unlink(entry.name, dir_fd=topfd)
+OSError: [Errno 16] Device or resource busy: '.nfs00000000000da9d400012515'
+
+[2026-03-08 22:50:44,311] [INFO] [axolotl.train.save_trained_model:226] [PID:1659682] Training completed! Saving trained model to /home/jiaruil5/math_rl/mix_teachers/r3lit_rl/mix_teachers/checkpoints/math_operations/primitive_atomic_full_sft_50k_lr1e4_t20260308.
+
Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]
Writing model shards: 100%|██████████| 1/1 [00:22<00:00, 22.28s/it]
Writing model shards: 100%|██████████| 1/1 [00:22<00:00, 22.28s/it]
+[2026-03-08 22:51:06,643] [INFO] [axolotl.train.save_trained_model:340] [PID:1659682] Model successfully saved to /home/jiaruil5/math_rl/mix_teachers/r3lit_rl/mix_teachers/checkpoints/math_operations/primitive_atomic_full_sft_50k_lr1e4_t20260308