Upload folder using huggingface_hub

CREAM/outputs/sft/qwen_short_cot-lora_None-ckpt_None-25-10-06-01_35_46/00args.json

@@ -0,0 +1,43 @@
+{
+    "epoch": 3,
+    "train_bsz": 1,
+    "eval_bsz": 2,
+    "lr": 5e-06,
+    "deepspeed": "./configs/ds_stage2.json",
+    "weight_decay": 0.01,
+    "save_eval_step_ratio": 0.999999,
+    "warmup_step_ratio": 0.1,
+    "grad_checkpointing": true,
+    "model": "qwen14b",
+    "common": {
+        "debug": false,
+        "device": "0,1,2,3,4,5,6,7",
+        "world_size": 8,
+        "rank": 0,
+        "master_address": "localhost",
+        "master_port": 43485,
+        "bf16": true,
+        "wandb_project_name": "CREAM",
+        "wandb_entity_name": "your_wandb_entity_name",
+        "run_name": "qwen_short_cot-lora_None-ckpt_None-25-10-06-01_35_46",
+        "output_dir": "outputs/sft/qwen_short_cot-lora_None-ckpt_None-25-10-06-01_35_46",
+        "load_args_path": null
+    },
+    "checkpoint": null,
+    "train_stage": "sft",
+    "dataset": {
+        "name": "qwen_short_cot",
+        "limit_size": null,
+        "max_length": 4096
+    },
+    "lora": {
+        "enable": false,
+        "alpha": 64,
+        "r": 32,
+        "dropout": 0.1
+    },
+    "dpo": {
+        "beta": 0.1,
+        "method": "original"
+    }
+}

CREAM/outputs/sft/qwen_short_cot-lora_None-ckpt_None-25-10-06-01_35_46/checkpoint-15066/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

CREAM/outputs/sft/qwen_short_cot-lora_None-ckpt_None-25-10-06-01_35_46/checkpoint-15066/config.json

@@ -0,0 +1,28 @@
+{
+  "_name_or_path": "Qwen/Qwen2.5-14B-Instruct",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 5120,
+  "initializer_range": 0.02,
+  "intermediate_size": 13824,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 70,
+  "model_type": "qwen2",
+  "num_attention_heads": 40,
+  "num_hidden_layers": 48,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.44.2",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 152064
+}

CREAM/outputs/sft/qwen_short_cot-lora_None-ckpt_None-25-10-06-01_35_46/checkpoint-15066/generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.05,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8,
+  "transformers_version": "4.44.2"
+}

CREAM/outputs/sft/qwen_short_cot-lora_None-ckpt_None-25-10-06-01_35_46/checkpoint-15066/model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e37d654e9452a48265132887269004c24b05eb871af5650dff128c9ccd75101
+size 9941058640

CREAM/outputs/sft/qwen_short_cot-lora_None-ckpt_None-25-10-06-01_35_46/checkpoint-15066/model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:038f6c1a134f3c4d1acdcf06e0fdaa4231f9c7e7f245abc4b16966918805535a
+size 9909694792

CREAM/outputs/sft/qwen_short_cot-lora_None-ckpt_None-25-10-06-01_35_46/checkpoint-15066/model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:802a2c422675f3b719efb6aa54ee1b6df24f7a1790c4400f03d2e329e662433a
+size 9689380560

CREAM/outputs/sft/qwen_short_cot-lora_None-ckpt_None-25-10-06-01_35_46/checkpoint-15066/model.safetensors.index.json

@@ -0,0 +1,586 @@
+{
+  "metadata": {
+    "total_size": 29540067328
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.33.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.34.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.34.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.34.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.35.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.35.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.35.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.36.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.36.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.36.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.36.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.36.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.36.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.36.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.36.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.36.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.36.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.36.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.36.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.37.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.37.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.37.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.37.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.37.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.37.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.37.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.37.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.37.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.37.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.37.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.37.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.38.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.38.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.38.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.38.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.38.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.38.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.38.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.38.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.38.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.38.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.38.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.38.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.39.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.39.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.39.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.39.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.39.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.39.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.39.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.39.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.39.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.39.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.39.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.39.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.40.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.40.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.40.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.40.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.40.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.40.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.40.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.40.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.40.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.40.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.40.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.40.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.41.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.41.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.41.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.41.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.41.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.41.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.41.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.41.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.41.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.41.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.41.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.41.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.42.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.42.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.42.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.42.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.42.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.42.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.42.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.42.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.42.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.42.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.42.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.42.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.43.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.43.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.43.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.43.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.43.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.43.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.43.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.43.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.43.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.43.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.43.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.43.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.44.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.44.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.44.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.44.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.44.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.44.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.44.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.44.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.44.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.44.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.44.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.44.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.45.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.45.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.45.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.45.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.45.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.45.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.45.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.45.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.45.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.45.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.45.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.45.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.46.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.46.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.46.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.46.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.46.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.46.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.46.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.46.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.46.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.46.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.46.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.46.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.47.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.47.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.47.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.47.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.47.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.47.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.47.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.47.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.47.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.47.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.47.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.47.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

CREAM/outputs/sft/qwen_short_cot-lora_None-ckpt_None-25-10-06-01_35_46/checkpoint-15066/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

CREAM/outputs/sft/qwen_short_cot-lora_None-ckpt_None-25-10-06-01_35_46/checkpoint-15066/tokenizer_config.json

@@ -0,0 +1,207 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

CREAM/outputs/sft/qwen_short_cot-lora_None-ckpt_None-25-10-06-01_35_46/checkpoint-15066/trainer_state.json

@@ -0,0 +1,3330 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 15066,
+  "global_step": 15066,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "grad_norm": 15.861784934997559,
+      "learning_rate": 3.3178500331785007e-09,
+      "loss": 2.1545,
+      "step": 1
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 17.788606643676758,
+      "learning_rate": 1.0617120106171202e-07,
+      "loss": 2.3653,
+      "step": 32
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 16.857004165649414,
+      "learning_rate": 2.1234240212342404e-07,
+      "loss": 2.1664,
+      "step": 64
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 8.184218406677246,
+      "learning_rate": 3.185136031851361e-07,
+      "loss": 1.9016,
+      "step": 96
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 11.089315414428711,
+      "learning_rate": 4.246848042468481e-07,
+      "loss": 1.2687,
+      "step": 128
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 2.3433008193969727,
+      "learning_rate": 5.308560053085602e-07,
+      "loss": 0.7031,
+      "step": 160
+    },
+    {
+      "epoch": 0.04,
+      "grad_norm": 2.8761188983917236,
+      "learning_rate": 6.370272063702722e-07,
+      "loss": 0.6355,
+      "step": 192
+    },
+    {
+      "epoch": 0.04,
+      "grad_norm": 1.8086258172988892,
+      "learning_rate": 7.431984074319841e-07,
+      "loss": 0.6157,
+      "step": 224
+    },
+    {
+      "epoch": 0.05,
+      "grad_norm": 1.3361790180206299,
+      "learning_rate": 8.493696084936962e-07,
+      "loss": 0.5823,
+      "step": 256
+    },
+    {
+      "epoch": 0.06,
+      "grad_norm": 1.6081678867340088,
+      "learning_rate": 9.555408095554083e-07,
+      "loss": 0.5539,
+      "step": 288
+    },
+    {
+      "epoch": 0.06,
+      "grad_norm": 6.2013325691223145,
+      "learning_rate": 1.0617120106171203e-06,
+      "loss": 0.5335,
+      "step": 320
+    },
+    {
+      "epoch": 0.07,
+      "grad_norm": 1.5980337858200073,
+      "learning_rate": 1.1678832116788322e-06,
+      "loss": 0.5306,
+      "step": 352
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 1.3959029912948608,
+      "learning_rate": 1.2740544127405444e-06,
+      "loss": 0.5101,
+      "step": 384
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 1.574424386024475,
+      "learning_rate": 1.3802256138022562e-06,
+      "loss": 0.5119,
+      "step": 416
+    },
+    {
+      "epoch": 0.09,
+      "grad_norm": 1.6043046712875366,
+      "learning_rate": 1.4863968148639683e-06,
+      "loss": 0.4991,
+      "step": 448
+    },
+    {
+      "epoch": 0.1,
+      "grad_norm": 1.544350028038025,
+      "learning_rate": 1.5925680159256803e-06,
+      "loss": 0.4923,
+      "step": 480
+    },
+    {
+      "epoch": 0.1,
+      "grad_norm": 1.3767935037612915,
+      "learning_rate": 1.6987392169873923e-06,
+      "loss": 0.493,
+      "step": 512
+    },
+    {
+      "epoch": 0.11,
+      "grad_norm": 1.2990076541900635,
+      "learning_rate": 1.8049104180491042e-06,
+      "loss": 0.4829,
+      "step": 544
+    },
+    {
+      "epoch": 0.11,
+      "grad_norm": 1.4568612575531006,
+      "learning_rate": 1.9110816191108166e-06,
+      "loss": 0.4707,
+      "step": 576
+    },
+    {
+      "epoch": 0.12,
+      "grad_norm": 1.4638981819152832,
+      "learning_rate": 2.0172528201725284e-06,
+      "loss": 0.4694,
+      "step": 608
+    },
+    {
+      "epoch": 0.13,
+      "grad_norm": 1.5782190561294556,
+      "learning_rate": 2.1234240212342407e-06,
+      "loss": 0.4664,
+      "step": 640
+    },
+    {
+      "epoch": 0.13,
+      "grad_norm": 1.4637874364852905,
+      "learning_rate": 2.2295952222959525e-06,
+      "loss": 0.4602,
+      "step": 672
+    },
+    {
+      "epoch": 0.14,
+      "grad_norm": 1.2158172130584717,
+      "learning_rate": 2.3357664233576643e-06,
+      "loss": 0.4595,
+      "step": 704
+    },
+    {
+      "epoch": 0.15,
+      "grad_norm": 1.7185492515563965,
+      "learning_rate": 2.4419376244193766e-06,
+      "loss": 0.4526,
+      "step": 736
+    },
+    {
+      "epoch": 0.15,
+      "grad_norm": 1.2440986633300781,
+      "learning_rate": 2.548108825481089e-06,
+      "loss": 0.4523,
+      "step": 768
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 1.3548916578292847,
+      "learning_rate": 2.6542800265428002e-06,
+      "loss": 0.4464,
+      "step": 800
+    },
+    {
+      "epoch": 0.17,
+      "grad_norm": 1.7101390361785889,
+      "learning_rate": 2.7604512276045125e-06,
+      "loss": 0.4465,
+      "step": 832
+    },
+    {
+      "epoch": 0.17,
+      "grad_norm": 1.2852809429168701,
+      "learning_rate": 2.8666224286662247e-06,
+      "loss": 0.4371,
+      "step": 864
+    },
+    {
+      "epoch": 0.18,
+      "grad_norm": 1.0453855991363525,
+      "learning_rate": 2.9727936297279365e-06,
+      "loss": 0.4419,
+      "step": 896
+    },
+    {
+      "epoch": 0.18,
+      "grad_norm": 1.2790417671203613,
+      "learning_rate": 3.0789648307896488e-06,
+      "loss": 0.4222,
+      "step": 928
+    },
+    {
+      "epoch": 0.19,
+      "grad_norm": 1.1465721130371094,
+      "learning_rate": 3.1851360318513606e-06,
+      "loss": 0.4409,
+      "step": 960
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 1.2378520965576172,
+      "learning_rate": 3.2913072329130724e-06,
+      "loss": 0.4387,
+      "step": 992
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 1.2385942935943604,
+      "learning_rate": 3.3974784339747847e-06,
+      "loss": 0.4349,
+      "step": 1024
+    },
+    {
+      "epoch": 0.21,
+      "grad_norm": 1.356612205505371,
+      "learning_rate": 3.503649635036497e-06,
+      "loss": 0.4274,
+      "step": 1056
+    },
+    {
+      "epoch": 0.22,
+      "grad_norm": 1.1762096881866455,
+      "learning_rate": 3.6098208360982083e-06,
+      "loss": 0.4224,
+      "step": 1088
+    },
+    {
+      "epoch": 0.22,
+      "grad_norm": 1.3073230981826782,
+      "learning_rate": 3.7159920371599206e-06,
+      "loss": 0.4249,
+      "step": 1120
+    },
+    {
+      "epoch": 0.23,
+      "grad_norm": 1.12436842918396,
+      "learning_rate": 3.822163238221633e-06,
+      "loss": 0.4136,
+      "step": 1152
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 1.2152477502822876,
+      "learning_rate": 3.928334439283345e-06,
+      "loss": 0.4265,
+      "step": 1184
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 1.4322705268859863,
+      "learning_rate": 4.034505640345057e-06,
+      "loss": 0.4228,
+      "step": 1216
+    },
+    {
+      "epoch": 0.25,
+      "grad_norm": 1.2259882688522339,
+      "learning_rate": 4.140676841406769e-06,
+      "loss": 0.423,
+      "step": 1248
+    },
+    {
+      "epoch": 0.25,
+      "grad_norm": 1.4468910694122314,
+      "learning_rate": 4.246848042468481e-06,
+      "loss": 0.4312,
+      "step": 1280
+    },
+    {
+      "epoch": 0.26,
+      "grad_norm": 1.10471510887146,
+      "learning_rate": 4.353019243530193e-06,
+      "loss": 0.4047,
+      "step": 1312
+    },
+    {
+      "epoch": 0.27,
+      "grad_norm": 1.398850917816162,
+      "learning_rate": 4.459190444591905e-06,
+      "loss": 0.4108,
+      "step": 1344
+    },
+    {
+      "epoch": 0.27,
+      "grad_norm": 1.2144904136657715,
+      "learning_rate": 4.565361645653617e-06,
+      "loss": 0.4145,
+      "step": 1376
+    },
+    {
+      "epoch": 0.28,
+      "grad_norm": 1.1715329885482788,
+      "learning_rate": 4.671532846715329e-06,
+      "loss": 0.4197,
+      "step": 1408
+    },
+    {
+      "epoch": 0.29,
+      "grad_norm": 1.1917423009872437,
+      "learning_rate": 4.777704047777041e-06,
+      "loss": 0.4093,
+      "step": 1440
+    },
+    {
+      "epoch": 0.29,
+      "grad_norm": 1.0857406854629517,
+      "learning_rate": 4.883875248838753e-06,
+      "loss": 0.4094,
+      "step": 1472
+    },
+    {
+      "epoch": 0.3,
+      "grad_norm": 1.4424810409545898,
+      "learning_rate": 4.9900464499004645e-06,
+      "loss": 0.4158,
+      "step": 1504
+    },
+    {
+      "epoch": 0.31,
+      "grad_norm": 1.1842881441116333,
+      "learning_rate": 4.999943564964996e-06,
+      "loss": 0.4122,
+      "step": 1536
+    },
+    {
+      "epoch": 0.31,
+      "grad_norm": 1.138179063796997,
+      "learning_rate": 4.999750306706706e-06,
+      "loss": 0.4036,
+      "step": 1568
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 1.3266323804855347,
+      "learning_rate": 4.999419631904584e-06,
+      "loss": 0.4083,
+      "step": 1600
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 1.3375203609466553,
+      "learning_rate": 4.9989515587365245e-06,
+      "loss": 0.4072,
+      "step": 1632
+    },
+    {
+      "epoch": 0.33,
+      "grad_norm": 1.2711100578308105,
+      "learning_rate": 4.9983461129334935e-06,
+      "loss": 0.4047,
+      "step": 1664
+    },
+    {
+      "epoch": 0.34,
+      "grad_norm": 1.3271673917770386,
+      "learning_rate": 4.9976033277781236e-06,
+      "loss": 0.4034,
+      "step": 1696
+    },
+    {
+      "epoch": 0.34,
+      "grad_norm": 1.138608694076538,
+      "learning_rate": 4.9967232441028815e-06,
+      "loss": 0.4083,
+      "step": 1728
+    },
+    {
+      "epoch": 0.35,
+      "grad_norm": 1.1976908445358276,
+      "learning_rate": 4.995705910287821e-06,
+      "loss": 0.3974,
+      "step": 1760
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 1.210739016532898,
+      "learning_rate": 4.994551382257927e-06,
+      "loss": 0.3927,
+      "step": 1792
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 1.2446238994598389,
+      "learning_rate": 4.993259723480039e-06,
+      "loss": 0.4082,
+      "step": 1824
+    },
+    {
+      "epoch": 0.37,
+      "grad_norm": 1.5098798274993896,
+      "learning_rate": 4.991831004959362e-06,
+      "loss": 0.4045,
+      "step": 1856
+    },
+    {
+      "epoch": 0.38,
+      "grad_norm": 1.0394607782363892,
+      "learning_rate": 4.990265305235567e-06,
+      "loss": 0.4007,
+      "step": 1888
+    },
+    {
+      "epoch": 0.38,
+      "grad_norm": 1.268601417541504,
+      "learning_rate": 4.988562710378466e-06,
+      "loss": 0.3961,
+      "step": 1920
+    },
+    {
+      "epoch": 0.39,
+      "grad_norm": 1.0634905099868774,
+      "learning_rate": 4.986723313983288e-06,
+      "loss": 0.398,
+      "step": 1952
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 1.2129184007644653,
+      "learning_rate": 4.984747217165531e-06,
+      "loss": 0.3938,
+      "step": 1984
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 1.0884493589401245,
+      "learning_rate": 4.9826345285554015e-06,
+      "loss": 0.3909,
+      "step": 2016
+    },
+    {
+      "epoch": 0.41,
+      "grad_norm": 1.1943061351776123,
+      "learning_rate": 4.980385364291847e-06,
+      "loss": 0.4054,
+      "step": 2048
+    },
+    {
+      "epoch": 0.41,
+      "grad_norm": 1.081852674484253,
+      "learning_rate": 4.977999848016168e-06,
+      "loss": 0.4064,
+      "step": 2080
+    },
+    {
+      "epoch": 0.42,
+      "grad_norm": 1.2224043607711792,
+      "learning_rate": 4.975478110865223e-06,
+      "loss": 0.3972,
+      "step": 2112
+    },
+    {
+      "epoch": 0.43,
+      "grad_norm": 1.047973394393921,
+      "learning_rate": 4.972820291464219e-06,
+      "loss": 0.3891,
+      "step": 2144
+    },
+    {
+      "epoch": 0.43,
+      "grad_norm": 1.5962953567504883,
+      "learning_rate": 4.97002653591909e-06,
+      "loss": 0.3929,
+      "step": 2176
+    },
+    {
+      "epoch": 0.44,
+      "grad_norm": 1.0555881261825562,
+      "learning_rate": 4.9670969978084695e-06,
+      "loss": 0.4022,
+      "step": 2208
+    },
+    {
+      "epoch": 0.45,
+      "grad_norm": 1.1123954057693481,
+      "learning_rate": 4.964031838175241e-06,
+      "loss": 0.395,
+      "step": 2240
+    },
+    {
+      "epoch": 0.45,
+      "grad_norm": 1.3058280944824219,
+      "learning_rate": 4.960831225517693e-06,
+      "loss": 0.4066,
+      "step": 2272
+    },
+    {
+      "epoch": 0.46,
+      "grad_norm": 1.2261545658111572,
+      "learning_rate": 4.957495335780248e-06,
+      "loss": 0.3947,
+      "step": 2304
+    },
+    {
+      "epoch": 0.47,
+      "grad_norm": 1.0844892263412476,
+      "learning_rate": 4.954024352343798e-06,
+      "loss": 0.3901,
+      "step": 2336
+    },
+    {
+      "epoch": 0.47,
+      "grad_norm": 1.1942882537841797,
+      "learning_rate": 4.950418466015619e-06,
+      "loss": 0.4099,
+      "step": 2368
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 1.4109705686569214,
+      "learning_rate": 4.9466778750188845e-06,
+      "loss": 0.4035,
+      "step": 2400
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 1.1272732019424438,
+      "learning_rate": 4.942802784981766e-06,
+      "loss": 0.3931,
+      "step": 2432
+    },
+    {
+      "epoch": 0.49,
+      "grad_norm": 1.1475287675857544,
+      "learning_rate": 4.938793408926131e-06,
+      "loss": 0.3922,
+      "step": 2464
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 1.1184591054916382,
+      "learning_rate": 4.934649967255834e-06,
+      "loss": 0.3891,
+      "step": 2496
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 1.0866855382919312,
+      "learning_rate": 4.930372687744598e-06,
+      "loss": 0.4029,
+      "step": 2528
+    },
+    {
+      "epoch": 0.51,
+      "grad_norm": 1.1108638048171997,
+      "learning_rate": 4.925961805523494e-06,
+      "loss": 0.3933,
+      "step": 2560
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 1.1988886594772339,
+      "learning_rate": 4.921417563068015e-06,
+      "loss": 0.3774,
+      "step": 2592
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 1.0548280477523804,
+      "learning_rate": 4.9167402101847496e-06,
+      "loss": 0.3876,
+      "step": 2624
+    },
+    {
+      "epoch": 0.53,
+      "grad_norm": 1.2720234394073486,
+      "learning_rate": 4.911930003997645e-06,
+      "loss": 0.3935,
+      "step": 2656
+    },
+    {
+      "epoch": 0.54,
+      "grad_norm": 1.017087459564209,
+      "learning_rate": 4.906987208933874e-06,
+      "loss": 0.3886,
+      "step": 2688
+    },
+    {
+      "epoch": 0.54,
+      "grad_norm": 1.1831703186035156,
+      "learning_rate": 4.901912096709302e-06,
+      "loss": 0.4013,
+      "step": 2720
+    },
+    {
+      "epoch": 0.55,
+      "grad_norm": 1.4336239099502563,
+      "learning_rate": 4.896704946313546e-06,
+      "loss": 0.3886,
+      "step": 2752
+    },
+    {
+      "epoch": 0.55,
+      "grad_norm": 1.3524258136749268,
+      "learning_rate": 4.891366043994641e-06,
+      "loss": 0.3911,
+      "step": 2784
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 1.2585910558700562,
+      "learning_rate": 4.8858956832433e-06,
+      "loss": 0.3985,
+      "step": 2816
+    },
+    {
+      "epoch": 0.57,
+      "grad_norm": 1.2802609205245972,
+      "learning_rate": 4.880294164776785e-06,
+      "loss": 0.3894,
+      "step": 2848
+    },
+    {
+      "epoch": 0.57,
+      "grad_norm": 1.154340386390686,
+      "learning_rate": 4.874561796522377e-06,
+      "loss": 0.3905,
+      "step": 2880
+    },
+    {
+      "epoch": 0.58,
+      "grad_norm": 1.0568464994430542,
+      "learning_rate": 4.8686988936004386e-06,
+      "loss": 0.3875,
+      "step": 2912
+    },
+    {
+      "epoch": 0.59,
+      "grad_norm": 1.0620440244674683,
+      "learning_rate": 4.862705778307105e-06,
+      "loss": 0.3935,
+      "step": 2944
+    },
+    {
+      "epoch": 0.59,
+      "grad_norm": 1.1272939443588257,
+      "learning_rate": 4.856582780096558e-06,
+      "loss": 0.3819,
+      "step": 2976
+    },
+    {
+      "epoch": 0.6,
+      "grad_norm": 1.0595206022262573,
+      "learning_rate": 4.850330235562915e-06,
+      "loss": 0.3876,
+      "step": 3008
+    },
+    {
+      "epoch": 0.61,
+      "grad_norm": 1.1870075464248657,
+      "learning_rate": 4.843948488421728e-06,
+      "loss": 0.3816,
+      "step": 3040
+    },
+    {
+      "epoch": 0.61,
+      "grad_norm": 1.2812995910644531,
+      "learning_rate": 4.837437889491094e-06,
+      "loss": 0.3929,
+      "step": 3072
+    },
+    {
+      "epoch": 0.62,
+      "grad_norm": 1.0136667490005493,
+      "learning_rate": 4.830798796672357e-06,
+      "loss": 0.3826,
+      "step": 3104
+    },
+    {
+      "epoch": 0.62,
+      "grad_norm": 1.0142866373062134,
+      "learning_rate": 4.8240315749304465e-06,
+      "loss": 0.3823,
+      "step": 3136
+    },
+    {
+      "epoch": 0.63,
+      "grad_norm": 1.391076683998108,
+      "learning_rate": 4.817136596273806e-06,
+      "loss": 0.3865,
+      "step": 3168
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 1.0642139911651611,
+      "learning_rate": 4.810114239733948e-06,
+      "loss": 0.387,
+      "step": 3200
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 1.2507323026657104,
+      "learning_rate": 4.802964891344611e-06,
+      "loss": 0.3837,
+      "step": 3232
+    },
+    {
+      "epoch": 0.65,
+      "grad_norm": 1.1517506837844849,
+      "learning_rate": 4.7956889441205525e-06,
+      "loss": 0.3868,
+      "step": 3264
+    },
+    {
+      "epoch": 0.66,
+      "grad_norm": 1.0549813508987427,
+      "learning_rate": 4.7882867980359245e-06,
+      "loss": 0.3883,
+      "step": 3296
+    },
+    {
+      "epoch": 0.66,
+      "grad_norm": 1.1579716205596924,
+      "learning_rate": 4.780758860002303e-06,
+      "loss": 0.3913,
+      "step": 3328
+    },
+    {
+      "epoch": 0.67,
+      "grad_norm": 1.0801048278808594,
+      "learning_rate": 4.77310554384631e-06,
+      "loss": 0.3916,
+      "step": 3360
+    },
+    {
+      "epoch": 0.68,
+      "grad_norm": 0.9929106831550598,
+      "learning_rate": 4.765327270286868e-06,
+      "loss": 0.3839,
+      "step": 3392
+    },
+    {
+      "epoch": 0.68,
+      "grad_norm": 1.0502469539642334,
+      "learning_rate": 4.7574244669120715e-06,
+      "loss": 0.3965,
+      "step": 3424
+    },
+    {
+      "epoch": 0.69,
+      "grad_norm": 1.2819041013717651,
+      "learning_rate": 4.749397568155681e-06,
+      "loss": 0.3893,
+      "step": 3456
+    },
+    {
+      "epoch": 0.69,
+      "grad_norm": 1.3111693859100342,
+      "learning_rate": 4.7412470152732405e-06,
+      "loss": 0.3876,
+      "step": 3488
+    },
+    {
+      "epoch": 0.7,
+      "grad_norm": 0.9922152161598206,
+      "learning_rate": 4.732973256317825e-06,
+      "loss": 0.368,
+      "step": 3520
+    },
+    {
+      "epoch": 0.71,
+      "grad_norm": 1.0946770906448364,
+      "learning_rate": 4.724576746115405e-06,
+      "loss": 0.3793,
+      "step": 3552
+    },
+    {
+      "epoch": 0.71,
+      "grad_norm": 0.9933319091796875,
+      "learning_rate": 4.716057946239845e-06,
+      "loss": 0.3866,
+      "step": 3584
+    },
+    {
+      "epoch": 0.72,
+      "grad_norm": 1.0144922733306885,
+      "learning_rate": 4.7074173249875335e-06,
+      "loss": 0.3927,
+      "step": 3616
+    },
+    {
+      "epoch": 0.73,
+      "grad_norm": 1.1155210733413696,
+      "learning_rate": 4.698655357351633e-06,
+      "loss": 0.397,
+      "step": 3648
+    },
+    {
+      "epoch": 0.73,
+      "grad_norm": 1.0693238973617554,
+      "learning_rate": 4.689772524995977e-06,
+      "loss": 0.3761,
+      "step": 3680
+    },
+    {
+      "epoch": 0.74,
+      "grad_norm": 1.2205610275268555,
+      "learning_rate": 4.680769316228585e-06,
+      "loss": 0.3891,
+      "step": 3712
+    },
+    {
+      "epoch": 0.75,
+      "grad_norm": 0.9990103840827942,
+      "learning_rate": 4.6716462259748195e-06,
+      "loss": 0.3799,
+      "step": 3744
+    },
+    {
+      "epoch": 0.75,
+      "grad_norm": 1.162751317024231,
+      "learning_rate": 4.662403755750185e-06,
+      "loss": 0.3899,
+      "step": 3776
+    },
+    {
+      "epoch": 0.76,
+      "grad_norm": 1.1127140522003174,
+      "learning_rate": 4.653042413632751e-06,
+      "loss": 0.3847,
+      "step": 3808
+    },
+    {
+      "epoch": 0.76,
+      "grad_norm": 1.1862664222717285,
+      "learning_rate": 4.64356271423523e-06,
+      "loss": 0.3975,
+      "step": 3840
+    },
+    {
+      "epoch": 0.77,
+      "grad_norm": 1.087441325187683,
+      "learning_rate": 4.633965178676679e-06,
+      "loss": 0.3733,
+      "step": 3872
+    },
+    {
+      "epoch": 0.78,
+      "grad_norm": 0.9520412087440491,
+      "learning_rate": 4.624250334553862e-06,
+      "loss": 0.3818,
+      "step": 3904
+    },
+    {
+      "epoch": 0.78,
+      "grad_norm": 1.0426771640777588,
+      "learning_rate": 4.6144187159122355e-06,
+      "loss": 0.3844,
+      "step": 3936
+    },
+    {
+      "epoch": 0.79,
+      "grad_norm": 1.1824151277542114,
+      "learning_rate": 4.604470863216604e-06,
+      "loss": 0.3861,
+      "step": 3968
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 1.155782699584961,
+      "learning_rate": 4.594407323321398e-06,
+      "loss": 0.3762,
+      "step": 4000
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 1.1668391227722168,
+      "learning_rate": 4.5842286494406224e-06,
+      "loss": 0.3872,
+      "step": 4032
+    },
+    {
+      "epoch": 0.81,
+      "grad_norm": 1.2379133701324463,
+      "learning_rate": 4.5739354011174355e-06,
+      "loss": 0.3913,
+      "step": 4064
+    },
+    {
+      "epoch": 0.82,
+      "grad_norm": 1.2492693662643433,
+      "learning_rate": 4.563528144193395e-06,
+      "loss": 0.3909,
+      "step": 4096
+    },
+    {
+      "epoch": 0.82,
+      "grad_norm": 1.0948158502578735,
+      "learning_rate": 4.5530074507773535e-06,
+      "loss": 0.3867,
+      "step": 4128
+    },
+    {
+      "epoch": 0.83,
+      "grad_norm": 1.0528464317321777,
+      "learning_rate": 4.542373899214006e-06,
+      "loss": 0.3776,
+      "step": 4160
+    },
+    {
+      "epoch": 0.83,
+      "grad_norm": 1.0977307558059692,
+      "learning_rate": 4.531628074052096e-06,
+      "loss": 0.38,
+      "step": 4192
+    },
+    {
+      "epoch": 0.84,
+      "grad_norm": 1.07651686668396,
+      "learning_rate": 4.5207705660122855e-06,
+      "loss": 0.3761,
+      "step": 4224
+    },
+    {
+      "epoch": 0.85,
+      "grad_norm": 1.13429856300354,
+      "learning_rate": 4.509801971954681e-06,
+      "loss": 0.3683,
+      "step": 4256
+    },
+    {
+      "epoch": 0.85,
+      "grad_norm": 1.122430682182312,
+      "learning_rate": 4.4987228948460166e-06,
+      "loss": 0.3787,
+      "step": 4288
+    },
+    {
+      "epoch": 0.86,
+      "grad_norm": 1.0269503593444824,
+      "learning_rate": 4.487533943726518e-06,
+      "loss": 0.3826,
+      "step": 4320
+    },
+    {
+      "epoch": 0.87,
+      "grad_norm": 1.30316162109375,
+      "learning_rate": 4.476235733676412e-06,
+      "loss": 0.3784,
+      "step": 4352
+    },
+    {
+      "epoch": 0.87,
+      "grad_norm": 1.0606482028961182,
+      "learning_rate": 4.4648288857821245e-06,
+      "loss": 0.3714,
+      "step": 4384
+    },
+    {
+      "epoch": 0.88,
+      "grad_norm": 1.002287745475769,
+      "learning_rate": 4.453314027102128e-06,
+      "loss": 0.3911,
+      "step": 4416
+    },
+    {
+      "epoch": 0.89,
+      "grad_norm": 0.965377688407898,
+      "learning_rate": 4.441691790632479e-06,
+      "loss": 0.3744,
+      "step": 4448
+    },
+    {
+      "epoch": 0.89,
+      "grad_norm": 0.940441370010376,
+      "learning_rate": 4.429962815272014e-06,
+      "loss": 0.3823,
+      "step": 4480
+    },
+    {
+      "epoch": 0.9,
+      "grad_norm": 1.1644710302352905,
+      "learning_rate": 4.418127745787234e-06,
+      "loss": 0.3854,
+      "step": 4512
+    },
+    {
+      "epoch": 0.9,
+      "grad_norm": 1.0051690340042114,
+      "learning_rate": 4.406187232776858e-06,
+      "loss": 0.378,
+      "step": 4544
+    },
+    {
+      "epoch": 0.91,
+      "grad_norm": 0.9830784201622009,
+      "learning_rate": 4.394141932636055e-06,
+      "loss": 0.3838,
+      "step": 4576
+    },
+    {
+      "epoch": 0.92,
+      "grad_norm": 0.9700829386711121,
+      "learning_rate": 4.381992507520366e-06,
+      "loss": 0.3689,
+      "step": 4608
+    },
+    {
+      "epoch": 0.92,
+      "grad_norm": 0.9510435461997986,
+      "learning_rate": 4.369739625309301e-06,
+      "loss": 0.3729,
+      "step": 4640
+    },
+    {
+      "epoch": 0.93,
+      "grad_norm": 1.12239408493042,
+      "learning_rate": 4.357383959569624e-06,
+      "loss": 0.3797,
+      "step": 4672
+    },
+    {
+      "epoch": 0.94,
+      "grad_norm": 1.1104124784469604,
+      "learning_rate": 4.344926189518325e-06,
+      "loss": 0.3732,
+      "step": 4704
+    },
+    {
+      "epoch": 0.94,
+      "grad_norm": 1.3127937316894531,
+      "learning_rate": 4.332366999985287e-06,
+      "loss": 0.373,
+      "step": 4736
+    },
+    {
+      "epoch": 0.95,
+      "grad_norm": 0.9870919585227966,
+      "learning_rate": 4.319707081375631e-06,
+      "loss": 0.3725,
+      "step": 4768
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 1.0262207984924316,
+      "learning_rate": 4.3069471296317735e-06,
+      "loss": 0.3763,
+      "step": 4800
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 1.137619137763977,
+      "learning_rate": 4.294087846195157e-06,
+      "loss": 0.3707,
+      "step": 4832
+    },
+    {
+      "epoch": 0.97,
+      "grad_norm": 1.0497642755508423,
+      "learning_rate": 4.2811299379677e-06,
+      "loss": 0.3823,
+      "step": 4864
+    },
+    {
+      "epoch": 0.97,
+      "grad_norm": 1.0143197774887085,
+      "learning_rate": 4.2680741172729325e-06,
+      "loss": 0.3772,
+      "step": 4896
+    },
+    {
+      "epoch": 0.98,
+      "grad_norm": 1.0744478702545166,
+      "learning_rate": 4.25492110181684e-06,
+      "loss": 0.3763,
+      "step": 4928
+    },
+    {
+      "epoch": 0.99,
+      "grad_norm": 1.113259196281433,
+      "learning_rate": 4.241671614648411e-06,
+      "loss": 0.3763,
+      "step": 4960
+    },
+    {
+      "epoch": 0.99,
+      "grad_norm": 1.1132539510726929,
+      "learning_rate": 4.228326384119882e-06,
+      "loss": 0.3679,
+      "step": 4992
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 0.9480674862861633,
+      "learning_rate": 4.214886143846707e-06,
+      "loss": 0.3763,
+      "step": 5024
+    },
+    {
+      "epoch": 1.01,
+      "grad_norm": 1.1497920751571655,
+      "learning_rate": 4.201351632667227e-06,
+      "loss": 0.3217,
+      "step": 5056
+    },
+    {
+      "epoch": 1.01,
+      "grad_norm": 1.0451420545578003,
+      "learning_rate": 4.187723594602054e-06,
+      "loss": 0.3261,
+      "step": 5088
+    },
+    {
+      "epoch": 1.02,
+      "grad_norm": 1.050591230392456,
+      "learning_rate": 4.174002778813164e-06,
+      "loss": 0.3213,
+      "step": 5120
+    },
+    {
+      "epoch": 1.03,
+      "grad_norm": 1.0211747884750366,
+      "learning_rate": 4.1601899395627285e-06,
+      "loss": 0.3286,
+      "step": 5152
+    },
+    {
+      "epoch": 1.03,
+      "grad_norm": 1.0216166973114014,
+      "learning_rate": 4.146285836171636e-06,
+      "loss": 0.3185,
+      "step": 5184
+    },
+    {
+      "epoch": 1.04,
+      "grad_norm": 1.0201820135116577,
+      "learning_rate": 4.13229123297776e-06,
+      "loss": 0.3168,
+      "step": 5216
+    },
+    {
+      "epoch": 1.05,
+      "grad_norm": 1.0070463418960571,
+      "learning_rate": 4.11820689929394e-06,
+      "loss": 0.3225,
+      "step": 5248
+    },
+    {
+      "epoch": 1.05,
+      "grad_norm": 1.1246079206466675,
+      "learning_rate": 4.104033609365687e-06,
+      "loss": 0.3335,
+      "step": 5280
+    },
+    {
+      "epoch": 1.06,
+      "grad_norm": 1.0078269243240356,
+      "learning_rate": 4.089772142328628e-06,
+      "loss": 0.3168,
+      "step": 5312
+    },
+    {
+      "epoch": 1.06,
+      "grad_norm": 1.0369760990142822,
+      "learning_rate": 4.075423282165665e-06,
+      "loss": 0.3304,
+      "step": 5344
+    },
+    {
+      "epoch": 1.07,
+      "grad_norm": 0.9748265147209167,
+      "learning_rate": 4.0609878176638925e-06,
+      "loss": 0.3283,
+      "step": 5376
+    },
+    {
+      "epoch": 1.08,
+      "grad_norm": 1.0375062227249146,
+      "learning_rate": 4.046466542371222e-06,
+      "loss": 0.321,
+      "step": 5408
+    },
+    {
+      "epoch": 1.08,
+      "grad_norm": 1.1058626174926758,
+      "learning_rate": 4.031860254552767e-06,
+      "loss": 0.3284,
+      "step": 5440
+    },
+    {
+      "epoch": 1.09,
+      "grad_norm": 1.070448637008667,
+      "learning_rate": 4.01716975714696e-06,
+      "loss": 0.3224,
+      "step": 5472
+    },
+    {
+      "epoch": 1.1,
+      "grad_norm": 1.1282106637954712,
+      "learning_rate": 4.002395857721411e-06,
+      "loss": 0.3159,
+      "step": 5504
+    },
+    {
+      "epoch": 1.1,
+      "grad_norm": 1.093741536140442,
+      "learning_rate": 3.987539368428514e-06,
+      "loss": 0.3328,
+      "step": 5536
+    },
+    {
+      "epoch": 1.11,
+      "grad_norm": 1.2170640230178833,
+      "learning_rate": 3.972601105960804e-06,
+      "loss": 0.3197,
+      "step": 5568
+    },
+    {
+      "epoch": 1.12,
+      "grad_norm": 1.2086021900177002,
+      "learning_rate": 3.957581891506057e-06,
+      "loss": 0.3223,
+      "step": 5600
+    },
+    {
+      "epoch": 1.12,
+      "grad_norm": 1.052445888519287,
+      "learning_rate": 3.9424825507021534e-06,
+      "loss": 0.317,
+      "step": 5632
+    },
+    {
+      "epoch": 1.13,
+      "grad_norm": 1.2977054119110107,
+      "learning_rate": 3.927303913591685e-06,
+      "loss": 0.323,
+      "step": 5664
+    },
+    {
+      "epoch": 1.13,
+      "grad_norm": 1.0437968969345093,
+      "learning_rate": 3.912046814576334e-06,
+      "loss": 0.3285,
+      "step": 5696
+    },
+    {
+      "epoch": 1.14,
+      "grad_norm": 1.1401695013046265,
+      "learning_rate": 3.896712092370991e-06,
+      "loss": 0.3246,
+      "step": 5728
+    },
+    {
+      "epoch": 1.15,
+      "grad_norm": 0.9710681438446045,
+      "learning_rate": 3.881300589957663e-06,
+      "loss": 0.3201,
+      "step": 5760
+    },
+    {
+      "epoch": 1.15,
+      "grad_norm": 1.0901442766189575,
+      "learning_rate": 3.865813154539125e-06,
+      "loss": 0.3275,
+      "step": 5792
+    },
+    {
+      "epoch": 1.16,
+      "grad_norm": 1.0210119485855103,
+      "learning_rate": 3.85025063749235e-06,
+      "loss": 0.3208,
+      "step": 5824
+    },
+    {
+      "epoch": 1.17,
+      "grad_norm": 1.0676419734954834,
+      "learning_rate": 3.834613894321705e-06,
+      "loss": 0.3309,
+      "step": 5856
+    },
+    {
+      "epoch": 1.17,
+      "grad_norm": 0.9977128505706787,
+      "learning_rate": 3.818903784611925e-06,
+      "loss": 0.3377,
+      "step": 5888
+    },
+    {
+      "epoch": 1.18,
+      "grad_norm": 1.1163454055786133,
+      "learning_rate": 3.803121171980857e-06,
+      "loss": 0.3191,
+      "step": 5920
+    },
+    {
+      "epoch": 1.19,
+      "grad_norm": 1.0978645086288452,
+      "learning_rate": 3.787266924031987e-06,
+      "loss": 0.32,
+      "step": 5952
+    },
+    {
+      "epoch": 1.19,
+      "grad_norm": 1.113274097442627,
+      "learning_rate": 3.7713419123067464e-06,
+      "loss": 0.3328,
+      "step": 5984
+    },
+    {
+      "epoch": 1.2,
+      "grad_norm": 1.1274746656417847,
+      "learning_rate": 3.755347012236599e-06,
+      "loss": 0.3205,
+      "step": 6016
+    },
+    {
+      "epoch": 1.2,
+      "grad_norm": 1.1300851106643677,
+      "learning_rate": 3.7392831030949182e-06,
+      "loss": 0.3111,
+      "step": 6048
+    },
+    {
+      "epoch": 1.21,
+      "grad_norm": 1.0431143045425415,
+      "learning_rate": 3.723151067948653e-06,
+      "loss": 0.3319,
+      "step": 6080
+    },
+    {
+      "epoch": 1.22,
+      "grad_norm": 1.0392550230026245,
+      "learning_rate": 3.706951793609782e-06,
+      "loss": 0.3278,
+      "step": 6112
+    },
+    {
+      "epoch": 1.22,
+      "grad_norm": 0.8391209244728088,
+      "learning_rate": 3.6906861705865632e-06,
+      "loss": 0.3273,
+      "step": 6144
+    },
+    {
+      "epoch": 1.23,
+      "grad_norm": 0.9675939083099365,
+      "learning_rate": 3.674355093034582e-06,
+      "loss": 0.3258,
+      "step": 6176
+    },
+    {
+      "epoch": 1.24,
+      "grad_norm": 1.1863322257995605,
+      "learning_rate": 3.657959458707598e-06,
+      "loss": 0.3283,
+      "step": 6208
+    },
+    {
+      "epoch": 1.24,
+      "grad_norm": 1.0589362382888794,
+      "learning_rate": 3.6415001689081912e-06,
+      "loss": 0.335,
+      "step": 6240
+    },
+    {
+      "epoch": 1.25,
+      "grad_norm": 1.1069774627685547,
+      "learning_rate": 3.624978128438219e-06,
+      "loss": 0.3314,
+      "step": 6272
+    },
+    {
+      "epoch": 1.26,
+      "grad_norm": 0.9723530411720276,
+      "learning_rate": 3.608394245549074e-06,
+      "loss": 0.3195,
+      "step": 6304
+    },
+    {
+      "epoch": 1.26,
+      "grad_norm": 0.9945602416992188,
+      "learning_rate": 3.5917494318917573e-06,
+      "loss": 0.3252,
+      "step": 6336
+    },
+    {
+      "epoch": 1.27,
+      "grad_norm": 1.0707299709320068,
+      "learning_rate": 3.575044602466763e-06,
+      "loss": 0.3204,
+      "step": 6368
+    },
+    {
+      "epoch": 1.27,
+      "grad_norm": 1.0709584951400757,
+      "learning_rate": 3.558280675573778e-06,
+      "loss": 0.3191,
+      "step": 6400
+    },
+    {
+      "epoch": 1.28,
+      "grad_norm": 1.215785026550293,
+      "learning_rate": 3.5414585727612026e-06,
+      "loss": 0.3242,
+      "step": 6432
+    },
+    {
+      "epoch": 1.29,
+      "grad_norm": 1.1426632404327393,
+      "learning_rate": 3.524579218775489e-06,
+      "loss": 0.3256,
+      "step": 6464
+    },
+    {
+      "epoch": 1.29,
+      "grad_norm": 1.0948857069015503,
+      "learning_rate": 3.5076435415103072e-06,
+      "loss": 0.3312,
+      "step": 6496
+    },
+    {
+      "epoch": 1.3,
+      "grad_norm": 1.0663847923278809,
+      "learning_rate": 3.490652471955538e-06,
+      "loss": 0.3356,
+      "step": 6528
+    },
+    {
+      "epoch": 1.31,
+      "grad_norm": 1.0345847606658936,
+      "learning_rate": 3.47360694414609e-06,
+      "loss": 0.3437,
+      "step": 6560
+    },
+    {
+      "epoch": 1.31,
+      "grad_norm": 1.0221242904663086,
+      "learning_rate": 3.456507895110561e-06,
+      "loss": 0.3271,
+      "step": 6592
+    },
+    {
+      "epoch": 1.32,
+      "grad_norm": 1.0684123039245605,
+      "learning_rate": 3.4393562648197197e-06,
+      "loss": 0.3206,
+      "step": 6624
+    },
+    {
+      "epoch": 1.33,
+      "grad_norm": 1.2369773387908936,
+      "learning_rate": 3.4221529961348394e-06,
+      "loss": 0.328,
+      "step": 6656
+    },
+    {
+      "epoch": 1.33,
+      "grad_norm": 1.1667596101760864,
+      "learning_rate": 3.404899034755864e-06,
+      "loss": 0.3278,
+      "step": 6688
+    },
+    {
+      "epoch": 1.34,
+      "grad_norm": 0.9238635301589966,
+      "learning_rate": 3.387595329169423e-06,
+      "loss": 0.3337,
+      "step": 6720
+    },
+    {
+      "epoch": 1.34,
+      "grad_norm": 1.0524715185165405,
+      "learning_rate": 3.3702428305966874e-06,
+      "loss": 0.3218,
+      "step": 6752
+    },
+    {
+      "epoch": 1.35,
+      "grad_norm": 0.9200053215026855,
+      "learning_rate": 3.3528424929410814e-06,
+      "loss": 0.3209,
+      "step": 6784
+    },
+    {
+      "epoch": 1.36,
+      "grad_norm": 1.0644574165344238,
+      "learning_rate": 3.3353952727358485e-06,
+      "loss": 0.3269,
+      "step": 6816
+    },
+    {
+      "epoch": 1.36,
+      "grad_norm": 1.0416203737258911,
+      "learning_rate": 3.3179021290914614e-06,
+      "loss": 0.3276,
+      "step": 6848
+    },
+    {
+      "epoch": 1.37,
+      "grad_norm": 1.1865791082382202,
+      "learning_rate": 3.3003640236429012e-06,
+      "loss": 0.3223,
+      "step": 6880
+    },
+    {
+      "epoch": 1.38,
+      "grad_norm": 1.1126179695129395,
+      "learning_rate": 3.2827819204967948e-06,
+      "loss": 0.3241,
+      "step": 6912
+    },
+    {
+      "epoch": 1.38,
+      "grad_norm": 1.0376890897750854,
+      "learning_rate": 3.265156786178415e-06,
+      "loss": 0.3299,
+      "step": 6944
+    },
+    {
+      "epoch": 1.39,
+      "grad_norm": 0.9320166110992432,
+      "learning_rate": 3.2474895895785503e-06,
+      "loss": 0.3224,
+      "step": 6976
+    },
+    {
+      "epoch": 1.4,
+      "grad_norm": 1.3843213319778442,
+      "learning_rate": 3.2297813019002383e-06,
+      "loss": 0.3272,
+      "step": 7008
+    },
+    {
+      "epoch": 1.4,
+      "grad_norm": 1.0767802000045776,
+      "learning_rate": 3.2120328966053836e-06,
+      "loss": 0.326,
+      "step": 7040
+    },
+    {
+      "epoch": 1.41,
+      "grad_norm": 0.9737852811813354,
+      "learning_rate": 3.194245349361238e-06,
+      "loss": 0.3253,
+      "step": 7072
+    },
+    {
+      "epoch": 1.41,
+      "grad_norm": 1.1968495845794678,
+      "learning_rate": 3.176419637986769e-06,
+      "loss": 0.3237,
+      "step": 7104
+    },
+    {
+      "epoch": 1.42,
+      "grad_norm": 1.086237907409668,
+      "learning_rate": 3.158556742398908e-06,
+      "loss": 0.3174,
+      "step": 7136
+    },
+    {
+      "epoch": 1.43,
+      "grad_norm": 1.1582037210464478,
+      "learning_rate": 3.1406576445586813e-06,
+      "loss": 0.322,
+      "step": 7168
+    },
+    {
+      "epoch": 1.43,
+      "grad_norm": 1.0444362163543701,
+      "learning_rate": 3.1227233284172265e-06,
+      "loss": 0.3299,
+      "step": 7200
+    },
+    {
+      "epoch": 1.44,
+      "grad_norm": 0.9630576372146606,
+      "learning_rate": 3.1047547798617094e-06,
+      "loss": 0.3215,
+      "step": 7232
+    },
+    {
+      "epoch": 1.45,
+      "grad_norm": 1.123935341835022,
+      "learning_rate": 3.086752986661121e-06,
+      "loss": 0.3234,
+      "step": 7264
+    },
+    {
+      "epoch": 1.45,
+      "grad_norm": 0.9781158566474915,
+      "learning_rate": 3.068718938411981e-06,
+      "loss": 0.3273,
+      "step": 7296
+    },
+    {
+      "epoch": 1.46,
+      "grad_norm": 1.1303229331970215,
+      "learning_rate": 3.050653626483937e-06,
+      "loss": 0.3256,
+      "step": 7328
+    },
+    {
+      "epoch": 1.47,
+      "grad_norm": 1.056015968322754,
+      "learning_rate": 3.0325580439652685e-06,
+      "loss": 0.3248,
+      "step": 7360
+    },
+    {
+      "epoch": 1.47,
+      "grad_norm": 1.1457360982894897,
+      "learning_rate": 3.014433185608292e-06,
+      "loss": 0.3351,
+      "step": 7392
+    },
+    {
+      "epoch": 1.48,
+      "grad_norm": 1.2207893133163452,
+      "learning_rate": 2.9962800477746774e-06,
+      "loss": 0.3239,
+      "step": 7424
+    },
+    {
+      "epoch": 1.48,
+      "grad_norm": 1.0306360721588135,
+      "learning_rate": 2.978099628380679e-06,
+      "loss": 0.3268,
+      "step": 7456
+    },
+    {
+      "epoch": 1.49,
+      "grad_norm": 1.1373237371444702,
+      "learning_rate": 2.9598929268422754e-06,
+      "loss": 0.3244,
+      "step": 7488
+    },
+    {
+      "epoch": 1.5,
+      "grad_norm": 1.2428505420684814,
+      "learning_rate": 2.941660944020229e-06,
+      "loss": 0.3225,
+      "step": 7520
+    },
+    {
+      "epoch": 1.5,
+      "grad_norm": 1.006510615348816,
+      "learning_rate": 2.92340468216507e-06,
+      "loss": 0.3094,
+      "step": 7552
+    },
+    {
+      "epoch": 1.51,
+      "grad_norm": 1.0351347923278809,
+      "learning_rate": 2.905125144861994e-06,
+      "loss": 0.3269,
+      "step": 7584
+    },
+    {
+      "epoch": 1.52,
+      "grad_norm": 1.1568477153778076,
+      "learning_rate": 2.886823336975703e-06,
+      "loss": 0.3213,
+      "step": 7616
+    },
+    {
+      "epoch": 1.52,
+      "grad_norm": 0.9311596155166626,
+      "learning_rate": 2.8685002645951553e-06,
+      "loss": 0.3259,
+      "step": 7648
+    },
+    {
+      "epoch": 1.53,
+      "grad_norm": 1.3714416027069092,
+      "learning_rate": 2.8501569349782643e-06,
+      "loss": 0.3226,
+      "step": 7680
+    },
+    {
+      "epoch": 1.54,
+      "grad_norm": 1.212634801864624,
+      "learning_rate": 2.8317943564965273e-06,
+      "loss": 0.3178,
+      "step": 7712
+    },
+    {
+      "epoch": 1.54,
+      "grad_norm": 1.128100872039795,
+      "learning_rate": 2.813413538579592e-06,
+      "loss": 0.3235,
+      "step": 7744
+    },
+    {
+      "epoch": 1.55,
+      "grad_norm": 1.0316797494888306,
+      "learning_rate": 2.7950154916597653e-06,
+      "loss": 0.32,
+      "step": 7776
+    },
+    {
+      "epoch": 1.55,
+      "grad_norm": 1.144771695137024,
+      "learning_rate": 2.77660122711647e-06,
+      "loss": 0.3213,
+      "step": 7808
+    },
+    {
+      "epoch": 1.56,
+      "grad_norm": 1.0653690099716187,
+      "learning_rate": 2.7581717572206445e-06,
+      "loss": 0.3227,
+      "step": 7840
+    },
+    {
+      "epoch": 1.57,
+      "grad_norm": 0.9339531660079956,
+      "learning_rate": 2.739728095079099e-06,
+      "loss": 0.3202,
+      "step": 7872
+    },
+    {
+      "epoch": 1.57,
+      "grad_norm": 1.2507269382476807,
+      "learning_rate": 2.72127125457882e-06,
+      "loss": 0.321,
+      "step": 7904
+    },
+    {
+      "epoch": 1.58,
+      "grad_norm": 1.0491323471069336,
+      "learning_rate": 2.702802250331238e-06,
+      "loss": 0.328,
+      "step": 7936
+    },
+    {
+      "epoch": 1.59,
+      "grad_norm": 0.987471342086792,
+      "learning_rate": 2.684322097616448e-06,
+      "loss": 0.3254,
+      "step": 7968
+    },
+    {
+      "epoch": 1.59,
+      "grad_norm": 1.098031759262085,
+      "learning_rate": 2.6658318123274013e-06,
+      "loss": 0.3208,
+      "step": 8000
+    },
+    {
+      "epoch": 1.6,
+      "grad_norm": 0.9676088690757751,
+      "learning_rate": 2.647332410914058e-06,
+      "loss": 0.3301,
+      "step": 8032
+    },
+    {
+      "epoch": 1.61,
+      "grad_norm": 0.9914807081222534,
+      "learning_rate": 2.62882491032751e-06,
+      "loss": 0.334,
+      "step": 8064
+    },
+    {
+      "epoch": 1.61,
+      "grad_norm": 1.0016244649887085,
+      "learning_rate": 2.6103103279640786e-06,
+      "loss": 0.3207,
+      "step": 8096
+    },
+    {
+      "epoch": 1.62,
+      "grad_norm": 1.1957006454467773,
+      "learning_rate": 2.5917896816093848e-06,
+      "loss": 0.3293,
+      "step": 8128
+    },
+    {
+      "epoch": 1.62,
+      "grad_norm": 1.0136693716049194,
+      "learning_rate": 2.5732639893824003e-06,
+      "loss": 0.3217,
+      "step": 8160
+    },
+    {
+      "epoch": 1.63,
+      "grad_norm": 1.02032470703125,
+      "learning_rate": 2.5547342696794792e-06,
+      "loss": 0.327,
+      "step": 8192
+    },
+    {
+      "epoch": 1.64,
+      "grad_norm": 1.0116775035858154,
+      "learning_rate": 2.5362015411183736e-06,
+      "loss": 0.3157,
+      "step": 8224
+    },
+    {
+      "epoch": 1.64,
+      "grad_norm": 1.040771722793579,
+      "learning_rate": 2.51766682248224e-06,
+      "loss": 0.3131,
+      "step": 8256
+    },
+    {
+      "epoch": 1.65,
+      "grad_norm": 0.9721716642379761,
+      "learning_rate": 2.4991311326636344e-06,
+      "loss": 0.3132,
+      "step": 8288
+    },
+    {
+      "epoch": 1.66,
+      "grad_norm": 0.9759429097175598,
+      "learning_rate": 2.480595490608496e-06,
+      "loss": 0.3178,
+      "step": 8320
+    },
+    {
+      "epoch": 1.66,
+      "grad_norm": 0.9927918910980225,
+      "learning_rate": 2.462060915260145e-06,
+      "loss": 0.3214,
+      "step": 8352
+    },
+    {
+      "epoch": 1.67,
+      "grad_norm": 1.1041028499603271,
+      "learning_rate": 2.4435284255032562e-06,
+      "loss": 0.3128,
+      "step": 8384
+    },
+    {
+      "epoch": 1.68,
+      "grad_norm": 0.9102107286453247,
+      "learning_rate": 2.4249990401078603e-06,
+      "loss": 0.3279,
+      "step": 8416
+    },
+    {
+      "epoch": 1.68,
+      "grad_norm": 1.1079301834106445,
+      "learning_rate": 2.4064737776733317e-06,
+      "loss": 0.3249,
+      "step": 8448
+    },
+    {
+      "epoch": 1.69,
+      "grad_norm": 1.1040772199630737,
+      "learning_rate": 2.387953656572397e-06,
+      "loss": 0.3209,
+      "step": 8480
+    },
+    {
+      "epoch": 1.69,
+      "grad_norm": 1.0137909650802612,
+      "learning_rate": 2.3694396948951553e-06,
+      "loss": 0.3188,
+      "step": 8512
+    },
+    {
+      "epoch": 1.7,
+      "grad_norm": 1.2013212442398071,
+      "learning_rate": 2.3509329103931046e-06,
+      "loss": 0.3229,
+      "step": 8544
+    },
+    {
+      "epoch": 1.71,
+      "grad_norm": 1.1045948266983032,
+      "learning_rate": 2.3324343204232046e-06,
+      "loss": 0.3272,
+      "step": 8576
+    },
+    {
+      "epoch": 1.71,
+      "grad_norm": 1.0970181226730347,
+      "learning_rate": 2.3139449418919374e-06,
+      "loss": 0.3242,
+      "step": 8608
+    },
+    {
+      "epoch": 1.72,
+      "grad_norm": 0.8859126567840576,
+      "learning_rate": 2.2954657911994195e-06,
+      "loss": 0.3231,
+      "step": 8640
+    },
+    {
+      "epoch": 1.73,
+      "grad_norm": 0.8855715394020081,
+      "learning_rate": 2.2769978841835164e-06,
+      "loss": 0.3076,
+      "step": 8672
+    },
+    {
+      "epoch": 1.73,
+      "grad_norm": 1.0653135776519775,
+      "learning_rate": 2.258542236064009e-06,
+      "loss": 0.3219,
+      "step": 8704
+    },
+    {
+      "epoch": 1.74,
+      "grad_norm": 1.136330008506775,
+      "learning_rate": 2.240099861386776e-06,
+      "loss": 0.3298,
+      "step": 8736
+    },
+    {
+      "epoch": 1.75,
+      "grad_norm": 1.0157207250595093,
+      "learning_rate": 2.221671773968034e-06,
+      "loss": 0.3201,
+      "step": 8768
+    },
+    {
+      "epoch": 1.75,
+      "grad_norm": 1.1037923097610474,
+      "learning_rate": 2.2032589868385912e-06,
+      "loss": 0.3146,
+      "step": 8800
+    },
+    {
+      "epoch": 1.76,
+      "grad_norm": 1.1770648956298828,
+      "learning_rate": 2.184862512188172e-06,
+      "loss": 0.316,
+      "step": 8832
+    },
+    {
+      "epoch": 1.77,
+      "grad_norm": 0.9664033651351929,
+      "learning_rate": 2.1664833613097667e-06,
+      "loss": 0.3272,
+      "step": 8864
+    },
+    {
+      "epoch": 1.77,
+      "grad_norm": 1.1088409423828125,
+      "learning_rate": 2.1481225445440445e-06,
+      "loss": 0.3183,
+      "step": 8896
+    },
+    {
+      "epoch": 1.78,
+      "grad_norm": 1.0327556133270264,
+      "learning_rate": 2.129781071223804e-06,
+      "loss": 0.3187,
+      "step": 8928
+    },
+    {
+      "epoch": 1.78,
+      "grad_norm": 1.0837349891662598,
+      "learning_rate": 2.111459949618501e-06,
+      "loss": 0.3192,
+      "step": 8960
+    },
+    {
+      "epoch": 1.79,
+      "grad_norm": 1.1200575828552246,
+      "learning_rate": 2.0931601868788076e-06,
+      "loss": 0.3079,
+      "step": 8992
+    },
+    {
+      "epoch": 1.8,
+      "grad_norm": 0.9722447991371155,
+      "learning_rate": 2.07488278898126e-06,
+      "loss": 0.3238,
+      "step": 9024
+    },
+    {
+      "epoch": 1.8,
+      "grad_norm": 1.1324560642242432,
+      "learning_rate": 2.0566287606729495e-06,
+      "loss": 0.3187,
+      "step": 9056
+    },
+    {
+      "epoch": 1.81,
+      "grad_norm": 0.9103668332099915,
+      "learning_rate": 2.03839910541629e-06,
+      "loss": 0.3245,
+      "step": 9088
+    },
+    {
+      "epoch": 1.82,
+      "grad_norm": 1.1157492399215698,
+      "learning_rate": 2.0201948253338605e-06,
+      "loss": 0.3169,
+      "step": 9120
+    },
+    {
+      "epoch": 1.82,
+      "grad_norm": 1.0973373651504517,
+      "learning_rate": 2.002016921153311e-06,
+      "loss": 0.3123,
+      "step": 9152
+    },
+    {
+      "epoch": 1.83,
+      "grad_norm": 1.1510241031646729,
+      "learning_rate": 1.983866392152353e-06,
+      "loss": 0.3199,
+      "step": 9184
+    },
+    {
+      "epoch": 1.84,
+      "grad_norm": 1.159393310546875,
+      "learning_rate": 1.965744236103828e-06,
+      "loss": 0.3206,
+      "step": 9216
+    },
+    {
+      "epoch": 1.84,
+      "grad_norm": 1.053611397743225,
+      "learning_rate": 1.9476514492208553e-06,
+      "loss": 0.3282,
+      "step": 9248
+    },
+    {
+      "epoch": 1.85,
+      "grad_norm": 1.0583845376968384,
+      "learning_rate": 1.9295890261020682e-06,
+      "loss": 0.3251,
+      "step": 9280
+    },
+    {
+      "epoch": 1.85,
+      "grad_norm": 0.9920005798339844,
+      "learning_rate": 1.9115579596769425e-06,
+      "loss": 0.319,
+      "step": 9312
+    },
+    {
+      "epoch": 1.86,
+      "grad_norm": 1.0284650325775146,
+      "learning_rate": 1.8935592411512077e-06,
+      "loss": 0.319,
+      "step": 9344
+    },
+    {
+      "epoch": 1.87,
+      "grad_norm": 1.1910771131515503,
+      "learning_rate": 1.875593859952364e-06,
+      "loss": 0.3265,
+      "step": 9376
+    },
+    {
+      "epoch": 1.87,
+      "grad_norm": 1.1365597248077393,
+      "learning_rate": 1.8576628036752852e-06,
+      "loss": 0.3089,
+      "step": 9408
+    },
+    {
+      "epoch": 1.88,
+      "grad_norm": 0.9488004446029663,
+      "learning_rate": 1.8397670580279357e-06,
+      "loss": 0.314,
+      "step": 9440
+    },
+    {
+      "epoch": 1.89,
+      "grad_norm": 1.0142693519592285,
+      "learning_rate": 1.8219076067771755e-06,
+      "loss": 0.3136,
+      "step": 9472
+    },
+    {
+      "epoch": 1.89,
+      "grad_norm": 1.1403080224990845,
+      "learning_rate": 1.8040854316946896e-06,
+      "loss": 0.3131,
+      "step": 9504
+    },
+    {
+      "epoch": 1.9,
+      "grad_norm": 0.9643501043319702,
+      "learning_rate": 1.7863015125030102e-06,
+      "loss": 0.3177,
+      "step": 9536
+    },
+    {
+      "epoch": 1.91,
+      "grad_norm": 0.974354088306427,
+      "learning_rate": 1.768556826821664e-06,
+      "loss": 0.3137,
+      "step": 9568
+    },
+    {
+      "epoch": 1.91,
+      "grad_norm": 1.1438343524932861,
+      "learning_rate": 1.750852350113429e-06,
+      "loss": 0.3211,
+      "step": 9600
+    },
+    {
+      "epoch": 1.92,
+      "grad_norm": 0.9480288028717041,
+      "learning_rate": 1.7331890556307096e-06,
+      "loss": 0.3167,
+      "step": 9632
+    },
+    {
+      "epoch": 1.92,
+      "grad_norm": 1.0253633260726929,
+      "learning_rate": 1.71556791436204e-06,
+      "loss": 0.3008,
+      "step": 9664
+    },
+    {
+      "epoch": 1.93,
+      "grad_norm": 1.1114031076431274,
+      "learning_rate": 1.6979898949786988e-06,
+      "loss": 0.3193,
+      "step": 9696
+    },
+    {
+      "epoch": 1.94,
+      "grad_norm": 1.1946877241134644,
+      "learning_rate": 1.6804559637814682e-06,
+      "loss": 0.3191,
+      "step": 9728
+    },
+    {
+      "epoch": 1.94,
+      "grad_norm": 0.8322132229804993,
+      "learning_rate": 1.6629670846475055e-06,
+      "loss": 0.3108,
+      "step": 9760
+    },
+    {
+      "epoch": 1.95,
+      "grad_norm": 1.1155304908752441,
+      "learning_rate": 1.6455242189773646e-06,
+      "loss": 0.3212,
+      "step": 9792
+    },
+    {
+      "epoch": 1.96,
+      "grad_norm": 1.2104612588882446,
+      "learning_rate": 1.6281283256421404e-06,
+      "loss": 0.3188,
+      "step": 9824
+    },
+    {
+      "epoch": 1.96,
+      "grad_norm": 1.1790159940719604,
+      "learning_rate": 1.6107803609307598e-06,
+      "loss": 0.3207,
+      "step": 9856
+    },
+    {
+      "epoch": 1.97,
+      "grad_norm": 1.0311027765274048,
+      "learning_rate": 1.593481278497413e-06,
+      "loss": 0.3193,
+      "step": 9888
+    },
+    {
+      "epoch": 1.98,
+      "grad_norm": 1.0055537223815918,
+      "learning_rate": 1.5762320293091281e-06,
+      "loss": 0.3107,
+      "step": 9920
+    },
+    {
+      "epoch": 1.98,
+      "grad_norm": 1.201463222503662,
+      "learning_rate": 1.5590335615934928e-06,
+      "loss": 0.3122,
+      "step": 9952
+    },
+    {
+      "epoch": 1.99,
+      "grad_norm": 1.1516947746276855,
+      "learning_rate": 1.5418868207865339e-06,
+      "loss": 0.3075,
+      "step": 9984
+    },
+    {
+      "epoch": 1.99,
+      "grad_norm": 0.9799574613571167,
+      "learning_rate": 1.5247927494807368e-06,
+      "loss": 0.3172,
+      "step": 10016
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 0.9726473689079285,
+      "learning_rate": 1.5077522873732375e-06,
+      "loss": 0.3095,
+      "step": 10048
+    },
+    {
+      "epoch": 2.01,
+      "grad_norm": 0.9812408685684204,
+      "learning_rate": 1.4907663712141573e-06,
+      "loss": 0.2575,
+      "step": 10080
+    },
+    {
+      "epoch": 2.01,
+      "grad_norm": 0.9772374629974365,
+      "learning_rate": 1.4738359347551156e-06,
+      "loss": 0.2344,
+      "step": 10112
+    },
+    {
+      "epoch": 2.02,
+      "grad_norm": 1.1157349348068237,
+      "learning_rate": 1.4569619086978926e-06,
+      "loss": 0.2444,
+      "step": 10144
+    },
+    {
+      "epoch": 2.03,
+      "grad_norm": 0.9350994825363159,
+      "learning_rate": 1.440145220643272e-06,
+      "loss": 0.2429,
+      "step": 10176
+    },
+    {
+      "epoch": 2.03,
+      "grad_norm": 1.1283538341522217,
+      "learning_rate": 1.4233867950400444e-06,
+      "loss": 0.2333,
+      "step": 10208
+    },
+    {
+      "epoch": 2.04,
+      "grad_norm": 1.1436692476272583,
+      "learning_rate": 1.406687553134194e-06,
+      "loss": 0.2485,
+      "step": 10240
+    },
+    {
+      "epoch": 2.05,
+      "grad_norm": 1.1271458864212036,
+      "learning_rate": 1.390048412918248e-06,
+      "loss": 0.236,
+      "step": 10272
+    },
+    {
+      "epoch": 2.05,
+      "grad_norm": 1.089193344116211,
+      "learning_rate": 1.3734702890808205e-06,
+      "loss": 0.2403,
+      "step": 10304
+    },
+    {
+      "epoch": 2.06,
+      "grad_norm": 1.275738000869751,
+      "learning_rate": 1.3569540929563262e-06,
+      "loss": 0.2428,
+      "step": 10336
+    },
+    {
+      "epoch": 2.06,
+      "grad_norm": 0.9869887232780457,
+      "learning_rate": 1.340500732474882e-06,
+      "loss": 0.2514,
+      "step": 10368
+    },
+    {
+      "epoch": 2.07,
+      "grad_norm": 1.2141892910003662,
+      "learning_rate": 1.3241111121123956e-06,
+      "loss": 0.2322,
+      "step": 10400
+    },
+    {
+      "epoch": 2.08,
+      "grad_norm": 1.194463849067688,
+      "learning_rate": 1.3077861328408497e-06,
+      "loss": 0.247,
+      "step": 10432
+    },
+    {
+      "epoch": 2.08,
+      "grad_norm": 1.0947892665863037,
+      "learning_rate": 1.2915266920787672e-06,
+      "loss": 0.2423,
+      "step": 10464
+    },
+    {
+      "epoch": 2.09,
+      "grad_norm": 1.0153180360794067,
+      "learning_rate": 1.2753336836418805e-06,
+      "loss": 0.2397,
+      "step": 10496
+    },
+    {
+      "epoch": 2.1,
+      "grad_norm": 1.0111596584320068,
+      "learning_rate": 1.2592079976939998e-06,
+      "loss": 0.2505,
+      "step": 10528
+    },
+    {
+      "epoch": 2.1,
+      "grad_norm": 1.3424073457717896,
+      "learning_rate": 1.2431505206980732e-06,
+      "loss": 0.246,
+      "step": 10560
+    },
+    {
+      "epoch": 2.11,
+      "grad_norm": 1.0698063373565674,
+      "learning_rate": 1.227162135367462e-06,
+      "loss": 0.2445,
+      "step": 10592
+    },
+    {
+      "epoch": 2.12,
+      "grad_norm": 1.0742661952972412,
+      "learning_rate": 1.2112437206174094e-06,
+      "loss": 0.2423,
+      "step": 10624
+    },
+    {
+      "epoch": 2.12,
+      "grad_norm": 1.0715097188949585,
+      "learning_rate": 1.195396151516732e-06,
+      "loss": 0.2413,
+      "step": 10656
+    },
+    {
+      "epoch": 2.13,
+      "grad_norm": 1.041046142578125,
+      "learning_rate": 1.1796202992397082e-06,
+      "loss": 0.2445,
+      "step": 10688
+    },
+    {
+      "epoch": 2.13,
+      "grad_norm": 0.8946874737739563,
+      "learning_rate": 1.1639170310181952e-06,
+      "loss": 0.2452,
+      "step": 10720
+    },
+    {
+      "epoch": 2.14,
+      "grad_norm": 1.2150330543518066,
+      "learning_rate": 1.148287210093947e-06,
+      "loss": 0.2506,
+      "step": 10752
+    },
+    {
+      "epoch": 2.15,
+      "grad_norm": 1.0310112237930298,
+      "learning_rate": 1.1327316956711703e-06,
+      "loss": 0.2506,
+      "step": 10784
+    },
+    {
+      "epoch": 2.15,
+      "grad_norm": 1.1847726106643677,
+      "learning_rate": 1.1172513428692815e-06,
+      "loss": 0.2469,
+      "step": 10816
+    },
+    {
+      "epoch": 2.16,
+      "grad_norm": 1.0254921913146973,
+      "learning_rate": 1.1018470026759088e-06,
+      "loss": 0.246,
+      "step": 10848
+    },
+    {
+      "epoch": 2.17,
+      "grad_norm": 1.1089096069335938,
+      "learning_rate": 1.0865195219001028e-06,
+      "loss": 0.2441,
+      "step": 10880
+    },
+    {
+      "epoch": 2.17,
+      "grad_norm": 1.0616801977157593,
+      "learning_rate": 1.0712697431257931e-06,
+      "loss": 0.2456,
+      "step": 10912
+    },
+    {
+      "epoch": 2.18,
+      "grad_norm": 1.0265308618545532,
+      "learning_rate": 1.0560985046654666e-06,
+      "loss": 0.2393,
+      "step": 10944
+    },
+    {
+      "epoch": 2.19,
+      "grad_norm": 0.9837111234664917,
+      "learning_rate": 1.04100664051408e-06,
+      "loss": 0.2381,
+      "step": 10976
+    },
+    {
+      "epoch": 2.19,
+      "grad_norm": 1.1248365640640259,
+      "learning_rate": 1.0259949803032217e-06,
+      "loss": 0.246,
+      "step": 11008
+    },
+    {
+      "epoch": 2.2,
+      "grad_norm": 1.0951381921768188,
+      "learning_rate": 1.0110643492554964e-06,
+      "loss": 0.2359,
+      "step": 11040
+    },
+    {
+      "epoch": 2.2,
+      "grad_norm": 1.0301274061203003,
+      "learning_rate": 9.962155681391678e-07,
+      "loss": 0.2455,
+      "step": 11072
+    },
+    {
+      "epoch": 2.21,
+      "grad_norm": 1.0847768783569336,
+      "learning_rate": 9.814494532230343e-07,
+      "loss": 0.2392,
+      "step": 11104
+    },
+    {
+      "epoch": 2.22,
+      "grad_norm": 1.191899061203003,
+      "learning_rate": 9.66766816231559e-07,
+      "loss": 0.2473,
+      "step": 11136
+    },
+    {
+      "epoch": 2.22,
+      "grad_norm": 1.6308176517486572,
+      "learning_rate": 9.521684643002501e-07,
+      "loss": 0.251,
+      "step": 11168
+    },
+    {
+      "epoch": 2.23,
+      "grad_norm": 1.107154130935669,
+      "learning_rate": 9.376551999312858e-07,
+      "loss": 0.2459,
+      "step": 11200
+    },
+    {
+      "epoch": 2.24,
+      "grad_norm": 1.1042745113372803,
+      "learning_rate": 9.232278209494013e-07,
+      "loss": 0.2486,
+      "step": 11232
+    },
+    {
+      "epoch": 2.24,
+      "grad_norm": 1.080151081085205,
+      "learning_rate": 9.088871204580341e-07,
+      "loss": 0.2533,
+      "step": 11264
+    },
+    {
+      "epoch": 2.25,
+      "grad_norm": 1.0867422819137573,
+      "learning_rate": 8.946338867957183e-07,
+      "loss": 0.2493,
+      "step": 11296
+    },
+    {
+      "epoch": 2.26,
+      "grad_norm": 1.1670057773590088,
+      "learning_rate": 8.804689034927564e-07,
+      "loss": 0.236,
+      "step": 11328
+    },
+    {
+      "epoch": 2.26,
+      "grad_norm": 1.1410850286483765,
+      "learning_rate": 8.663929492281379e-07,
+      "loss": 0.2437,
+      "step": 11360
+    },
+    {
+      "epoch": 2.27,
+      "grad_norm": 1.1354265213012695,
+      "learning_rate": 8.524067977867431e-07,
+      "loss": 0.2436,
+      "step": 11392
+    },
+    {
+      "epoch": 2.27,
+      "grad_norm": 1.142817735671997,
+      "learning_rate": 8.385112180167979e-07,
+      "loss": 0.2349,
+      "step": 11424
+    },
+    {
+      "epoch": 2.28,
+      "grad_norm": 1.1933698654174805,
+      "learning_rate": 8.247069737876151e-07,
+      "loss": 0.2479,
+      "step": 11456
+    },
+    {
+      "epoch": 2.29,
+      "grad_norm": 1.0865373611450195,
+      "learning_rate": 8.109948239475984e-07,
+      "loss": 0.2379,
+      "step": 11488
+    },
+    {
+      "epoch": 2.29,
+      "grad_norm": 1.1452354192733765,
+      "learning_rate": 7.97375522282531e-07,
+      "loss": 0.2414,
+      "step": 11520
+    },
+    {
+      "epoch": 2.3,
+      "grad_norm": 1.1934701204299927,
+      "learning_rate": 7.838498174741332e-07,
+      "loss": 0.2533,
+      "step": 11552
+    },
+    {
+      "epoch": 2.31,
+      "grad_norm": 0.9815391302108765,
+      "learning_rate": 7.704184530589114e-07,
+      "loss": 0.2552,
+      "step": 11584
+    },
+    {
+      "epoch": 2.31,
+      "grad_norm": 0.9655036926269531,
+      "learning_rate": 7.570821673872819e-07,
+      "loss": 0.2423,
+      "step": 11616
+    },
+    {
+      "epoch": 2.32,
+      "grad_norm": 1.091679573059082,
+      "learning_rate": 7.43841693582979e-07,
+      "loss": 0.2474,
+      "step": 11648
+    },
+    {
+      "epoch": 2.33,
+      "grad_norm": 1.0601979494094849,
+      "learning_rate": 7.306977595027601e-07,
+      "loss": 0.2533,
+      "step": 11680
+    },
+    {
+      "epoch": 2.33,
+      "grad_norm": 1.1217659711837769,
+      "learning_rate": 7.176510876963877e-07,
+      "loss": 0.2495,
+      "step": 11712
+    },
+    {
+      "epoch": 2.34,
+      "grad_norm": 1.207133412361145,
+      "learning_rate": 7.047023953669147e-07,
+      "loss": 0.2471,
+      "step": 11744
+    },
+    {
+      "epoch": 2.34,
+      "grad_norm": 1.1191112995147705,
+      "learning_rate": 6.91852394331253e-07,
+      "loss": 0.2441,
+      "step": 11776
+    },
+    {
+      "epoch": 2.35,
+      "grad_norm": 1.109110951423645,
+      "learning_rate": 6.791017909810485e-07,
+      "loss": 0.2455,
+      "step": 11808
+    },
+    {
+      "epoch": 2.36,
+      "grad_norm": 1.1308753490447998,
+      "learning_rate": 6.664512862438452e-07,
+      "loss": 0.2519,
+      "step": 11840
+    },
+    {
+      "epoch": 2.36,
+      "grad_norm": 1.0771924257278442,
+      "learning_rate": 6.539015755445558e-07,
+      "loss": 0.2542,
+      "step": 11872
+    },
+    {
+      "epoch": 2.37,
+      "grad_norm": 1.2782262563705444,
+      "learning_rate": 6.414533487672309e-07,
+      "loss": 0.2445,
+      "step": 11904
+    },
+    {
+      "epoch": 2.38,
+      "grad_norm": 1.0829001665115356,
+      "learning_rate": 6.291072902171393e-07,
+      "loss": 0.2422,
+      "step": 11936
+    },
+    {
+      "epoch": 2.38,
+      "grad_norm": 1.0090858936309814,
+      "learning_rate": 6.168640785831434e-07,
+      "loss": 0.2413,
+      "step": 11968
+    },
+    {
+      "epoch": 2.39,
+      "grad_norm": 1.0461190938949585,
+      "learning_rate": 6.047243869003968e-07,
+      "loss": 0.2388,
+      "step": 12000
+    },
+    {
+      "epoch": 2.4,
+      "grad_norm": 0.9960162043571472,
+      "learning_rate": 5.926888825133404e-07,
+      "loss": 0.24,
+      "step": 12032
+    },
+    {
+      "epoch": 2.4,
+      "grad_norm": 1.309798002243042,
+      "learning_rate": 5.807582270390225e-07,
+      "loss": 0.2506,
+      "step": 12064
+    },
+    {
+      "epoch": 2.41,
+      "grad_norm": 1.129561185836792,
+      "learning_rate": 5.689330763307229e-07,
+      "loss": 0.2355,
+      "step": 12096
+    },
+    {
+      "epoch": 2.41,
+      "grad_norm": 1.2323921918869019,
+      "learning_rate": 5.572140804419049e-07,
+      "loss": 0.2442,
+      "step": 12128
+    },
+    {
+      "epoch": 2.42,
+      "grad_norm": 1.2903739213943481,
+      "learning_rate": 5.45601883590475e-07,
+      "loss": 0.2405,
+      "step": 12160
+    },
+    {
+      "epoch": 2.43,
+      "grad_norm": 1.1188290119171143,
+      "learning_rate": 5.340971241233728e-07,
+      "loss": 0.2451,
+      "step": 12192
+    },
+    {
+      "epoch": 2.43,
+      "grad_norm": 1.2080957889556885,
+      "learning_rate": 5.227004344814792e-07,
+      "loss": 0.2495,
+      "step": 12224
+    },
+    {
+      "epoch": 2.44,
+      "grad_norm": 1.0671383142471313,
+      "learning_rate": 5.114124411648466e-07,
+      "loss": 0.2367,
+      "step": 12256
+    },
+    {
+      "epoch": 2.45,
+      "grad_norm": 1.1058146953582764,
+      "learning_rate": 5.002337646982641e-07,
+      "loss": 0.2532,
+      "step": 12288
+    },
+    {
+      "epoch": 2.45,
+      "grad_norm": 1.0272908210754395,
+      "learning_rate": 4.891650195971406e-07,
+      "loss": 0.2455,
+      "step": 12320
+    },
+    {
+      "epoch": 2.46,
+      "grad_norm": 1.0757761001586914,
+      "learning_rate": 4.782068143337279e-07,
+      "loss": 0.2421,
+      "step": 12352
+    },
+    {
+      "epoch": 2.47,
+      "grad_norm": 1.003451943397522,
+      "learning_rate": 4.673597513036684e-07,
+      "loss": 0.24,
+      "step": 12384
+    },
+    {
+      "epoch": 2.47,
+      "grad_norm": 1.0739375352859497,
+      "learning_rate": 4.5662442679288376e-07,
+      "loss": 0.2467,
+      "step": 12416
+    },
+    {
+      "epoch": 2.48,
+      "grad_norm": 1.070870041847229,
+      "learning_rate": 4.4600143094479103e-07,
+      "loss": 0.2466,
+      "step": 12448
+    },
+    {
+      "epoch": 2.49,
+      "grad_norm": 1.0825251340866089,
+      "learning_rate": 4.354913477278663e-07,
+      "loss": 0.2441,
+      "step": 12480
+    },
+    {
+      "epoch": 2.49,
+      "grad_norm": 0.9770128130912781,
+      "learning_rate": 4.2509475490353893e-07,
+      "loss": 0.2421,
+      "step": 12512
+    },
+    {
+      "epoch": 2.5,
+      "grad_norm": 1.2835708856582642,
+      "learning_rate": 4.148122239944316e-07,
+      "loss": 0.2325,
+      "step": 12544
+    },
+    {
+      "epoch": 2.5,
+      "grad_norm": 1.1013844013214111,
+      "learning_rate": 4.0464432025294516e-07,
+      "loss": 0.2488,
+      "step": 12576
+    },
+    {
+      "epoch": 2.51,
+      "grad_norm": 1.2960447072982788,
+      "learning_rate": 3.9459160263018185e-07,
+      "loss": 0.2394,
+      "step": 12608
+    },
+    {
+      "epoch": 2.52,
+      "grad_norm": 1.1596546173095703,
+      "learning_rate": 3.846546237452198e-07,
+      "loss": 0.2439,
+      "step": 12640
+    },
+    {
+      "epoch": 2.52,
+      "grad_norm": 1.0801446437835693,
+      "learning_rate": 3.7483392985473654e-07,
+      "loss": 0.2397,
+      "step": 12672
+    },
+    {
+      "epoch": 2.53,
+      "grad_norm": 1.1258620023727417,
+      "learning_rate": 3.651300608229763e-07,
+      "loss": 0.2452,
+      "step": 12704
+    },
+    {
+      "epoch": 2.54,
+      "grad_norm": 1.116957664489746,
+      "learning_rate": 3.555435500920765e-07,
+      "loss": 0.2471,
+      "step": 12736
+    },
+    {
+      "epoch": 2.54,
+      "grad_norm": 1.0061496496200562,
+      "learning_rate": 3.4607492465273926e-07,
+      "loss": 0.252,
+      "step": 12768
+    },
+    {
+      "epoch": 2.55,
+      "grad_norm": 1.235310435295105,
+      "learning_rate": 3.3672470501526584e-07,
+      "loss": 0.2449,
+      "step": 12800
+    },
+    {
+      "epoch": 2.56,
+      "grad_norm": 1.0887601375579834,
+      "learning_rate": 3.2749340518093894e-07,
+      "loss": 0.2458,
+      "step": 12832
+    },
+    {
+      "epoch": 2.56,
+      "grad_norm": 1.1944972276687622,
+      "learning_rate": 3.183815326137707e-07,
+      "loss": 0.2425,
+      "step": 12864
+    },
+    {
+      "epoch": 2.57,
+      "grad_norm": 1.072562575340271,
+      "learning_rate": 3.093895882126044e-07,
+      "loss": 0.2472,
+      "step": 12896
+    },
+    {
+      "epoch": 2.57,
+      "grad_norm": 1.187567949295044,
+      "learning_rate": 3.0051806628357744e-07,
+      "loss": 0.2469,
+      "step": 12928
+    },
+    {
+      "epoch": 2.58,
+      "grad_norm": 1.0741044282913208,
+      "learning_rate": 2.9176745451295315e-07,
+      "loss": 0.2451,
+      "step": 12960
+    },
+    {
+      "epoch": 2.59,
+      "grad_norm": 1.0857914686203003,
+      "learning_rate": 2.831382339403049e-07,
+      "loss": 0.2453,
+      "step": 12992
+    },
+    {
+      "epoch": 2.59,
+      "grad_norm": 1.2118492126464844,
+      "learning_rate": 2.746308789320798e-07,
+      "loss": 0.2465,
+      "step": 13024
+    },
+    {
+      "epoch": 2.6,
+      "grad_norm": 1.0996769666671753,
+      "learning_rate": 2.662458571555149e-07,
+      "loss": 0.2498,
+      "step": 13056
+    },
+    {
+      "epoch": 2.61,
+      "grad_norm": 1.0050989389419556,
+      "learning_rate": 2.579836295529334e-07,
+      "loss": 0.2467,
+      "step": 13088
+    },
+    {
+      "epoch": 2.61,
+      "grad_norm": 1.2894530296325684,
+      "learning_rate": 2.4984465031640286e-07,
+      "loss": 0.2408,
+      "step": 13120
+    },
+    {
+      "epoch": 2.62,
+      "grad_norm": 1.3517787456512451,
+      "learning_rate": 2.4182936686277e-07,
+      "loss": 0.2448,
+      "step": 13152
+    },
+    {
+      "epoch": 2.63,
+      "grad_norm": 1.0544830560684204,
+      "learning_rate": 2.339382198090609e-07,
+      "loss": 0.2422,
+      "step": 13184
+    },
+    {
+      "epoch": 2.63,
+      "grad_norm": 1.0484036207199097,
+      "learning_rate": 2.261716429482652e-07,
+      "loss": 0.2444,
+      "step": 13216
+    },
+    {
+      "epoch": 2.64,
+      "grad_norm": 1.094588279724121,
+      "learning_rate": 2.1853006322548388e-07,
+      "loss": 0.2461,
+      "step": 13248
+    },
+    {
+      "epoch": 2.64,
+      "grad_norm": 1.103279948234558,
+      "learning_rate": 2.1101390071446265e-07,
+      "loss": 0.2435,
+      "step": 13280
+    },
+    {
+      "epoch": 2.65,
+      "grad_norm": 1.0666862726211548,
+      "learning_rate": 2.0362356859449954e-07,
+      "loss": 0.2487,
+      "step": 13312
+    },
+    {
+      "epoch": 2.66,
+      "grad_norm": 1.2903568744659424,
+      "learning_rate": 1.963594731277299e-07,
+      "loss": 0.2513,
+      "step": 13344
+    },
+    {
+      "epoch": 2.66,
+      "grad_norm": 1.0378022193908691,
+      "learning_rate": 1.8922201363679338e-07,
+      "loss": 0.2491,
+      "step": 13376
+    },
+    {
+      "epoch": 2.67,
+      "grad_norm": 1.1184123754501343,
+      "learning_rate": 1.8221158248288463e-07,
+      "loss": 0.2464,
+      "step": 13408
+    },
+    {
+      "epoch": 2.68,
+      "grad_norm": 1.0083402395248413,
+      "learning_rate": 1.7532856504418188e-07,
+      "loss": 0.2491,
+      "step": 13440
+    },
+    {
+      "epoch": 2.68,
+      "grad_norm": 1.2430527210235596,
+      "learning_rate": 1.6857333969466344e-07,
+      "loss": 0.2397,
+      "step": 13472
+    },
+    {
+      "epoch": 2.69,
+      "grad_norm": 1.0730229616165161,
+      "learning_rate": 1.6194627778330696e-07,
+      "loss": 0.2423,
+      "step": 13504
+    },
+    {
+      "epoch": 2.7,
+      "grad_norm": 1.1082255840301514,
+      "learning_rate": 1.5544774361367476e-07,
+      "loss": 0.2519,
+      "step": 13536
+    },
+    {
+      "epoch": 2.7,
+      "grad_norm": 1.084023356437683,
+      "learning_rate": 1.490780944238898e-07,
+      "loss": 0.2362,
+      "step": 13568
+    },
+    {
+      "epoch": 2.71,
+      "grad_norm": 1.0378090143203735,
+      "learning_rate": 1.4283768036699481e-07,
+      "loss": 0.2404,
+      "step": 13600
+    },
+    {
+      "epoch": 2.71,
+      "grad_norm": 1.2818440198898315,
+      "learning_rate": 1.3672684449170582e-07,
+      "loss": 0.2534,
+      "step": 13632
+    },
+    {
+      "epoch": 2.72,
+      "grad_norm": 1.190710186958313,
+      "learning_rate": 1.3074592272355164e-07,
+      "loss": 0.2439,
+      "step": 13664
+    },
+    {
+      "epoch": 2.73,
+      "grad_norm": 1.080256700515747,
+      "learning_rate": 1.248952438464107e-07,
+      "loss": 0.2507,
+      "step": 13696
+    },
+    {
+      "epoch": 2.73,
+      "grad_norm": 1.0920093059539795,
+      "learning_rate": 1.1917512948443288e-07,
+      "loss": 0.2414,
+      "step": 13728
+    },
+    {
+      "epoch": 2.74,
+      "grad_norm": 0.9547931551933289,
+      "learning_rate": 1.1358589408436399e-07,
+      "loss": 0.2363,
+      "step": 13760
+    },
+    {
+      "epoch": 2.75,
+      "grad_norm": 1.108270287513733,
+      "learning_rate": 1.0812784489825506e-07,
+      "loss": 0.2407,
+      "step": 13792
+    },
+    {
+      "epoch": 2.75,
+      "grad_norm": 1.083067536354065,
+      "learning_rate": 1.0280128196657579e-07,
+      "loss": 0.2494,
+      "step": 13824
+    },
+    {
+      "epoch": 2.76,
+      "grad_norm": 1.0772831439971924,
+      "learning_rate": 9.760649810171885e-08,
+      "loss": 0.2398,
+      "step": 13856
+    },
+    {
+      "epoch": 2.77,
+      "grad_norm": 1.2348062992095947,
+      "learning_rate": 9.254377887190414e-08,
+      "loss": 0.2442,
+      "step": 13888
+    },
+    {
+      "epoch": 2.77,
+      "grad_norm": 1.1681050062179565,
+      "learning_rate": 8.76134025854794e-08,
+      "loss": 0.2368,
+      "step": 13920
+    },
+    {
+      "epoch": 2.78,
+      "grad_norm": 1.2962055206298828,
+      "learning_rate": 8.281564027562305e-08,
+      "loss": 0.2409,
+      "step": 13952
+    },
+    {
+      "epoch": 2.78,
+      "grad_norm": 1.2106220722198486,
+      "learning_rate": 7.815075568544244e-08,
+      "loss": 0.2407,
+      "step": 13984
+    },
+    {
+      "epoch": 2.79,
+      "grad_norm": 1.3177499771118164,
+      "learning_rate": 7.361900525347687e-08,
+      "loss": 0.2523,
+      "step": 14016
+    },
+    {
+      "epoch": 2.8,
+      "grad_norm": 1.024245262145996,
+      "learning_rate": 6.922063809960134e-08,
+      "loss": 0.2529,
+      "step": 14048
+    },
+    {
+      "epoch": 2.8,
+      "grad_norm": 1.0427746772766113,
+      "learning_rate": 6.49558960113289e-08,
+      "loss": 0.2462,
+      "step": 14080
+    },
+    {
+      "epoch": 2.81,
+      "grad_norm": 1.2270079851150513,
+      "learning_rate": 6.082501343052327e-08,
+      "loss": 0.228,
+      "step": 14112
+    },
+    {
+      "epoch": 2.82,
+      "grad_norm": 1.09956693649292,
+      "learning_rate": 5.682821744050715e-08,
+      "loss": 0.2554,
+      "step": 14144
+    },
+    {
+      "epoch": 2.82,
+      "grad_norm": 1.0953526496887207,
+      "learning_rate": 5.296572775358166e-08,
+      "loss": 0.2422,
+      "step": 14176
+    },
+    {
+      "epoch": 2.83,
+      "grad_norm": 1.0091789960861206,
+      "learning_rate": 4.923775669894659e-08,
+      "loss": 0.247,
+      "step": 14208
+    },
+    {
+      "epoch": 2.84,
+      "grad_norm": 1.0742987394332886,
+      "learning_rate": 4.5644509211030253e-08,
+      "loss": 0.2408,
+      "step": 14240
+    },
+    {
+      "epoch": 2.84,
+      "grad_norm": 1.212641716003418,
+      "learning_rate": 4.2186182818220734e-08,
+      "loss": 0.2442,
+      "step": 14272
+    },
+    {
+      "epoch": 2.85,
+      "grad_norm": 1.024273157119751,
+      "learning_rate": 3.886296763201042e-08,
+      "loss": 0.2463,
+      "step": 14304
+    },
+    {
+      "epoch": 2.85,
+      "grad_norm": 1.2065999507904053,
+      "learning_rate": 3.567504633654323e-08,
+      "loss": 0.2456,
+      "step": 14336
+    },
+    {
+      "epoch": 2.86,
+      "grad_norm": 1.0646214485168457,
+      "learning_rate": 3.2622594178573216e-08,
+      "loss": 0.2482,
+      "step": 14368
+    },
+    {
+      "epoch": 2.87,
+      "grad_norm": 1.0027880668640137,
+      "learning_rate": 2.9705778957829777e-08,
+      "loss": 0.2411,
+      "step": 14400
+    },
+    {
+      "epoch": 2.87,
+      "grad_norm": 1.4213387966156006,
+      "learning_rate": 2.6924761017794453e-08,
+      "loss": 0.2491,
+      "step": 14432
+    },
+    {
+      "epoch": 2.88,
+      "grad_norm": 1.0677354335784912,
+      "learning_rate": 2.4279693236885503e-08,
+      "loss": 0.2421,
+      "step": 14464
+    },
+    {
+      "epoch": 2.89,
+      "grad_norm": 1.1713104248046875,
+      "learning_rate": 2.1770721020054895e-08,
+      "loss": 0.2382,
+      "step": 14496
+    },
+    {
+      "epoch": 2.89,
+      "grad_norm": 1.1479196548461914,
+      "learning_rate": 1.9397982290794147e-08,
+      "loss": 0.2359,
+      "step": 14528
+    },
+    {
+      "epoch": 2.9,
+      "grad_norm": 1.1078578233718872,
+      "learning_rate": 1.7161607483553165e-08,
+      "loss": 0.2466,
+      "step": 14560
+    },
+    {
+      "epoch": 2.91,
+      "grad_norm": 1.036629319190979,
+      "learning_rate": 1.506171953656904e-08,
+      "loss": 0.2409,
+      "step": 14592
+    },
+    {
+      "epoch": 2.91,
+      "grad_norm": 1.0767490863800049,
+      "learning_rate": 1.3098433885108953e-08,
+      "loss": 0.2406,
+      "step": 14624
+    },
+    {
+      "epoch": 2.92,
+      "grad_norm": 1.531785011291504,
+      "learning_rate": 1.1271858455123852e-08,
+      "loss": 0.249,
+      "step": 14656
+    },
+    {
+      "epoch": 2.92,
+      "grad_norm": 1.5319863557815552,
+      "learning_rate": 9.582093657315716e-09,
+      "loss": 0.24,
+      "step": 14688
+    },
+    {
+      "epoch": 2.93,
+      "grad_norm": 1.0609209537506104,
+      "learning_rate": 8.029232381616958e-09,
+      "loss": 0.2362,
+      "step": 14720
+    },
+    {
+      "epoch": 2.94,
+      "grad_norm": 1.0139707326889038,
+      "learning_rate": 6.613359992085622e-09,
+      "loss": 0.235,
+      "step": 14752
+    },
+    {
+      "epoch": 2.94,
+      "grad_norm": 1.0467095375061035,
+      "learning_rate": 5.334554322211638e-09,
+      "loss": 0.2378,
+      "step": 14784
+    },
+    {
+      "epoch": 2.95,
+      "grad_norm": 1.1878961324691772,
+      "learning_rate": 4.1928856706385805e-09,
+      "loss": 0.2405,
+      "step": 14816
+    },
+    {
+      "epoch": 2.96,
+      "grad_norm": 1.018918514251709,
+      "learning_rate": 3.1884167972984213e-09,
+      "loss": 0.2409,
+      "step": 14848
+    },
+    {
+      "epoch": 2.96,
+      "grad_norm": 1.081251859664917,
+      "learning_rate": 2.321202919962906e-09,
+      "loss": 0.2285,
+      "step": 14880
+    },
+    {
+      "epoch": 2.97,
+      "grad_norm": 1.11882483959198,
+      "learning_rate": 1.591291711207088e-09,
+      "loss": 0.247,
+      "step": 14912
+    },
+    {
+      "epoch": 2.98,
+      "grad_norm": 1.1757475137710571,
+      "learning_rate": 9.987232957880954e-10,
+      "loss": 0.2485,
+      "step": 14944
+    },
+    {
+      "epoch": 2.98,
+      "grad_norm": 1.1407628059387207,
+      "learning_rate": 5.43530248441615e-10,
+      "loss": 0.2426,
+      "step": 14976
+    },
+    {
+      "epoch": 2.99,
+      "grad_norm": 1.1428720951080322,
+      "learning_rate": 2.2573759208860401e-10,
+      "loss": 0.241,
+      "step": 15008
+    },
+    {
+      "epoch": 2.99,
+      "grad_norm": 1.1644700765609741,
+      "learning_rate": 4.536279646111252e-11,
+      "loss": 0.2475,
+      "step": 15040
+    }
+  ],
+  "logging_steps": 32,
+  "max_steps": 15066,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 15066,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

CREAM/outputs/sft/qwen_short_cot-lora_None-ckpt_None-25-10-06-01_35_46/checkpoint-15066/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f78a96d0395d693193812deedfce30bf69dd2ed10f4dc07f0a02830d90edd7c8
+size 6584