Upload folder using huggingface_hub

.gitattributes

@@ -39,3 +39,4 @@ checkpoint-5211/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoint-6948/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoint-8685/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-10422/tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,28 +1,9 @@
 ---
 base_model: Qwen/Qwen2-VL-2B-Instruct
 library_name: peft
-model_name: output
 tags:
-- adapter
 - lora
-- sft
-- transformers
-- trl
-license: apache-2.0
-pipeline_tag: text-generation
+- qwen2-vl
+- adapter
+- vision-language
 ---
-
-# Model Card for output
-
-This model is a LoRA fine-tuned version of  
-[Qwen/Qwen2-VL-2B-Instruct](https://huggingface.co/Qwen/Qwen2-VL-2B-Instruct).
-
-It has been trained using the TRL SFT pipeline.
-
-## Quick start
-
-```python
-from transformers import pipeline
-
-pipe = pipeline("text-generation", model="RetrO21/agrofinetune", device="cuda")
-print(pipe("What is nitrogen deficiency?")[0]["generated_text"])

adapter_config.json

@@ -16,7 +16,7 @@
   "layers_pattern": null,
   "layers_to_transform": null,
   "loftq_config": {},
-  "lora_alpha": 16,
+  "lora_alpha": 32,
   "lora_bias": false,
   "lora_dropout": 0.1,
   "megatron_config": null,
@@ -25,12 +25,14 @@
   "peft_type": "LORA",
   "peft_version": "0.18.0",
   "qalora_group_size": 16,
-  "r": 8,
+  "r": 24,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
     "q_proj",
-    "v_proj"
+    "k_proj",
+    "v_proj",
+    "o_proj"
   ],
   "target_parameters": null,
   "task_type": "CAUSAL_LM",

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:76b5201211b5dac5150a2b3a87809a5671a1239a76fdfafed2618f15a157a612
-size 4374520
+oid sha256:a42655e5c5bf5a17388c99c67741b81d97a904a649f92d5298361717c78abaac
+size 26182176

checkpoint-10422/README.md

@@ -0,0 +1,209 @@
+---
+base_model: ''
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen2-VL-2B-Instruct
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.0

checkpoint-10422/adapter_config.json

@@ -0,0 +1,43 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen2-VL-2B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.0",
+  "qalora_group_size": 16,
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "k_proj",
+    "v_proj",
+    "q_proj",
+    "o_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-10422/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:267663f0833a741d0dff42c5b9b564413305e7f5771f9bc4b0265a1464819af9
+size 8749064

checkpoint-10422/added_tokens.json

@@ -0,0 +1,16 @@
+{
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

checkpoint-10422/chat_template.jinja

@@ -0,0 +1,7 @@
+{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system
+You are a helpful assistant.<|im_end|>
+{% endif %}<|im_start|>{{ message['role'] }}
+{% if message['content'] is string %}{{ message['content'] }}<|im_end|>
+{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>
+{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant
+{% endif %}

checkpoint-10422/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8616e361fa88f6623812655f00db40780ff24fdbf0e3076512427426785c35f
+size 17621003

checkpoint-10422/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1b0f920b0d7a4950d6a2138471d4f887a620a6c310ba5c47cd3fdb370773865
+size 14645

checkpoint-10422/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be9fd8f2e5282151739ba25b300ba9565edc934ba51e045424e2e775d92c6b36
+size 1465

checkpoint-10422/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-10422/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f33787292af226c4a4842be48a0e614d9524e25dc248e48bb1af0593de5564f9
+size 11420539

checkpoint-10422/tokenizer_config.json

@@ -0,0 +1,143 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoint-10422/trainer_state.json

@@ -0,0 +1,2186 @@
+{
+  "best_global_step": 10422,
+  "best_metric": 6.123514175415039,
+  "best_model_checkpoint": "./output/checkpoint-10422",
+  "epoch": 6.0,
+  "eval_steps": 500,
+  "global_step": 10422,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 3.852523431777954,
+      "epoch": 0.028785261945883708,
+      "grad_norm": 6.76995325088501,
+      "learning_rate": 4.9e-07,
+      "loss": 15.3204,
+      "mean_token_accuracy": 0.10081263825297355,
+      "num_tokens": 47319.0,
+      "step": 50
+    },
+    {
+      "entropy": 3.862936944961548,
+      "epoch": 0.057570523891767415,
+      "grad_norm": 6.614922046661377,
+      "learning_rate": 9.9e-07,
+      "loss": 15.6327,
+      "mean_token_accuracy": 0.09312776654958725,
+      "num_tokens": 96809.0,
+      "step": 100
+    },
+    {
+      "entropy": 3.786734127998352,
+      "epoch": 0.08635578583765112,
+      "grad_norm": 7.106786727905273,
+      "learning_rate": 1.49e-06,
+      "loss": 14.619,
+      "mean_token_accuracy": 0.11782759010791778,
+      "num_tokens": 139962.0,
+      "step": 150
+    },
+    {
+      "entropy": 3.929054970741272,
+      "epoch": 0.11514104778353483,
+      "grad_norm": 8.141215324401855,
+      "learning_rate": 1.99e-06,
+      "loss": 15.1343,
+      "mean_token_accuracy": 0.10053366936743259,
+      "num_tokens": 188029.0,
+      "step": 200
+    },
+    {
+      "entropy": 4.070261402130127,
+      "epoch": 0.14392630972941853,
+      "grad_norm": 8.59919261932373,
+      "learning_rate": 1.9904128350616315e-06,
+      "loss": 14.4973,
+      "mean_token_accuracy": 0.11404540404677391,
+      "num_tokens": 234425.0,
+      "step": 250
+    },
+    {
+      "entropy": 4.341845245361328,
+      "epoch": 0.17271157167530224,
+      "grad_norm": 9.0352783203125,
+      "learning_rate": 1.98063001369595e-06,
+      "loss": 14.0316,
+      "mean_token_accuracy": 0.11713312789797783,
+      "num_tokens": 278885.0,
+      "step": 300
+    },
+    {
+      "entropy": 4.582782163619995,
+      "epoch": 0.20149683362118595,
+      "grad_norm": 12.131658554077148,
+      "learning_rate": 1.970847192330268e-06,
+      "loss": 13.6186,
+      "mean_token_accuracy": 0.11667421050369739,
+      "num_tokens": 325491.0,
+      "step": 350
+    },
+    {
+      "entropy": 5.054569063186645,
+      "epoch": 0.23028209556706966,
+      "grad_norm": 20.180458068847656,
+      "learning_rate": 1.961064370964586e-06,
+      "loss": 13.175,
+      "mean_token_accuracy": 0.11196398630738258,
+      "num_tokens": 372913.0,
+      "step": 400
+    },
+    {
+      "entropy": 5.657191934585572,
+      "epoch": 0.25906735751295334,
+      "grad_norm": 29.541980743408203,
+      "learning_rate": 1.9512815495989045e-06,
+      "loss": 11.8662,
+      "mean_token_accuracy": 0.11793715238571167,
+      "num_tokens": 419159.0,
+      "step": 450
+    },
+    {
+      "entropy": 6.518256826400757,
+      "epoch": 0.28785261945883706,
+      "grad_norm": 38.03327941894531,
+      "learning_rate": 1.9414987282332225e-06,
+      "loss": 10.1026,
+      "mean_token_accuracy": 0.1128273394703865,
+      "num_tokens": 466994.0,
+      "step": 500
+    },
+    {
+      "entropy": 6.959106483459473,
+      "epoch": 0.31663788140472077,
+      "grad_norm": 19.534976959228516,
+      "learning_rate": 1.9317159068675404e-06,
+      "loss": 8.5709,
+      "mean_token_accuracy": 0.12011336177587509,
+      "num_tokens": 513308.0,
+      "step": 550
+    },
+    {
+      "entropy": 7.0854248046875,
+      "epoch": 0.3454231433506045,
+      "grad_norm": 9.56103801727295,
+      "learning_rate": 1.921933085501859e-06,
+      "loss": 7.9893,
+      "mean_token_accuracy": 0.11719893589615822,
+      "num_tokens": 559679.0,
+      "step": 600
+    },
+    {
+      "entropy": 7.002319450378418,
+      "epoch": 0.3742084052964882,
+      "grad_norm": 7.537400245666504,
+      "learning_rate": 1.9121502641361767e-06,
+      "loss": 7.496,
+      "mean_token_accuracy": 0.13140189573168753,
+      "num_tokens": 603140.0,
+      "step": 650
+    },
+    {
+      "entropy": 7.21703164100647,
+      "epoch": 0.4029936672423719,
+      "grad_norm": 7.521403789520264,
+      "learning_rate": 1.902367442770495e-06,
+      "loss": 7.4496,
+      "mean_token_accuracy": 0.12282118022441864,
+      "num_tokens": 650179.0,
+      "step": 700
+    },
+    {
+      "entropy": 7.166195478439331,
+      "epoch": 0.4317789291882556,
+      "grad_norm": 5.631101608276367,
+      "learning_rate": 1.892584621404813e-06,
+      "loss": 7.2209,
+      "mean_token_accuracy": 0.1341453194618225,
+      "num_tokens": 696220.0,
+      "step": 750
+    },
+    {
+      "entropy": 7.179120960235596,
+      "epoch": 0.4605641911341393,
+      "grad_norm": 4.589099407196045,
+      "learning_rate": 1.8828018000391312e-06,
+      "loss": 7.1051,
+      "mean_token_accuracy": 0.13415986388921738,
+      "num_tokens": 743909.0,
+      "step": 800
+    },
+    {
+      "entropy": 7.132651596069336,
+      "epoch": 0.48934945308002303,
+      "grad_norm": 4.283458709716797,
+      "learning_rate": 1.8730189786734493e-06,
+      "loss": 6.9812,
+      "mean_token_accuracy": 0.13903394356369972,
+      "num_tokens": 792015.0,
+      "step": 850
+    },
+    {
+      "entropy": 7.095656795501709,
+      "epoch": 0.5181347150259067,
+      "grad_norm": 5.147945404052734,
+      "learning_rate": 1.8632361573077675e-06,
+      "loss": 6.8989,
+      "mean_token_accuracy": 0.14551860421895982,
+      "num_tokens": 841802.0,
+      "step": 900
+    },
+    {
+      "entropy": 6.79542833328247,
+      "epoch": 0.5469199769717904,
+      "grad_norm": 5.6646223068237305,
+      "learning_rate": 1.8534533359420857e-06,
+      "loss": 6.5807,
+      "mean_token_accuracy": 0.17714839324355125,
+      "num_tokens": 886492.0,
+      "step": 950
+    },
+    {
+      "entropy": 6.843292989730835,
+      "epoch": 0.5757052389176741,
+      "grad_norm": 4.838613033294678,
+      "learning_rate": 1.8436705145764038e-06,
+      "loss": 6.6068,
+      "mean_token_accuracy": 0.17419333070516585,
+      "num_tokens": 932807.0,
+      "step": 1000
+    },
+    {
+      "entropy": 6.8572557926177975,
+      "epoch": 0.6044905008635578,
+      "grad_norm": 4.200118541717529,
+      "learning_rate": 1.833887693210722e-06,
+      "loss": 6.6058,
+      "mean_token_accuracy": 0.1721690407395363,
+      "num_tokens": 980541.0,
+      "step": 1050
+    },
+    {
+      "entropy": 6.629744396209717,
+      "epoch": 0.6332757628094415,
+      "grad_norm": 5.378969192504883,
+      "learning_rate": 1.8241048718450401e-06,
+      "loss": 6.3675,
+      "mean_token_accuracy": 0.19682477086782454,
+      "num_tokens": 1023882.0,
+      "step": 1100
+    },
+    {
+      "entropy": 6.537129864692688,
+      "epoch": 0.6620610247553252,
+      "grad_norm": 6.9036736488342285,
+      "learning_rate": 1.814322050479358e-06,
+      "loss": 6.2659,
+      "mean_token_accuracy": 0.20461874470114708,
+      "num_tokens": 1068300.0,
+      "step": 1150
+    },
+    {
+      "entropy": 6.6863781929016115,
+      "epoch": 0.690846286701209,
+      "grad_norm": 4.751266002655029,
+      "learning_rate": 1.8045392291136762e-06,
+      "loss": 6.4062,
+      "mean_token_accuracy": 0.1890461677312851,
+      "num_tokens": 1115425.0,
+      "step": 1200
+    },
+    {
+      "entropy": 6.71572250366211,
+      "epoch": 0.7196315486470927,
+      "grad_norm": 3.819430351257324,
+      "learning_rate": 1.7947564077479944e-06,
+      "loss": 6.4309,
+      "mean_token_accuracy": 0.1853047838807106,
+      "num_tokens": 1162319.0,
+      "step": 1250
+    },
+    {
+      "entropy": 6.632803421020508,
+      "epoch": 0.7484168105929764,
+      "grad_norm": 6.341519832611084,
+      "learning_rate": 1.7849735863823125e-06,
+      "loss": 6.3439,
+      "mean_token_accuracy": 0.19532681837677957,
+      "num_tokens": 1208230.0,
+      "step": 1300
+    },
+    {
+      "entropy": 6.5115529870986935,
+      "epoch": 0.7772020725388601,
+      "grad_norm": 6.07994270324707,
+      "learning_rate": 1.7751907650166307e-06,
+      "loss": 6.227,
+      "mean_token_accuracy": 0.20865949630737304,
+      "num_tokens": 1253074.0,
+      "step": 1350
+    },
+    {
+      "entropy": 6.603812961578369,
+      "epoch": 0.8059873344847438,
+      "grad_norm": 5.038186073303223,
+      "learning_rate": 1.7654079436509488e-06,
+      "loss": 6.3205,
+      "mean_token_accuracy": 0.19723004043102266,
+      "num_tokens": 1300179.0,
+      "step": 1400
+    },
+    {
+      "entropy": 6.614377698898315,
+      "epoch": 0.8347725964306275,
+      "grad_norm": 5.706110000610352,
+      "learning_rate": 1.755625122285267e-06,
+      "loss": 6.3415,
+      "mean_token_accuracy": 0.19426312118768693,
+      "num_tokens": 1347539.0,
+      "step": 1450
+    },
+    {
+      "entropy": 6.535960426330567,
+      "epoch": 0.8635578583765112,
+      "grad_norm": 3.8547616004943848,
+      "learning_rate": 1.7458423009195851e-06,
+      "loss": 6.2563,
+      "mean_token_accuracy": 0.20262083023786545,
+      "num_tokens": 1394157.0,
+      "step": 1500
+    },
+    {
+      "entropy": 6.635546321868897,
+      "epoch": 0.8923431203223949,
+      "grad_norm": 6.530314922332764,
+      "learning_rate": 1.7360594795539033e-06,
+      "loss": 6.3569,
+      "mean_token_accuracy": 0.1919993445277214,
+      "num_tokens": 1443892.0,
+      "step": 1550
+    },
+    {
+      "entropy": 6.463044328689575,
+      "epoch": 0.9211283822682786,
+      "grad_norm": 4.981988430023193,
+      "learning_rate": 1.7262766581882212e-06,
+      "loss": 6.1768,
+      "mean_token_accuracy": 0.21415023148059845,
+      "num_tokens": 1491050.0,
+      "step": 1600
+    },
+    {
+      "entropy": 6.614933052062988,
+      "epoch": 0.9499136442141624,
+      "grad_norm": 3.6533243656158447,
+      "learning_rate": 1.7164938368225394e-06,
+      "loss": 6.3213,
+      "mean_token_accuracy": 0.19727844208478929,
+      "num_tokens": 1540809.0,
+      "step": 1650
+    },
+    {
+      "entropy": 6.38832573890686,
+      "epoch": 0.9786989061600461,
+      "grad_norm": 5.930168151855469,
+      "learning_rate": 1.7067110154568575e-06,
+      "loss": 6.0941,
+      "mean_token_accuracy": 0.22453365564346314,
+      "num_tokens": 1585876.0,
+      "step": 1700
+    },
+    {
+      "epoch": 1.0,
+      "eval_entropy": 6.647906507764544,
+      "eval_loss": 6.368417263031006,
+      "eval_mean_token_accuracy": 0.1902703122334546,
+      "eval_model_preparation_time": 0.0046,
+      "eval_num_tokens": 1619719.0,
+      "eval_runtime": 79.1489,
+      "eval_samples_per_second": 5.483,
+      "eval_steps_per_second": 2.742,
+      "step": 1737
+    },
+    {
+      "entropy": 6.413019351959228,
+      "epoch": 1.0074841681059297,
+      "grad_norm": 4.029679298400879,
+      "learning_rate": 1.6969281940911757e-06,
+      "loss": 6.1183,
+      "mean_token_accuracy": 0.2218746316432953,
+      "num_tokens": 1632015.0,
+      "step": 1750
+    },
+    {
+      "entropy": 6.508082237243652,
+      "epoch": 1.0362694300518134,
+      "grad_norm": 4.616228103637695,
+      "learning_rate": 1.687145372725494e-06,
+      "loss": 6.2151,
+      "mean_token_accuracy": 0.2117026337981224,
+      "num_tokens": 1681154.0,
+      "step": 1800
+    },
+    {
+      "entropy": 6.401120796203613,
+      "epoch": 1.065054691997697,
+      "grad_norm": 4.693721771240234,
+      "learning_rate": 1.6773625513598122e-06,
+      "loss": 6.1039,
+      "mean_token_accuracy": 0.22995314985513687,
+      "num_tokens": 1728110.0,
+      "step": 1850
+    },
+    {
+      "entropy": 6.3726827430725095,
+      "epoch": 1.0938399539435808,
+      "grad_norm": 3.4032232761383057,
+      "learning_rate": 1.6675797299941304e-06,
+      "loss": 6.0875,
+      "mean_token_accuracy": 0.23210913449525833,
+      "num_tokens": 1775703.0,
+      "step": 1900
+    },
+    {
+      "entropy": 6.328955335617065,
+      "epoch": 1.1226252158894645,
+      "grad_norm": 5.2645440101623535,
+      "learning_rate": 1.6577969086284485e-06,
+      "loss": 6.0535,
+      "mean_token_accuracy": 0.23541803926229476,
+      "num_tokens": 1821980.0,
+      "step": 1950
+    },
+    {
+      "entropy": 6.272669095993042,
+      "epoch": 1.1514104778353482,
+      "grad_norm": 3.2737088203430176,
+      "learning_rate": 1.6480140872627667e-06,
+      "loss": 6.0091,
+      "mean_token_accuracy": 0.24000794380903245,
+      "num_tokens": 1867547.0,
+      "step": 2000
+    },
+    {
+      "entropy": 6.346148128509522,
+      "epoch": 1.180195739781232,
+      "grad_norm": 29.210887908935547,
+      "learning_rate": 1.6382312658970846e-06,
+      "loss": 6.0883,
+      "mean_token_accuracy": 0.22887098014354706,
+      "num_tokens": 1915411.0,
+      "step": 2050
+    },
+    {
+      "entropy": 6.279903531074524,
+      "epoch": 1.2089810017271156,
+      "grad_norm": 3.977229356765747,
+      "learning_rate": 1.6284484445314028e-06,
+      "loss": 6.0318,
+      "mean_token_accuracy": 0.23324142932891845,
+      "num_tokens": 1964009.0,
+      "step": 2100
+    },
+    {
+      "entropy": 6.166584692001343,
+      "epoch": 1.2377662636729994,
+      "grad_norm": 3.1655149459838867,
+      "learning_rate": 1.618665623165721e-06,
+      "loss": 5.917,
+      "mean_token_accuracy": 0.24853385210037232,
+      "num_tokens": 2008595.0,
+      "step": 2150
+    },
+    {
+      "entropy": 6.322167301177979,
+      "epoch": 1.266551525618883,
+      "grad_norm": 2.878366708755493,
+      "learning_rate": 1.608882801800039e-06,
+      "loss": 6.0756,
+      "mean_token_accuracy": 0.2293447071313858,
+      "num_tokens": 2057229.0,
+      "step": 2200
+    },
+    {
+      "entropy": 6.205327453613282,
+      "epoch": 1.2953367875647668,
+      "grad_norm": 2.5909852981567383,
+      "learning_rate": 1.5990999804343572e-06,
+      "loss": 5.962,
+      "mean_token_accuracy": 0.24392724603414537,
+      "num_tokens": 2103631.0,
+      "step": 2250
+    },
+    {
+      "entropy": 6.157236385345459,
+      "epoch": 1.3241220495106505,
+      "grad_norm": 6.303485870361328,
+      "learning_rate": 1.5893171590686754e-06,
+      "loss": 5.9162,
+      "mean_token_accuracy": 0.24806494176387786,
+      "num_tokens": 2150369.0,
+      "step": 2300
+    },
+    {
+      "entropy": 6.351038675308228,
+      "epoch": 1.3529073114565342,
+      "grad_norm": 3.6839759349823,
+      "learning_rate": 1.5795343377029935e-06,
+      "loss": 6.1091,
+      "mean_token_accuracy": 0.22586097091436386,
+      "num_tokens": 2199724.0,
+      "step": 2350
+    },
+    {
+      "entropy": 6.134297747611999,
+      "epoch": 1.381692573402418,
+      "grad_norm": 2.5310072898864746,
+      "learning_rate": 1.5697515163373117e-06,
+      "loss": 5.9017,
+      "mean_token_accuracy": 0.24992096066474914,
+      "num_tokens": 2245341.0,
+      "step": 2400
+    },
+    {
+      "entropy": 6.322179689407348,
+      "epoch": 1.4104778353483016,
+      "grad_norm": 2.834397554397583,
+      "learning_rate": 1.5599686949716298e-06,
+      "loss": 6.0913,
+      "mean_token_accuracy": 0.22521918207407,
+      "num_tokens": 2294726.0,
+      "step": 2450
+    },
+    {
+      "entropy": 6.0980473279953005,
+      "epoch": 1.4392630972941853,
+      "grad_norm": 3.1855642795562744,
+      "learning_rate": 1.5501858736059478e-06,
+      "loss": 5.8841,
+      "mean_token_accuracy": 0.25087269872426987,
+      "num_tokens": 2342251.0,
+      "step": 2500
+    },
+    {
+      "entropy": 6.088696489334106,
+      "epoch": 1.468048359240069,
+      "grad_norm": 5.114110946655273,
+      "learning_rate": 1.540403052240266e-06,
+      "loss": 5.8739,
+      "mean_token_accuracy": 0.2518083402514458,
+      "num_tokens": 2387469.0,
+      "step": 2550
+    },
+    {
+      "entropy": 6.151525087356568,
+      "epoch": 1.4968336211859528,
+      "grad_norm": 2.6623592376708984,
+      "learning_rate": 1.530620230874584e-06,
+      "loss": 5.9402,
+      "mean_token_accuracy": 0.24315184772014617,
+      "num_tokens": 2434128.0,
+      "step": 2600
+    },
+    {
+      "entropy": 6.112346210479736,
+      "epoch": 1.5256188831318365,
+      "grad_norm": 4.4492950439453125,
+      "learning_rate": 1.5208374095089022e-06,
+      "loss": 5.9062,
+      "mean_token_accuracy": 0.24730559319257736,
+      "num_tokens": 2480366.0,
+      "step": 2650
+    },
+    {
+      "entropy": 6.088384003639221,
+      "epoch": 1.5544041450777202,
+      "grad_norm": 2.631941556930542,
+      "learning_rate": 1.5110545881432204e-06,
+      "loss": 5.8806,
+      "mean_token_accuracy": 0.2491958048939705,
+      "num_tokens": 2527357.0,
+      "step": 2700
+    },
+    {
+      "entropy": 6.054274072647095,
+      "epoch": 1.583189407023604,
+      "grad_norm": 3.9610729217529297,
+      "learning_rate": 1.5012717667775385e-06,
+      "loss": 5.851,
+      "mean_token_accuracy": 0.25282351911067963,
+      "num_tokens": 2574470.0,
+      "step": 2750
+    },
+    {
+      "entropy": 6.205899753570557,
+      "epoch": 1.6119746689694876,
+      "grad_norm": 2.052320957183838,
+      "learning_rate": 1.4914889454118567e-06,
+      "loss": 6.0034,
+      "mean_token_accuracy": 0.23449385523796082,
+      "num_tokens": 2622280.0,
+      "step": 2800
+    },
+    {
+      "entropy": 6.037883262634278,
+      "epoch": 1.6407599309153713,
+      "grad_norm": 1.5475044250488281,
+      "learning_rate": 1.4817061240461749e-06,
+      "loss": 5.8417,
+      "mean_token_accuracy": 0.2532995194196701,
+      "num_tokens": 2668624.0,
+      "step": 2850
+    },
+    {
+      "entropy": 6.011255393028259,
+      "epoch": 1.669545192861255,
+      "grad_norm": 1.50232994556427,
+      "learning_rate": 1.471923302680493e-06,
+      "loss": 5.8213,
+      "mean_token_accuracy": 0.2553745821118355,
+      "num_tokens": 2714388.0,
+      "step": 2900
+    },
+    {
+      "entropy": 6.172232007980346,
+      "epoch": 1.6983304548071387,
+      "grad_norm": 1.9385855197906494,
+      "learning_rate": 1.462140481314811e-06,
+      "loss": 5.9765,
+      "mean_token_accuracy": 0.23626189529895783,
+      "num_tokens": 2761465.0,
+      "step": 2950
+    },
+    {
+      "entropy": 6.066384444236755,
+      "epoch": 1.7271157167530224,
+      "grad_norm": 2.6527063846588135,
+      "learning_rate": 1.452357659949129e-06,
+      "loss": 5.8701,
+      "mean_token_accuracy": 0.24866942584514617,
+      "num_tokens": 2808066.0,
+      "step": 3000
+    },
+    {
+      "entropy": 5.876337275505066,
+      "epoch": 1.7559009786989062,
+      "grad_norm": 2.4430501461029053,
+      "learning_rate": 1.4425748385834473e-06,
+      "loss": 5.6826,
+      "mean_token_accuracy": 0.27362876415252685,
+      "num_tokens": 2851822.0,
+      "step": 3050
+    },
+    {
+      "entropy": 6.044622054100037,
+      "epoch": 1.7846862406447899,
+      "grad_norm": 3.2790579795837402,
+      "learning_rate": 1.4327920172177654e-06,
+      "loss": 5.8551,
+      "mean_token_accuracy": 0.25071538865566256,
+      "num_tokens": 2897737.0,
+      "step": 3100
+    },
+    {
+      "entropy": 5.777814731597901,
+      "epoch": 1.8134715025906736,
+      "grad_norm": 1.7892365455627441,
+      "learning_rate": 1.4230091958520836e-06,
+      "loss": 5.5954,
+      "mean_token_accuracy": 0.284136081635952,
+      "num_tokens": 2939511.0,
+      "step": 3150
+    },
+    {
+      "entropy": 6.034259614944458,
+      "epoch": 1.8422567645365573,
+      "grad_norm": 1.7564071416854858,
+      "learning_rate": 1.413226374486402e-06,
+      "loss": 5.848,
+      "mean_token_accuracy": 0.25160137861967086,
+      "num_tokens": 2986368.0,
+      "step": 3200
+    },
+    {
+      "entropy": 6.0115156078338625,
+      "epoch": 1.871042026482441,
+      "grad_norm": 2.3167052268981934,
+      "learning_rate": 1.40344355312072e-06,
+      "loss": 5.8269,
+      "mean_token_accuracy": 0.25526676297187806,
+      "num_tokens": 3031770.0,
+      "step": 3250
+    },
+    {
+      "entropy": 6.0657948303222655,
+      "epoch": 1.8998272884283247,
+      "grad_norm": 1.765837550163269,
+      "learning_rate": 1.3936607317550382e-06,
+      "loss": 5.8765,
+      "mean_token_accuracy": 0.24879903554916383,
+      "num_tokens": 3078322.0,
+      "step": 3300
+    },
+    {
+      "entropy": 6.146444616317749,
+      "epoch": 1.9286125503742084,
+      "grad_norm": 2.933809518814087,
+      "learning_rate": 1.3838779103893564e-06,
+      "loss": 5.9625,
+      "mean_token_accuracy": 0.23642315745353698,
+      "num_tokens": 3125572.0,
+      "step": 3350
+    },
+    {
+      "entropy": 6.007315292358398,
+      "epoch": 1.9573978123200921,
+      "grad_norm": 1.7006982564926147,
+      "learning_rate": 1.3740950890236743e-06,
+      "loss": 5.8227,
+      "mean_token_accuracy": 0.25394665479660034,
+      "num_tokens": 3171974.0,
+      "step": 3400
+    },
+    {
+      "entropy": 6.091508469581604,
+      "epoch": 1.9861830742659758,
+      "grad_norm": 1.8032574653625488,
+      "learning_rate": 1.3643122676579925e-06,
+      "loss": 5.9103,
+      "mean_token_accuracy": 0.24359373539686202,
+      "num_tokens": 3219624.0,
+      "step": 3450
+    },
+    {
+      "epoch": 2.0,
+      "eval_entropy": 6.323629730857462,
+      "eval_loss": 6.1541829109191895,
+      "eval_mean_token_accuracy": 0.20884785385725135,
+      "eval_model_preparation_time": 0.0046,
+      "eval_num_tokens": 3239438.0,
+      "eval_runtime": 79.0616,
+      "eval_samples_per_second": 5.489,
+      "eval_steps_per_second": 2.745,
+      "step": 3474
+    },
+    {
+      "entropy": 5.967442779541016,
+      "epoch": 2.0149683362118593,
+      "grad_norm": 1.3615084886550903,
+      "learning_rate": 1.3545294462923106e-06,
+      "loss": 5.7883,
+      "mean_token_accuracy": 0.25975353181362154,
+      "num_tokens": 3263994.0,
+      "step": 3500
+    },
+    {
+      "entropy": 6.085220527648926,
+      "epoch": 2.043753598157743,
+      "grad_norm": 2.406777858734131,
+      "learning_rate": 1.3447466249266288e-06,
+      "loss": 5.9002,
+      "mean_token_accuracy": 0.24500031709671022,
+      "num_tokens": 3311182.0,
+      "step": 3550
+    },
+    {
+      "entropy": 6.010667142868042,
+      "epoch": 2.0725388601036268,
+      "grad_norm": 4.209227561950684,
+      "learning_rate": 1.334963803560947e-06,
+      "loss": 5.8366,
+      "mean_token_accuracy": 0.2527648264169693,
+      "num_tokens": 3358036.0,
+      "step": 3600
+    },
+    {
+      "entropy": 6.040924577713013,
+      "epoch": 2.1013241220495105,
+      "grad_norm": 3.2806403636932373,
+      "learning_rate": 1.325180982195265e-06,
+      "loss": 5.8649,
+      "mean_token_accuracy": 0.24962294459342957,
+      "num_tokens": 3404058.0,
+      "step": 3650
+    },
+    {
+      "entropy": 6.023610129356384,
+      "epoch": 2.130109383995394,
+      "grad_norm": 1.1922718286514282,
+      "learning_rate": 1.3153981608295833e-06,
+      "loss": 5.8519,
+      "mean_token_accuracy": 0.25007107347249985,
+      "num_tokens": 3449834.0,
+      "step": 3700
+    },
+    {
+      "entropy": 5.89481824874878,
+      "epoch": 2.158894645941278,
+      "grad_norm": 1.8002029657363892,
+      "learning_rate": 1.3056153394639014e-06,
+      "loss": 5.7206,
+      "mean_token_accuracy": 0.26725934326648715,
+      "num_tokens": 3494740.0,
+      "step": 3750
+    },
+    {
+      "entropy": 6.008778114318847,
+      "epoch": 2.1876799078871616,
+      "grad_norm": 2.3413538932800293,
+      "learning_rate": 1.2958325180982196e-06,
+      "loss": 5.8343,
+      "mean_token_accuracy": 0.2532099911570549,
+      "num_tokens": 3541342.0,
+      "step": 3800
+    },
+    {
+      "entropy": 6.085102453231811,
+      "epoch": 2.2164651698330453,
+      "grad_norm": 1.7294431924819946,
+      "learning_rate": 1.2860496967325375e-06,
+      "loss": 5.9104,
+      "mean_token_accuracy": 0.2436734887957573,
+      "num_tokens": 3588995.0,
+      "step": 3850
+    },
+    {
+      "entropy": 5.947706546783447,
+      "epoch": 2.245250431778929,
+      "grad_norm": 1.6259620189666748,
+      "learning_rate": 1.2762668753668557e-06,
+      "loss": 5.78,
+      "mean_token_accuracy": 0.2602892768383026,
+      "num_tokens": 3634252.0,
+      "step": 3900
+    },
+    {
+      "entropy": 5.989762544631958,
+      "epoch": 2.2740356937248127,
+      "grad_norm": 1.664301872253418,
+      "learning_rate": 1.2664840540011738e-06,
+      "loss": 5.8189,
+      "mean_token_accuracy": 0.2546903318166733,
+      "num_tokens": 3681197.0,
+      "step": 3950
+    },
+    {
+      "entropy": 6.187751932144165,
+      "epoch": 2.3028209556706964,
+      "grad_norm": 3.428220748901367,
+      "learning_rate": 1.256701232635492e-06,
+      "loss": 6.0137,
+      "mean_token_accuracy": 0.23033296406269074,
+      "num_tokens": 3729955.0,
+      "step": 4000
+    },
+    {
+      "entropy": 6.038392038345337,
+      "epoch": 2.33160621761658,
+      "grad_norm": 2.1140899658203125,
+      "learning_rate": 1.2469184112698101e-06,
+      "loss": 5.8655,
+      "mean_token_accuracy": 0.24881428897380828,
+      "num_tokens": 3777043.0,
+      "step": 4050
+    },
+    {
+      "entropy": 6.071309795379639,
+      "epoch": 2.360391479562464,
+      "grad_norm": 1.344217300415039,
+      "learning_rate": 1.2371355899041283e-06,
+      "loss": 5.8991,
+      "mean_token_accuracy": 0.2440922862291336,
+      "num_tokens": 3824067.0,
+      "step": 4100
+    },
+    {
+      "entropy": 6.129210476875305,
+      "epoch": 2.3891767415083476,
+      "grad_norm": 1.578134536743164,
+      "learning_rate": 1.2273527685384464e-06,
+      "loss": 5.9573,
+      "mean_token_accuracy": 0.23632006645202636,
+      "num_tokens": 3872769.0,
+      "step": 4150
+    },
+    {
+      "entropy": 6.0412983751297,
+      "epoch": 2.4179620034542313,
+      "grad_norm": 1.5530976057052612,
+      "learning_rate": 1.2175699471727646e-06,
+      "loss": 5.8701,
+      "mean_token_accuracy": 0.24784765332937242,
+      "num_tokens": 3919379.0,
+      "step": 4200
+    },
+    {
+      "entropy": 6.002105917930603,
+      "epoch": 2.446747265400115,
+      "grad_norm": 1.6028035879135132,
+      "learning_rate": 1.2077871258070827e-06,
+      "loss": 5.8313,
+      "mean_token_accuracy": 0.25332365930080414,
+      "num_tokens": 3965593.0,
+      "step": 4250
+    },
+    {
+      "entropy": 6.06869218826294,
+      "epoch": 2.4755325273459987,
+      "grad_norm": 1.5630944967269897,
+      "learning_rate": 1.1980043044414007e-06,
+      "loss": 5.8973,
+      "mean_token_accuracy": 0.24471112668514253,
+      "num_tokens": 4012300.0,
+      "step": 4300
+    },
+    {
+      "entropy": 6.019678201675415,
+      "epoch": 2.5043177892918824,
+      "grad_norm": 1.9821183681488037,
+      "learning_rate": 1.1882214830757188e-06,
+      "loss": 5.8526,
+      "mean_token_accuracy": 0.24976039975881575,
+      "num_tokens": 4059323.0,
+      "step": 4350
+    },
+    {
+      "entropy": 6.157129697799682,
+      "epoch": 2.533103051237766,
+      "grad_norm": 3.1856675148010254,
+      "learning_rate": 1.178438661710037e-06,
+      "loss": 5.9868,
+      "mean_token_accuracy": 0.23358212381601334,
+      "num_tokens": 4107616.0,
+      "step": 4400
+    },
+    {
+      "entropy": 5.9199522733688354,
+      "epoch": 2.56188831318365,
+      "grad_norm": 2.0129523277282715,
+      "learning_rate": 1.1686558403443551e-06,
+      "loss": 5.7537,
+      "mean_token_accuracy": 0.26219907581806184,
+      "num_tokens": 4152400.0,
+      "step": 4450
+    },
+    {
+      "entropy": 6.141581220626831,
+      "epoch": 2.5906735751295336,
+      "grad_norm": 1.4197176694869995,
+      "learning_rate": 1.1588730189786733e-06,
+      "loss": 5.9746,
+      "mean_token_accuracy": 0.23547348588705064,
+      "num_tokens": 4200994.0,
+      "step": 4500
+    },
+    {
+      "entropy": 6.024065284729004,
+      "epoch": 2.6194588370754173,
+      "grad_norm": 2.5414512157440186,
+      "learning_rate": 1.1490901976129917e-06,
+      "loss": 5.8542,
+      "mean_token_accuracy": 0.24906692177057266,
+      "num_tokens": 4247548.0,
+      "step": 4550
+    },
+    {
+      "entropy": 6.087933650016785,
+      "epoch": 2.648244099021301,
+      "grad_norm": 1.2823543548583984,
+      "learning_rate": 1.1393073762473098e-06,
+      "loss": 5.9213,
+      "mean_token_accuracy": 0.24062541306018828,
+      "num_tokens": 4295250.0,
+      "step": 4600
+    },
+    {
+      "entropy": 6.043813619613648,
+      "epoch": 2.6770293609671847,
+      "grad_norm": 1.046730637550354,
+      "learning_rate": 1.129524554881628e-06,
+      "loss": 5.8741,
+      "mean_token_accuracy": 0.24775829553604126,
+      "num_tokens": 4341599.0,
+      "step": 4650
+    },
+    {
+      "entropy": 6.004058070182801,
+      "epoch": 2.7058146229130684,
+      "grad_norm": 1.243298053741455,
+      "learning_rate": 1.1197417335159461e-06,
+      "loss": 5.8366,
+      "mean_token_accuracy": 0.25220848590135575,
+      "num_tokens": 4388673.0,
+      "step": 4700
+    },
+    {
+      "entropy": 5.835509791374206,
+      "epoch": 2.734599884858952,
+      "grad_norm": 2.773327350616455,
+      "learning_rate": 1.109958912150264e-06,
+      "loss": 5.6767,
+      "mean_token_accuracy": 0.2719315069913864,
+      "num_tokens": 4432959.0,
+      "step": 4750
+    },
+    {
+      "entropy": 6.021662483215332,
+      "epoch": 2.763385146804836,
+      "grad_norm": 24.627521514892578,
+      "learning_rate": 1.1001760907845822e-06,
+      "loss": 5.8559,
+      "mean_token_accuracy": 0.24981790155172348,
+      "num_tokens": 4479190.0,
+      "step": 4800
+    },
+    {
+      "entropy": 5.997534699440003,
+      "epoch": 2.7921704087507195,
+      "grad_norm": 0.936356246471405,
+      "learning_rate": 1.0903932694189004e-06,
+      "loss": 5.8337,
+      "mean_token_accuracy": 0.2518752273917198,
+      "num_tokens": 4525674.0,
+      "step": 4850
+    },
+    {
+      "entropy": 5.853120732307434,
+      "epoch": 2.8209556706966032,
+      "grad_norm": 1.5253357887268066,
+      "learning_rate": 1.0806104480532185e-06,
+      "loss": 5.6906,
+      "mean_token_accuracy": 0.2703215056657791,
+      "num_tokens": 4570379.0,
+      "step": 4900
+    },
+    {
+      "entropy": 6.003798789978028,
+      "epoch": 2.849740932642487,
+      "grad_norm": 7.387447834014893,
+      "learning_rate": 1.0708276266875367e-06,
+      "loss": 5.8363,
+      "mean_token_accuracy": 0.2520116460323334,
+      "num_tokens": 4617184.0,
+      "step": 4950
+    },
+    {
+      "entropy": 6.044828844070435,
+      "epoch": 2.8785261945883707,
+      "grad_norm": 1.7473825216293335,
+      "learning_rate": 1.0610448053218548e-06,
+      "loss": 5.8824,
+      "mean_token_accuracy": 0.2459094214439392,
+      "num_tokens": 4664180.0,
+      "step": 5000
+    },
+    {
+      "entropy": 5.870430383682251,
+      "epoch": 2.9073114565342544,
+      "grad_norm": 0.9340764880180359,
+      "learning_rate": 1.051261983956173e-06,
+      "loss": 5.7101,
+      "mean_token_accuracy": 0.2679078412055969,
+      "num_tokens": 4708377.0,
+      "step": 5050
+    },
+    {
+      "entropy": 5.880399878025055,
+      "epoch": 2.936096718480138,
+      "grad_norm": 1.3693302869796753,
+      "learning_rate": 1.0414791625904911e-06,
+      "loss": 5.72,
+      "mean_token_accuracy": 0.2662310737371445,
+      "num_tokens": 4753587.0,
+      "step": 5100
+    },
+    {
+      "entropy": 6.051638517379761,
+      "epoch": 2.964881980426022,
+      "grad_norm": 1.886895775794983,
+      "learning_rate": 1.0316963412248093e-06,
+      "loss": 5.8882,
+      "mean_token_accuracy": 0.2440450206398964,
+      "num_tokens": 4800508.0,
+      "step": 5150
+    },
+    {
+      "entropy": 6.135327701568603,
+      "epoch": 2.9936672423719055,
+      "grad_norm": 1.1313307285308838,
+      "learning_rate": 1.0219135198591272e-06,
+      "loss": 5.9694,
+      "mean_token_accuracy": 0.23429417878389358,
+      "num_tokens": 4849415.0,
+      "step": 5200
+    },
+    {
+      "epoch": 3.0,
+      "eval_entropy": 6.290762787041027,
+      "eval_loss": 6.133134365081787,
+      "eval_mean_token_accuracy": 0.21000784566874878,
+      "eval_model_preparation_time": 0.0046,
+      "eval_num_tokens": 4859157.0,
+      "eval_runtime": 79.4078,
+      "eval_samples_per_second": 5.465,
+      "eval_steps_per_second": 2.733,
+      "step": 5211
+    },
+    {
+      "entropy": 5.866470074653625,
+      "epoch": 3.0224525043177892,
+      "grad_norm": 1.7904499769210815,
+      "learning_rate": 1.0121306984934454e-06,
+      "loss": 5.7064,
+      "mean_token_accuracy": 0.26748142033815386,
+      "num_tokens": 4893297.0,
+      "step": 5250
+    },
+    {
+      "entropy": 6.0223666858673095,
+      "epoch": 3.051237766263673,
+      "grad_norm": 1.4165620803833008,
+      "learning_rate": 1.0023478771277635e-06,
+      "loss": 5.855,
+      "mean_token_accuracy": 0.2492792472243309,
+      "num_tokens": 4940190.0,
+      "step": 5300
+    },
+    {
+      "entropy": 5.958261919021607,
+      "epoch": 3.0800230282095566,
+      "grad_norm": 1.7037155628204346,
+      "learning_rate": 9.925650557620817e-07,
+      "loss": 5.7943,
+      "mean_token_accuracy": 0.2563889327645302,
+      "num_tokens": 4986555.0,
+      "step": 5350
+    },
+    {
+      "entropy": 5.975116381645202,
+      "epoch": 3.1088082901554404,
+      "grad_norm": 1.5214799642562866,
+      "learning_rate": 9.827822343963998e-07,
+      "loss": 5.8129,
+      "mean_token_accuracy": 0.2538496914505959,
+      "num_tokens": 5033343.0,
+      "step": 5400
+    },
+    {
+      "entropy": 6.052438821792602,
+      "epoch": 3.137593552101324,
+      "grad_norm": 1.4624167680740356,
+      "learning_rate": 9.72999413030718e-07,
+      "loss": 5.8895,
+      "mean_token_accuracy": 0.24580927312374115,
+      "num_tokens": 5080498.0,
+      "step": 5450
+    },
+    {
+      "entropy": 5.986911368370056,
+      "epoch": 3.166378814047208,
+      "grad_norm": 0.6274769306182861,
+      "learning_rate": 9.632165916650362e-07,
+      "loss": 5.8291,
+      "mean_token_accuracy": 0.2532012587785721,
+      "num_tokens": 5126524.0,
+      "step": 5500
+    },
+    {
+      "entropy": 6.156058435440063,
+      "epoch": 3.1951640759930915,
+      "grad_norm": 1.5531014204025269,
+      "learning_rate": 9.534337702993543e-07,
+      "loss": 5.9887,
+      "mean_token_accuracy": 0.23208704799413682,
+      "num_tokens": 5176187.0,
+      "step": 5550
+    },
+    {
+      "entropy": 6.023375058174134,
+      "epoch": 3.223949337938975,
+      "grad_norm": 1.4891023635864258,
+      "learning_rate": 9.436509489336725e-07,
+      "loss": 5.8608,
+      "mean_token_accuracy": 0.249232979118824,
+      "num_tokens": 5223101.0,
+      "step": 5600
+    },
+    {
+      "entropy": 6.018003768920899,
+      "epoch": 3.252734599884859,
+      "grad_norm": 2.151552438735962,
+      "learning_rate": 9.338681275679906e-07,
+      "loss": 5.857,
+      "mean_token_accuracy": 0.24891259402036667,
+      "num_tokens": 5270200.0,
+      "step": 5650
+    },
+    {
+      "entropy": 6.016556148529053,
+      "epoch": 3.2815198618307426,
+      "grad_norm": 1.4065567255020142,
+      "learning_rate": 9.240853062023088e-07,
+      "loss": 5.8526,
+      "mean_token_accuracy": 0.24942608833312988,
+      "num_tokens": 5315872.0,
+      "step": 5700
+    },
+    {
+      "entropy": 6.045969610214233,
+      "epoch": 3.3103051237766263,
+      "grad_norm": 1.8933031558990479,
+      "learning_rate": 9.143024848366268e-07,
+      "loss": 5.8824,
+      "mean_token_accuracy": 0.24579768538475036,
+      "num_tokens": 5362860.0,
+      "step": 5750
+    },
+    {
+      "entropy": 5.900783424377441,
+      "epoch": 3.33909038572251,
+      "grad_norm": 2.198502540588379,
+      "learning_rate": 9.04519663470945e-07,
+      "loss": 5.7435,
+      "mean_token_accuracy": 0.2635406255722046,
+      "num_tokens": 5407854.0,
+      "step": 5800
+    },
+    {
+      "entropy": 5.739368691444397,
+      "epoch": 3.3678756476683938,
+      "grad_norm": 1.9872454404830933,
+      "learning_rate": 8.947368421052631e-07,
+      "loss": 5.581,
+      "mean_token_accuracy": 0.28485828697681426,
+      "num_tokens": 5450830.0,
+      "step": 5850
+    },
+    {
+      "entropy": 5.963817882537842,
+      "epoch": 3.3966609096142775,
+      "grad_norm": 1.0523409843444824,
+      "learning_rate": 8.849540207395813e-07,
+      "loss": 5.8033,
+      "mean_token_accuracy": 0.25546928733587265,
+      "num_tokens": 5497011.0,
+      "step": 5900
+    },
+    {
+      "entropy": 6.027821063995361,
+      "epoch": 3.425446171560161,
+      "grad_norm": 1.2537726163864136,
+      "learning_rate": 8.751711993738994e-07,
+      "loss": 5.8644,
+      "mean_token_accuracy": 0.2480815091729164,
+      "num_tokens": 5542914.0,
+      "step": 5950
+    },
+    {
+      "entropy": 6.06340226650238,
+      "epoch": 3.454231433506045,
+      "grad_norm": 1.3840627670288086,
+      "learning_rate": 8.653883780082175e-07,
+      "loss": 5.9024,
+      "mean_token_accuracy": 0.24260428220033645,
+      "num_tokens": 5591031.0,
+      "step": 6000
+    },
+    {
+      "entropy": 6.016174025535584,
+      "epoch": 3.4830166954519286,
+      "grad_norm": 1.4330769777297974,
+      "learning_rate": 8.556055566425356e-07,
+      "loss": 5.8533,
+      "mean_token_accuracy": 0.2491714572906494,
+      "num_tokens": 5638071.0,
+      "step": 6050
+    },
+    {
+      "entropy": 5.934744844436645,
+      "epoch": 3.5118019573978123,
+      "grad_norm": 1.0845732688903809,
+      "learning_rate": 8.458227352768538e-07,
+      "loss": 5.7747,
+      "mean_token_accuracy": 0.2592592638731003,
+      "num_tokens": 5684251.0,
+      "step": 6100
+    },
+    {
+      "entropy": 6.080287184715271,
+      "epoch": 3.540587219343696,
+      "grad_norm": 1.0975452661514282,
+      "learning_rate": 8.360399139111719e-07,
+      "loss": 5.9162,
+      "mean_token_accuracy": 0.24146371990442275,
+      "num_tokens": 5732144.0,
+      "step": 6150
+    },
+    {
+      "entropy": 5.930004096031189,
+      "epoch": 3.5693724812895797,
+      "grad_norm": 0.6231066584587097,
+      "learning_rate": 8.2625709254549e-07,
+      "loss": 5.7693,
+      "mean_token_accuracy": 0.2606990364193916,
+      "num_tokens": 5777711.0,
+      "step": 6200
+    },
+    {
+      "entropy": 5.9475119972229,
+      "epoch": 3.5981577432354634,
+      "grad_norm": 1.2649016380310059,
+      "learning_rate": 8.164742711798082e-07,
+      "loss": 5.7891,
+      "mean_token_accuracy": 0.2565712609887123,
+      "num_tokens": 5823662.0,
+      "step": 6250
+    },
+    {
+      "entropy": 6.142667779922485,
+      "epoch": 3.626943005181347,
+      "grad_norm": 2.129287004470825,
+      "learning_rate": 8.066914498141264e-07,
+      "loss": 5.9787,
+      "mean_token_accuracy": 0.23326011776924133,
+      "num_tokens": 5872695.0,
+      "step": 6300
+    },
+    {
+      "entropy": 5.959765286445617,
+      "epoch": 3.655728267127231,
+      "grad_norm": 1.467274785041809,
+      "learning_rate": 7.969086284484446e-07,
+      "loss": 5.7999,
+      "mean_token_accuracy": 0.2563468313217163,
+      "num_tokens": 5919280.0,
+      "step": 6350
+    },
+    {
+      "entropy": 6.116938819885254,
+      "epoch": 3.6845135290731146,
+      "grad_norm": 2.5099565982818604,
+      "learning_rate": 7.871258070827627e-07,
+      "loss": 5.951,
+      "mean_token_accuracy": 0.23610050201416016,
+      "num_tokens": 5967702.0,
+      "step": 6400
+    },
+    {
+      "entropy": 5.916635317802429,
+      "epoch": 3.7132987910189983,
+      "grad_norm": 1.8498667478561401,
+      "learning_rate": 7.773429857170808e-07,
+      "loss": 5.7564,
+      "mean_token_accuracy": 0.26092158019542694,
+      "num_tokens": 6012476.0,
+      "step": 6450
+    },
+    {
+      "entropy": 6.0112196683883665,
+      "epoch": 3.742084052964882,
+      "grad_norm": 1.475481390953064,
+      "learning_rate": 7.675601643513989e-07,
+      "loss": 5.8518,
+      "mean_token_accuracy": 0.2494723927974701,
+      "num_tokens": 6059915.0,
+      "step": 6500
+    },
+    {
+      "entropy": 6.143899393081665,
+      "epoch": 3.7708693149107657,
+      "grad_norm": 1.4096436500549316,
+      "learning_rate": 7.577773429857171e-07,
+      "loss": 5.9778,
+      "mean_token_accuracy": 0.23250365376472473,
+      "num_tokens": 6109703.0,
+      "step": 6550
+    },
+    {
+      "entropy": 5.922900657653809,
+      "epoch": 3.7996545768566494,
+      "grad_norm": 1.2578452825546265,
+      "learning_rate": 7.479945216200352e-07,
+      "loss": 5.7599,
+      "mean_token_accuracy": 0.26120502591133116,
+      "num_tokens": 6155107.0,
+      "step": 6600
+    },
+    {
+      "entropy": 5.908917541503906,
+      "epoch": 3.828439838802533,
+      "grad_norm": 2.6332685947418213,
+      "learning_rate": 7.382117002543533e-07,
+      "loss": 5.7503,
+      "mean_token_accuracy": 0.262827065885067,
+      "num_tokens": 6201565.0,
+      "step": 6650
+    },
+    {
+      "entropy": 6.0733087491989135,
+      "epoch": 3.857225100748417,
+      "grad_norm": 1.0848442316055298,
+      "learning_rate": 7.284288788886714e-07,
+      "loss": 5.9098,
+      "mean_token_accuracy": 0.24171414226293564,
+      "num_tokens": 6249569.0,
+      "step": 6700
+    },
+    {
+      "entropy": 6.002431573867798,
+      "epoch": 3.8860103626943006,
+      "grad_norm": 1.2640091180801392,
+      "learning_rate": 7.186460575229896e-07,
+      "loss": 5.84,
+      "mean_token_accuracy": 0.25067923456430435,
+      "num_tokens": 6296537.0,
+      "step": 6750
+    },
+    {
+      "entropy": 5.919499011039734,
+      "epoch": 3.9147956246401843,
+      "grad_norm": 1.4981272220611572,
+      "learning_rate": 7.088632361573077e-07,
+      "loss": 5.7591,
+      "mean_token_accuracy": 0.259833604991436,
+      "num_tokens": 6341373.0,
+      "step": 6800
+    },
+    {
+      "entropy": 6.17221610546112,
+      "epoch": 3.943580886586068,
+      "grad_norm": 1.38907790184021,
+      "learning_rate": 6.990804147916259e-07,
+      "loss": 6.0084,
+      "mean_token_accuracy": 0.22883434295654298,
+      "num_tokens": 6391079.0,
+      "step": 6850
+    },
+    {
+      "entropy": 5.79143741607666,
+      "epoch": 3.9723661485319517,
+      "grad_norm": 6.057252883911133,
+      "learning_rate": 6.892975934259439e-07,
+      "loss": 5.6354,
+      "mean_token_accuracy": 0.27610290706157686,
+      "num_tokens": 6434857.0,
+      "step": 6900
+    },
+    {
+      "epoch": 4.0,
+      "eval_entropy": 6.279375676185854,
+      "eval_loss": 6.126572132110596,
+      "eval_mean_token_accuracy": 0.21069503338655568,
+      "eval_model_preparation_time": 0.0046,
+      "eval_num_tokens": 6478876.0,
+      "eval_runtime": 79.2163,
+      "eval_samples_per_second": 5.479,
+      "eval_steps_per_second": 2.739,
+      "step": 6948
+    },
+    {
+      "entropy": 5.995638113021851,
+      "epoch": 4.001151410477835,
+      "grad_norm": 1.2027766704559326,
+      "learning_rate": 6.795147720602622e-07,
+      "loss": 5.8326,
+      "mean_token_accuracy": 0.2509008884429932,
+      "num_tokens": 6481003.0,
+      "step": 6950
+    },
+    {
+      "entropy": 5.7975999546051025,
+      "epoch": 4.029936672423719,
+      "grad_norm": 0.9509938359260559,
+      "learning_rate": 6.697319506945803e-07,
+      "loss": 5.64,
+      "mean_token_accuracy": 0.27601367354393,
+      "num_tokens": 6524470.0,
+      "step": 7000
+    },
+    {
+      "entropy": 6.142339401245117,
+      "epoch": 4.058721934369602,
+      "grad_norm": 1.7856882810592651,
+      "learning_rate": 6.599491293288985e-07,
+      "loss": 5.9798,
+      "mean_token_accuracy": 0.23329689502716064,
+      "num_tokens": 6572979.0,
+      "step": 7050
+    },
+    {
+      "entropy": 5.908348722457886,
+      "epoch": 4.087507196315486,
+      "grad_norm": 2.712480306625366,
+      "learning_rate": 6.501663079632165e-07,
+      "loss": 5.7511,
+      "mean_token_accuracy": 0.26176382452249525,
+      "num_tokens": 6617673.0,
+      "step": 7100
+    },
+    {
+      "entropy": 6.018770694732666,
+      "epoch": 4.11629245826137,
+      "grad_norm": 0.8292718529701233,
+      "learning_rate": 6.403834865975347e-07,
+      "loss": 5.8573,
+      "mean_token_accuracy": 0.248506840467453,
+      "num_tokens": 6664415.0,
+      "step": 7150
+    },
+    {
+      "entropy": 5.904297027587891,
+      "epoch": 4.1450777202072535,
+      "grad_norm": 0.6378379464149475,
+      "learning_rate": 6.306006652318528e-07,
+      "loss": 5.7486,
+      "mean_token_accuracy": 0.26151282787323,
+      "num_tokens": 6709826.0,
+      "step": 7200
+    },
+    {
+      "entropy": 5.967884268760681,
+      "epoch": 4.173862982153137,
+      "grad_norm": 1.2303566932678223,
+      "learning_rate": 6.20817843866171e-07,
+      "loss": 5.8065,
+      "mean_token_accuracy": 0.2549690026044846,
+      "num_tokens": 6756346.0,
+      "step": 7250
+    },
+    {
+      "entropy": 5.8746095514297485,
+      "epoch": 4.202648244099021,
+      "grad_norm": 1.3768641948699951,
+      "learning_rate": 6.110350225004892e-07,
+      "loss": 5.7161,
+      "mean_token_accuracy": 0.2668989074230194,
+      "num_tokens": 6801531.0,
+      "step": 7300
+    },
+    {
+      "entropy": 6.134030771255493,
+      "epoch": 4.231433506044905,
+      "grad_norm": 0.7624185085296631,
+      "learning_rate": 6.012522011348072e-07,
+      "loss": 5.9734,
+      "mean_token_accuracy": 0.2334815075993538,
+      "num_tokens": 6850251.0,
+      "step": 7350
+    },
+    {
+      "entropy": 6.01287202835083,
+      "epoch": 4.260218767990788,
+      "grad_norm": 1.4895133972167969,
+      "learning_rate": 5.914693797691254e-07,
+      "loss": 5.8533,
+      "mean_token_accuracy": 0.24807787895202638,
+      "num_tokens": 6896816.0,
+      "step": 7400
+    },
+    {
+      "entropy": 6.02873848438263,
+      "epoch": 4.289004029936672,
+      "grad_norm": 1.5440418720245361,
+      "learning_rate": 5.816865584034435e-07,
+      "loss": 5.8669,
+      "mean_token_accuracy": 0.2474558174610138,
+      "num_tokens": 6943702.0,
+      "step": 7450
+    },
+    {
+      "entropy": 6.010694708824158,
+      "epoch": 4.317789291882556,
+      "grad_norm": 2.088428258895874,
+      "learning_rate": 5.719037370377617e-07,
+      "loss": 5.8494,
+      "mean_token_accuracy": 0.25013200104236605,
+      "num_tokens": 6990895.0,
+      "step": 7500
+    },
+    {
+      "entropy": 6.087985677719116,
+      "epoch": 4.3465745538284395,
+      "grad_norm": 1.199644684791565,
+      "learning_rate": 5.621209156720797e-07,
+      "loss": 5.9273,
+      "mean_token_accuracy": 0.23846659421920777,
+      "num_tokens": 7039497.0,
+      "step": 7550
+    },
+    {
+      "entropy": 5.9547646045684814,
+      "epoch": 4.375359815774323,
+      "grad_norm": 1.8854912519454956,
+      "learning_rate": 5.52338094306398e-07,
+      "loss": 5.7951,
+      "mean_token_accuracy": 0.25700003176927566,
+      "num_tokens": 7084988.0,
+      "step": 7600
+    },
+    {
+      "entropy": 6.008062582015992,
+      "epoch": 4.404145077720207,
+      "grad_norm": 1.376185655593872,
+      "learning_rate": 5.425552729407161e-07,
+      "loss": 5.8469,
+      "mean_token_accuracy": 0.24962662607431413,
+      "num_tokens": 7131690.0,
+      "step": 7650
+    },
+    {
+      "entropy": 6.083430061340332,
+      "epoch": 4.432930339666091,
+      "grad_norm": 1.5763053894042969,
+      "learning_rate": 5.327724515750343e-07,
+      "loss": 5.9229,
+      "mean_token_accuracy": 0.24034427106380463,
+      "num_tokens": 7179874.0,
+      "step": 7700
+    },
+    {
+      "entropy": 5.94250883102417,
+      "epoch": 4.461715601611974,
+      "grad_norm": 1.1155059337615967,
+      "learning_rate": 5.229896302093524e-07,
+      "loss": 5.7841,
+      "mean_token_accuracy": 0.25782077729701997,
+      "num_tokens": 7225342.0,
+      "step": 7750
+    },
+    {
+      "entropy": 5.856548733711243,
+      "epoch": 4.490500863557858,
+      "grad_norm": 1.1634149551391602,
+      "learning_rate": 5.132068088436705e-07,
+      "loss": 5.6972,
+      "mean_token_accuracy": 0.26918380439281464,
+      "num_tokens": 7270136.0,
+      "step": 7800
+    },
+    {
+      "entropy": 5.866941246986389,
+      "epoch": 4.519286125503742,
+      "grad_norm": 1.6508464813232422,
+      "learning_rate": 5.034239874779886e-07,
+      "loss": 5.712,
+      "mean_token_accuracy": 0.26652718901634215,
+      "num_tokens": 7315462.0,
+      "step": 7850
+    },
+    {
+      "entropy": 6.119233846664429,
+      "epoch": 4.5480713874496255,
+      "grad_norm": 1.0165655612945557,
+      "learning_rate": 4.936411661123068e-07,
+      "loss": 5.9555,
+      "mean_token_accuracy": 0.2348495191335678,
+      "num_tokens": 7364611.0,
+      "step": 7900
+    },
+    {
+      "entropy": 6.090625686645508,
+      "epoch": 4.576856649395509,
+      "grad_norm": 0.7952129244804382,
+      "learning_rate": 4.838583447466249e-07,
+      "loss": 5.9278,
+      "mean_token_accuracy": 0.23983514040708542,
+      "num_tokens": 7412920.0,
+      "step": 7950
+    },
+    {
+      "entropy": 5.94470666885376,
+      "epoch": 4.605641911341393,
+      "grad_norm": 2.286240577697754,
+      "learning_rate": 4.7407552338094304e-07,
+      "loss": 5.7865,
+      "mean_token_accuracy": 0.25735649168491365,
+      "num_tokens": 7459505.0,
+      "step": 8000
+    },
+    {
+      "entropy": 6.177862458229065,
+      "epoch": 4.634427173287277,
+      "grad_norm": 2.1775429248809814,
+      "learning_rate": 4.6429270201526114e-07,
+      "loss": 6.012,
+      "mean_token_accuracy": 0.22966912269592285,
+      "num_tokens": 7508834.0,
+      "step": 8050
+    },
+    {
+      "entropy": 5.994407043457032,
+      "epoch": 4.66321243523316,
+      "grad_norm": 0.8207571506500244,
+      "learning_rate": 4.545098806495793e-07,
+      "loss": 5.8345,
+      "mean_token_accuracy": 0.25109571874141695,
+      "num_tokens": 7555981.0,
+      "step": 8100
+    },
+    {
+      "entropy": 5.867677879333496,
+      "epoch": 4.691997697179044,
+      "grad_norm": 2.5868327617645264,
+      "learning_rate": 4.4472705928389745e-07,
+      "loss": 5.7072,
+      "mean_token_accuracy": 0.2670168370008469,
+      "num_tokens": 7600529.0,
+      "step": 8150
+    },
+    {
+      "entropy": 5.971000475883484,
+      "epoch": 4.720782959124928,
+      "grad_norm": 1.0981251001358032,
+      "learning_rate": 4.349442379182156e-07,
+      "loss": 5.8118,
+      "mean_token_accuracy": 0.25474150747060775,
+      "num_tokens": 7646065.0,
+      "step": 8200
+    },
+    {
+      "entropy": 5.975438833236694,
+      "epoch": 4.7495682210708114,
+      "grad_norm": 1.0710279941558838,
+      "learning_rate": 4.2516141655253376e-07,
+      "loss": 5.8154,
+      "mean_token_accuracy": 0.2538798648118973,
+      "num_tokens": 7692863.0,
+      "step": 8250
+    },
+    {
+      "entropy": 6.000182151794434,
+      "epoch": 4.778353483016695,
+      "grad_norm": 0.7617666125297546,
+      "learning_rate": 4.1537859518685186e-07,
+      "loss": 5.8402,
+      "mean_token_accuracy": 0.2507925814390182,
+      "num_tokens": 7738799.0,
+      "step": 8300
+    },
+    {
+      "entropy": 5.834854488372803,
+      "epoch": 4.807138744962579,
+      "grad_norm": 2.2277169227600098,
+      "learning_rate": 4.0559577382117e-07,
+      "loss": 5.6785,
+      "mean_token_accuracy": 0.2707058879733086,
+      "num_tokens": 7782955.0,
+      "step": 8350
+    },
+    {
+      "entropy": 6.1768684530258176,
+      "epoch": 4.835924006908463,
+      "grad_norm": 0.7781999707221985,
+      "learning_rate": 3.958129524554881e-07,
+      "loss": 6.0105,
+      "mean_token_accuracy": 0.22812994629144667,
+      "num_tokens": 7832647.0,
+      "step": 8400
+    },
+    {
+      "entropy": 5.988465652465821,
+      "epoch": 4.864709268854346,
+      "grad_norm": 0.4961145222187042,
+      "learning_rate": 3.860301310898063e-07,
+      "loss": 5.8263,
+      "mean_token_accuracy": 0.2520375117659569,
+      "num_tokens": 7879636.0,
+      "step": 8450
+    },
+    {
+      "entropy": 6.066406717300415,
+      "epoch": 4.89349453080023,
+      "grad_norm": 1.1657921075820923,
+      "learning_rate": 3.762473097241244e-07,
+      "loss": 5.9054,
+      "mean_token_accuracy": 0.2423809215426445,
+      "num_tokens": 7927041.0,
+      "step": 8500
+    },
+    {
+      "entropy": 6.052256097793579,
+      "epoch": 4.922279792746114,
+      "grad_norm": 1.5335379838943481,
+      "learning_rate": 3.664644883584426e-07,
+      "loss": 5.8925,
+      "mean_token_accuracy": 0.24372650146484376,
+      "num_tokens": 7974219.0,
+      "step": 8550
+    },
+    {
+      "entropy": 5.889537000656128,
+      "epoch": 4.951065054691997,
+      "grad_norm": 1.3163872957229614,
+      "learning_rate": 3.566816669927607e-07,
+      "loss": 5.7296,
+      "mean_token_accuracy": 0.26416114032268523,
+      "num_tokens": 8018646.0,
+      "step": 8600
+    },
+    {
+      "entropy": 6.079032945632934,
+      "epoch": 4.979850316637881,
+      "grad_norm": 2.2353949546813965,
+      "learning_rate": 3.4689884562707883e-07,
+      "loss": 5.9181,
+      "mean_token_accuracy": 0.2414929136633873,
+      "num_tokens": 8066512.0,
+      "step": 8650
+    },
+    {
+      "epoch": 5.0,
+      "eval_entropy": 6.274539154246106,
+      "eval_loss": 6.1247029304504395,
+      "eval_mean_token_accuracy": 0.21092273377328424,
+      "eval_model_preparation_time": 0.0046,
+      "eval_num_tokens": 8098595.0,
+      "eval_runtime": 79.4734,
+      "eval_samples_per_second": 5.461,
+      "eval_steps_per_second": 2.73,
+      "step": 8685
+    },
+    {
+      "entropy": 5.957367534637451,
+      "epoch": 5.008635578583765,
+      "grad_norm": 0.680518388748169,
+      "learning_rate": 3.37116024261397e-07,
+      "loss": 5.7978,
+      "mean_token_accuracy": 0.2565177664160728,
+      "num_tokens": 8111610.0,
+      "step": 8700
+    },
+    {
+      "entropy": 5.974089093208313,
+      "epoch": 5.037420840529649,
+      "grad_norm": 2.036747932434082,
+      "learning_rate": 3.273332028957151e-07,
+      "loss": 5.8159,
+      "mean_token_accuracy": 0.2531487289071083,
+      "num_tokens": 8157841.0,
+      "step": 8750
+    },
+    {
+      "entropy": 5.812384562492371,
+      "epoch": 5.066206102475532,
+      "grad_norm": 1.5146092176437378,
+      "learning_rate": 3.175503815300333e-07,
+      "loss": 5.6566,
+      "mean_token_accuracy": 0.27390810728073123,
+      "num_tokens": 8201355.0,
+      "step": 8800
+    },
+    {
+      "entropy": 5.971576690673828,
+      "epoch": 5.094991364421416,
+      "grad_norm": 0.9713916778564453,
+      "learning_rate": 3.077675601643514e-07,
+      "loss": 5.8136,
+      "mean_token_accuracy": 0.25402570873498914,
+      "num_tokens": 8248348.0,
+      "step": 8850
+    },
+    {
+      "entropy": 5.940347299575806,
+      "epoch": 5.1237766263673,
+      "grad_norm": 1.1493933200836182,
+      "learning_rate": 2.9798473879866954e-07,
+      "loss": 5.7845,
+      "mean_token_accuracy": 0.2574089586734772,
+      "num_tokens": 8294009.0,
+      "step": 8900
+    },
+    {
+      "entropy": 5.9449573802948,
+      "epoch": 5.152561888313183,
+      "grad_norm": 1.647032618522644,
+      "learning_rate": 2.8820191743298764e-07,
+      "loss": 5.7871,
+      "mean_token_accuracy": 0.25562890857458115,
+      "num_tokens": 8340510.0,
+      "step": 8950
+    },
+    {
+      "entropy": 6.133461399078369,
+      "epoch": 5.181347150259067,
+      "grad_norm": 0.8065502047538757,
+      "learning_rate": 2.784190960673058e-07,
+      "loss": 5.969,
+      "mean_token_accuracy": 0.23392125099897385,
+      "num_tokens": 8390033.0,
+      "step": 9000
+    },
+    {
+      "entropy": 6.074811162948609,
+      "epoch": 5.210132412204951,
+      "grad_norm": 0.833552360534668,
+      "learning_rate": 2.686362747016239e-07,
+      "loss": 5.9127,
+      "mean_token_accuracy": 0.241038781106472,
+      "num_tokens": 8436914.0,
+      "step": 9050
+    },
+    {
+      "entropy": 5.771420574188232,
+      "epoch": 5.2389176741508345,
+      "grad_norm": 2.6249544620513916,
+      "learning_rate": 2.5885345333594205e-07,
+      "loss": 5.6153,
+      "mean_token_accuracy": 0.27925612688064577,
+      "num_tokens": 8481570.0,
+      "step": 9100
+    },
+    {
+      "entropy": 5.9740171718597415,
+      "epoch": 5.267702936096718,
+      "grad_norm": 1.484552025794983,
+      "learning_rate": 2.490706319702602e-07,
+      "loss": 5.8145,
+      "mean_token_accuracy": 0.25337011635303497,
+      "num_tokens": 8527697.0,
+      "step": 9150
+    },
+    {
+      "entropy": 6.084105367660523,
+      "epoch": 5.296488198042602,
+      "grad_norm": 1.5230190753936768,
+      "learning_rate": 2.3928781060457836e-07,
+      "loss": 5.9269,
+      "mean_token_accuracy": 0.23908233702182768,
+      "num_tokens": 8575933.0,
+      "step": 9200
+    },
+    {
+      "entropy": 5.878108925819397,
+      "epoch": 5.325273459988486,
+      "grad_norm": 1.7913310527801514,
+      "learning_rate": 2.2950498923889649e-07,
+      "loss": 5.7208,
+      "mean_token_accuracy": 0.26579217702150343,
+      "num_tokens": 8620214.0,
+      "step": 9250
+    },
+    {
+      "entropy": 6.043487319946289,
+      "epoch": 5.354058721934369,
+      "grad_norm": 1.173954963684082,
+      "learning_rate": 2.1972216787321461e-07,
+      "loss": 5.8806,
+      "mean_token_accuracy": 0.24593402802944184,
+      "num_tokens": 8668532.0,
+      "step": 9300
+    },
+    {
+      "entropy": 5.817543797492981,
+      "epoch": 5.382843983880253,
+      "grad_norm": 1.084602952003479,
+      "learning_rate": 2.0993934650753277e-07,
+      "loss": 5.6593,
+      "mean_token_accuracy": 0.2739328667521477,
+      "num_tokens": 8712689.0,
+      "step": 9350
+    },
+    {
+      "entropy": 6.036909718513488,
+      "epoch": 5.411629245826137,
+      "grad_norm": 1.4778636693954468,
+      "learning_rate": 2.0015652514185092e-07,
+      "loss": 5.8748,
+      "mean_token_accuracy": 0.24605893224477768,
+      "num_tokens": 8760376.0,
+      "step": 9400
+    },
+    {
+      "entropy": 6.050856218338013,
+      "epoch": 5.4404145077720205,
+      "grad_norm": 0.9816691875457764,
+      "learning_rate": 1.9037370377616905e-07,
+      "loss": 5.8885,
+      "mean_token_accuracy": 0.2441025686264038,
+      "num_tokens": 8808966.0,
+      "step": 9450
+    },
+    {
+      "entropy": 6.072160882949829,
+      "epoch": 5.469199769717904,
+      "grad_norm": 1.9699606895446777,
+      "learning_rate": 1.8059088241048718e-07,
+      "loss": 5.9096,
+      "mean_token_accuracy": 0.24258128613233565,
+      "num_tokens": 8856411.0,
+      "step": 9500
+    },
+    {
+      "entropy": 6.086222591400147,
+      "epoch": 5.497985031663788,
+      "grad_norm": 0.9021607637405396,
+      "learning_rate": 1.708080610448053e-07,
+      "loss": 5.9233,
+      "mean_token_accuracy": 0.24022864073514938,
+      "num_tokens": 8904412.0,
+      "step": 9550
+    },
+    {
+      "entropy": 5.965502681732178,
+      "epoch": 5.526770293609672,
+      "grad_norm": 1.0994197130203247,
+      "learning_rate": 1.6102523967912346e-07,
+      "loss": 5.8112,
+      "mean_token_accuracy": 0.2534236097335815,
+      "num_tokens": 8950118.0,
+      "step": 9600
+    },
+    {
+      "entropy": 5.795955166816712,
+      "epoch": 5.555555555555555,
+      "grad_norm": 1.9056462049484253,
+      "learning_rate": 1.5124241831344158e-07,
+      "loss": 5.6409,
+      "mean_token_accuracy": 0.2760310146212578,
+      "num_tokens": 8993094.0,
+      "step": 9650
+    },
+    {
+      "entropy": 5.992545394897461,
+      "epoch": 5.584340817501439,
+      "grad_norm": 1.1243247985839844,
+      "learning_rate": 1.414595969477597e-07,
+      "loss": 5.8337,
+      "mean_token_accuracy": 0.25096620470285413,
+      "num_tokens": 9039172.0,
+      "step": 9700
+    },
+    {
+      "entropy": 6.062644476890564,
+      "epoch": 5.613126079447323,
+      "grad_norm": 0.7900448441505432,
+      "learning_rate": 1.3167677558207786e-07,
+      "loss": 5.9029,
+      "mean_token_accuracy": 0.2425614431500435,
+      "num_tokens": 9087856.0,
+      "step": 9750
+    },
+    {
+      "entropy": 6.036457490921021,
+      "epoch": 5.6419113413932065,
+      "grad_norm": 17.477712631225586,
+      "learning_rate": 1.2189395421639602e-07,
+      "loss": 5.8757,
+      "mean_token_accuracy": 0.24608026653528214,
+      "num_tokens": 9134247.0,
+      "step": 9800
+    },
+    {
+      "entropy": 5.996092481613159,
+      "epoch": 5.67069660333909,
+      "grad_norm": 1.4077341556549072,
+      "learning_rate": 1.1211113285071413e-07,
+      "loss": 5.8395,
+      "mean_token_accuracy": 0.250471707880497,
+      "num_tokens": 9181486.0,
+      "step": 9850
+    },
+    {
+      "entropy": 6.087848567962647,
+      "epoch": 5.699481865284974,
+      "grad_norm": 1.3837510347366333,
+      "learning_rate": 1.0232831148503227e-07,
+      "loss": 5.9276,
+      "mean_token_accuracy": 0.24019749820232392,
+      "num_tokens": 9229355.0,
+      "step": 9900
+    },
+    {
+      "entropy": 6.053327779769898,
+      "epoch": 5.728267127230858,
+      "grad_norm": 1.0415576696395874,
+      "learning_rate": 9.254549011935043e-08,
+      "loss": 5.8926,
+      "mean_token_accuracy": 0.24374084115028383,
+      "num_tokens": 9276282.0,
+      "step": 9950
+    },
+    {
+      "entropy": 5.945247740745544,
+      "epoch": 5.757052389176741,
+      "grad_norm": 1.4692878723144531,
+      "learning_rate": 8.276266875366855e-08,
+      "loss": 5.79,
+      "mean_token_accuracy": 0.2562396174669266,
+      "num_tokens": 9322173.0,
+      "step": 10000
+    },
+    {
+      "entropy": 6.043243775367737,
+      "epoch": 5.785837651122625,
+      "grad_norm": 1.2986756563186646,
+      "learning_rate": 7.29798473879867e-08,
+      "loss": 5.8821,
+      "mean_token_accuracy": 0.24506330251693725,
+      "num_tokens": 9369917.0,
+      "step": 10050
+    },
+    {
+      "entropy": 6.070325479507447,
+      "epoch": 5.814622913068509,
+      "grad_norm": 2.0517385005950928,
+      "learning_rate": 6.319702602230482e-08,
+      "loss": 5.9093,
+      "mean_token_accuracy": 0.24180745720863342,
+      "num_tokens": 9417145.0,
+      "step": 10100
+    },
+    {
+      "entropy": 5.813769164085389,
+      "epoch": 5.8434081750143925,
+      "grad_norm": 1.4036266803741455,
+      "learning_rate": 5.341420465662297e-08,
+      "loss": 5.6553,
+      "mean_token_accuracy": 0.27383983492851255,
+      "num_tokens": 9462113.0,
+      "step": 10150
+    },
+    {
+      "entropy": 5.958855032920837,
+      "epoch": 5.872193436960276,
+      "grad_norm": 2.063570499420166,
+      "learning_rate": 4.363138329094111e-08,
+      "loss": 5.8008,
+      "mean_token_accuracy": 0.2549652716517448,
+      "num_tokens": 9507567.0,
+      "step": 10200
+    },
+    {
+      "entropy": 5.989646158218384,
+      "epoch": 5.90097869890616,
+      "grad_norm": 0.590006411075592,
+      "learning_rate": 3.384856192525924e-08,
+      "loss": 5.8287,
+      "mean_token_accuracy": 0.2527298724651337,
+      "num_tokens": 9554123.0,
+      "step": 10250
+    },
+    {
+      "entropy": 6.049757356643677,
+      "epoch": 5.929763960852044,
+      "grad_norm": 1.297501802444458,
+      "learning_rate": 2.4065740559577383e-08,
+      "loss": 5.8895,
+      "mean_token_accuracy": 0.24449419289827345,
+      "num_tokens": 9600615.0,
+      "step": 10300
+    },
+    {
+      "entropy": 6.052498106956482,
+      "epoch": 5.958549222797927,
+      "grad_norm": 1.1880056858062744,
+      "learning_rate": 1.4282919193895518e-08,
+      "loss": 5.8922,
+      "mean_token_accuracy": 0.24352416545152664,
+      "num_tokens": 9648757.0,
+      "step": 10350
+    },
+    {
+      "entropy": 6.125533571243286,
+      "epoch": 5.987334484743811,
+      "grad_norm": 1.0961848497390747,
+      "learning_rate": 4.500097828213657e-09,
+      "loss": 5.961,
+      "mean_token_accuracy": 0.23481910437345505,
+      "num_tokens": 9696666.0,
+      "step": 10400
+    },
+    {
+      "epoch": 6.0,
+      "eval_entropy": 6.273049378724692,
+      "eval_loss": 6.123514175415039,
+      "eval_mean_token_accuracy": 0.21104606045281282,
+      "eval_model_preparation_time": 0.0046,
+      "eval_num_tokens": 9718314.0,
+      "eval_runtime": 79.5368,
+      "eval_samples_per_second": 5.457,
+      "eval_steps_per_second": 2.728,
+      "step": 10422
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 10422,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 6,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.3590670746786202e+17,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-10422/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1adcf89e2ce7be508f245e87af0fd5d93d0d2e7562c07049b800c8d5a6b1822e
+size 6225

checkpoint-1737/adapter_config.json

@@ -16,7 +16,7 @@
   "layers_pattern": null,
   "layers_to_transform": null,
   "loftq_config": {},
-  "lora_alpha": 16,
+  "lora_alpha": 32,
   "lora_bias": false,
   "lora_dropout": 0.1,
   "megatron_config": null,
@@ -25,12 +25,14 @@
   "peft_type": "LORA",
   "peft_version": "0.18.0",
   "qalora_group_size": 16,
-  "r": 8,
+  "r": 24,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
     "q_proj",
-    "v_proj"
+    "k_proj",
+    "v_proj",
+    "o_proj"
   ],
   "target_parameters": null,
   "task_type": "CAUSAL_LM",

checkpoint-1737/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b5f4b9708eccf0370f9aaa1466d17c487ab3a9e4e84732d5cd39bbd229aedd5c
-size 4374520
+oid sha256:ca36c29cabd2e8ea449e6eadcd7f7db9042e00cae52ef5b042c56b58c200775a
+size 26182176

checkpoint-1737/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:84ee821de3d805218a80046b08a325803a2434e306b554e094f68548e53fbe41
-size 8783179
+oid sha256:8fab12e7cc07b84cfe33ab9be36e25b4dfa882f0ac9e6725dfb7608859ec3a87
+size 52486155

checkpoint-1737/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4e816ab59bde4778d4f30814a9146abbd7044e1640b72b0be4234c4aa55b98f1
+oid sha256:ac7868bb5d10a59d1042ca17d4fc89dc5beddcdf6df99c035480579667b84b19
 size 14645

checkpoint-1737/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f9121f4d6a6f445ab467d2762de7c0b86cf7fef9179d9273d56797386ca47712
+oid sha256:2c17d5ce4845692098064761cc4c713c4686c6a262dcb4177eea65f272ed234c
 size 1465

checkpoint-1737/trainer_state.json

@@ -1,6 +1,6 @@
 {
   "best_global_step": 1737,
-  "best_metric": 6.15173864364624,
+  "best_metric": 5.861395835876465,
   "best_model_checkpoint": "./output/checkpoint-1737",
   "epoch": 1.0,
   "eval_steps": 500,
@@ -10,362 +10,362 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "entropy": 3.864118957519531,
+      "entropy": 3.6583470726013183,
       "epoch": 0.028785261945883708,
-      "grad_norm": 2.7545533180236816,
-      "learning_rate": 9.800000000000001e-06,
-      "loss": 15.2997,
-      "mean_token_accuracy": 0.10086015284061432,
-      "num_tokens": 47319.0,
+      "grad_norm": 3.3817152976989746,
+      "learning_rate": 4.9e-07,
+      "loss": 13.8754,
+      "mean_token_accuracy": 0.15036460414528846,
+      "num_tokens": 53093.0,
       "step": 50
     },
     {
-      "entropy": 4.047076859474182,
+      "entropy": 3.669608063697815,
       "epoch": 0.057570523891767415,
-      "grad_norm": 5.0328264236450195,
-      "learning_rate": 1.98e-05,
-      "loss": 15.3264,
-      "mean_token_accuracy": 0.09582207053899765,
-      "num_tokens": 96809.0,
+      "grad_norm": 3.2541544437408447,
+      "learning_rate": 9.9e-07,
+      "loss": 14.2282,
+      "mean_token_accuracy": 0.14137721598148345,
+      "num_tokens": 108334.0,
       "step": 100
     },
     {
-      "entropy": 4.7578076648712155,
+      "entropy": 3.569736371040344,
       "epoch": 0.08635578583765112,
-      "grad_norm": 38.50589370727539,
-      "learning_rate": 1.988584740827024e-05,
-      "loss": 13.0056,
-      "mean_token_accuracy": 0.126854517608881,
-      "num_tokens": 139962.0,
+      "grad_norm": 3.6797454357147217,
+      "learning_rate": 1.49e-06,
+      "loss": 13.0735,
+      "mean_token_accuracy": 0.17473630651831626,
+      "num_tokens": 157491.0,
       "step": 150
     },
     {
-      "entropy": 6.80673882484436,
+      "entropy": 3.7253233194351196,
       "epoch": 0.11514104778353483,
-      "grad_norm": 12.030129432678223,
-      "learning_rate": 1.97693651718113e-05,
-      "loss": 9.2822,
-      "mean_token_accuracy": 0.11084575355052947,
-      "num_tokens": 188029.0,
+      "grad_norm": 4.297911643981934,
+      "learning_rate": 1.99e-06,
+      "loss": 13.7392,
+      "mean_token_accuracy": 0.1473099772632122,
+      "num_tokens": 211394.0,
       "step": 200
     },
     {
-      "entropy": 7.177925786972046,
+      "entropy": 3.8280500602722167,
       "epoch": 0.14392630972941853,
-      "grad_norm": 4.852536201477051,
-      "learning_rate": 1.965288293535236e-05,
-      "loss": 7.6333,
-      "mean_token_accuracy": 0.12398939326405525,
-      "num_tokens": 234425.0,
+      "grad_norm": 4.405268669128418,
+      "learning_rate": 1.9854771784232364e-06,
+      "loss": 13.0797,
+      "mean_token_accuracy": 0.16704789966344832,
+      "num_tokens": 263685.0,
       "step": 250
     },
     {
-      "entropy": 7.080496473312378,
+      "entropy": 4.066333084106446,
       "epoch": 0.17271157167530224,
-      "grad_norm": 4.10841178894043,
-      "learning_rate": 1.9536400698893422e-05,
-      "loss": 7.1632,
-      "mean_token_accuracy": 0.13563686355948448,
-      "num_tokens": 278885.0,
+      "grad_norm": 4.757556438446045,
+      "learning_rate": 1.9706579727326615e-06,
+      "loss": 12.6321,
+      "mean_token_accuracy": 0.1691790708899498,
+      "num_tokens": 314059.0,
       "step": 300
     },
     {
-      "entropy": 6.931579580307007,
+      "entropy": 4.257266030311585,
       "epoch": 0.20149683362118595,
-      "grad_norm": 14.636048316955566,
-      "learning_rate": 1.941991846243448e-05,
-      "loss": 6.8213,
-      "mean_token_accuracy": 0.16459846690297128,
-      "num_tokens": 325491.0,
+      "grad_norm": 6.406249523162842,
+      "learning_rate": 1.955838767042086e-06,
+      "loss": 12.2253,
+      "mean_token_accuracy": 0.17223650276660918,
+      "num_tokens": 367038.0,
       "step": 350
     },
     {
-      "entropy": 6.853660764694214,
+      "entropy": 4.694105777740479,
       "epoch": 0.23028209556706966,
-      "grad_norm": 5.966708183288574,
-      "learning_rate": 1.930343622597554e-05,
-      "loss": 6.6625,
-      "mean_token_accuracy": 0.17670693069696428,
-      "num_tokens": 372913.0,
+      "grad_norm": 12.57987117767334,
+      "learning_rate": 1.9410195613515113e-06,
+      "loss": 11.9714,
+      "mean_token_accuracy": 0.15997304677963256,
+      "num_tokens": 420327.0,
       "step": 400
     },
     {
-      "entropy": 6.684267387390137,
+      "entropy": 5.205010280609131,
       "epoch": 0.25906735751295334,
-      "grad_norm": 4.031010627746582,
-      "learning_rate": 1.91869539895166e-05,
-      "loss": 6.4505,
-      "mean_token_accuracy": 0.1943434515595436,
-      "num_tokens": 419159.0,
+      "grad_norm": 15.570313453674316,
+      "learning_rate": 1.9262003556609364e-06,
+      "loss": 10.8173,
+      "mean_token_accuracy": 0.16447648257017136,
+      "num_tokens": 472429.0,
       "step": 450
     },
     {
-      "entropy": 6.679989137649536,
+      "entropy": 5.917805089950561,
       "epoch": 0.28785261945883706,
-      "grad_norm": 6.251070022583008,
-      "learning_rate": 1.907047175305766e-05,
-      "loss": 6.4314,
-      "mean_token_accuracy": 0.19514557600021362,
-      "num_tokens": 466994.0,
+      "grad_norm": 23.61503791809082,
+      "learning_rate": 1.9113811499703615e-06,
+      "loss": 9.3196,
+      "mean_token_accuracy": 0.16179455041885377,
+      "num_tokens": 526315.0,
       "step": 500
     },
     {
-      "entropy": 6.477229623794556,
+      "entropy": 6.380368332862854,
       "epoch": 0.31663788140472077,
-      "grad_norm": 3.8656675815582275,
-      "learning_rate": 1.895398951659872e-05,
-      "loss": 6.2139,
-      "mean_token_accuracy": 0.21764743447303772,
-      "num_tokens": 513308.0,
+      "grad_norm": 13.846810340881348,
+      "learning_rate": 1.8965619442797864e-06,
+      "loss": 7.9636,
+      "mean_token_accuracy": 0.16881170988082886,
+      "num_tokens": 578511.0,
       "step": 550
     },
     {
-      "entropy": 6.408129243850708,
+      "entropy": 6.507339992523193,
       "epoch": 0.3454231433506045,
-      "grad_norm": 8.688581466674805,
-      "learning_rate": 1.883750728013978e-05,
-      "loss": 6.1224,
-      "mean_token_accuracy": 0.23438037544488907,
-      "num_tokens": 559679.0,
+      "grad_norm": 4.569090366363525,
+      "learning_rate": 1.8817427385892115e-06,
+      "loss": 7.4171,
+      "mean_token_accuracy": 0.16941152423620223,
+      "num_tokens": 630937.0,
       "step": 600
     },
     {
-      "entropy": 6.128518767356873,
+      "entropy": 6.392864561080932,
       "epoch": 0.3742084052964882,
-      "grad_norm": 5.419503688812256,
-      "learning_rate": 1.872102504368084e-05,
-      "loss": 5.8692,
-      "mean_token_accuracy": 0.26634690463542937,
-      "num_tokens": 603140.0,
+      "grad_norm": 4.594696521759033,
+      "learning_rate": 1.8669235328986366e-06,
+      "loss": 6.9389,
+      "mean_token_accuracy": 0.1844496901333332,
+      "num_tokens": 680501.0,
       "step": 650
     },
     {
-      "entropy": 6.322700729370117,
+      "entropy": 6.6726202869415285,
       "epoch": 0.4029936672423719,
-      "grad_norm": 2.2213082313537598,
-      "learning_rate": 1.86045428072219e-05,
-      "loss": 6.0717,
-      "mean_token_accuracy": 0.24038562417030335,
-      "num_tokens": 650179.0,
+      "grad_norm": 4.768734931945801,
+      "learning_rate": 1.8521043272080617e-06,
+      "loss": 6.9818,
+      "mean_token_accuracy": 0.16990411713719367,
+      "num_tokens": 733231.0,
       "step": 700
     },
     {
-      "entropy": 6.236415157318115,
+      "entropy": 6.592793455123902,
       "epoch": 0.4317789291882556,
-      "grad_norm": 4.804980278015137,
-      "learning_rate": 1.848806057076296e-05,
-      "loss": 5.9986,
-      "mean_token_accuracy": 0.24596781462430953,
-      "num_tokens": 696220.0,
+      "grad_norm": 3.253056764602661,
+      "learning_rate": 1.8372851215174864e-06,
+      "loss": 6.7105,
+      "mean_token_accuracy": 0.18250102579593658,
+      "num_tokens": 785373.0,
       "step": 750
     },
     {
-      "entropy": 6.269758443832398,
+      "entropy": 6.683582029342651,
       "epoch": 0.4605641911341393,
-      "grad_norm": 2.2888853549957275,
-      "learning_rate": 1.837157833430402e-05,
-      "loss": 6.0385,
-      "mean_token_accuracy": 0.24074893474578857,
-      "num_tokens": 743909.0,
+      "grad_norm": 2.1871063709259033,
+      "learning_rate": 1.8224659158269115e-06,
+      "loss": 6.6685,
+      "mean_token_accuracy": 0.17129646152257919,
+      "num_tokens": 838646.0,
       "step": 800
     },
     {
-      "entropy": 6.270364007949829,
+      "entropy": 6.636875295639038,
       "epoch": 0.48934945308002303,
-      "grad_norm": 3.0903279781341553,
-      "learning_rate": 1.825509609784508e-05,
-      "loss": 6.0481,
-      "mean_token_accuracy": 0.23740622967481614,
-      "num_tokens": 792015.0,
+      "grad_norm": 3.2284677028656006,
+      "learning_rate": 1.8076467101363366e-06,
+      "loss": 6.53,
+      "mean_token_accuracy": 0.18053789794445038,
+      "num_tokens": 892380.0,
       "step": 850
     },
     {
-      "entropy": 6.3037636184692385,
+      "entropy": 6.610673260688782,
       "epoch": 0.5181347150259067,
-      "grad_norm": 3.969320058822632,
-      "learning_rate": 1.813861386138614e-05,
-      "loss": 6.0855,
-      "mean_token_accuracy": 0.2309597587585449,
-      "num_tokens": 841802.0,
+      "grad_norm": 2.2088730335235596,
+      "learning_rate": 1.7928275044457617e-06,
+      "loss": 6.4429,
+      "mean_token_accuracy": 0.18492739230394364,
+      "num_tokens": 947971.0,
       "step": 900
     },
     {
-      "entropy": 6.038041458129883,
+      "entropy": 6.242899022102356,
       "epoch": 0.5469199769717904,
-      "grad_norm": 2.2712185382843018,
-      "learning_rate": 1.80221316249272e-05,
-      "loss": 5.8285,
-      "mean_token_accuracy": 0.26099125802516937,
-      "num_tokens": 886492.0,
+      "grad_norm": 2.3000030517578125,
+      "learning_rate": 1.7780082987551866e-06,
+      "loss": 6.047,
+      "mean_token_accuracy": 0.2291259828209877,
+      "num_tokens": 998810.0,
       "step": 950
     },
     {
-      "entropy": 6.142958383560181,
+      "entropy": 6.311488924026489,
       "epoch": 0.5757052389176741,
-      "grad_norm": 1.2311755418777466,
-      "learning_rate": 1.790564938846826e-05,
-      "loss": 5.9357,
-      "mean_token_accuracy": 0.24810438305139543,
-      "num_tokens": 932807.0,
+      "grad_norm": 2.1333675384521484,
+      "learning_rate": 1.7631890930646115e-06,
+      "loss": 6.0919,
+      "mean_token_accuracy": 0.22644571751356124,
+      "num_tokens": 1050860.0,
       "step": 1000
     },
     {
-      "entropy": 6.199834351539612,
+      "entropy": 6.3254336166381835,
       "epoch": 0.6044905008635578,
-      "grad_norm": 2.2788379192352295,
-      "learning_rate": 1.7789167152009318e-05,
-      "loss": 5.9964,
-      "mean_token_accuracy": 0.23942562609910964,
-      "num_tokens": 980541.0,
+      "grad_norm": 2.0400779247283936,
+      "learning_rate": 1.7483698873740366e-06,
+      "loss": 6.094,
+      "mean_token_accuracy": 0.2222653564810753,
+      "num_tokens": 1104304.0,
       "step": 1050
     },
     {
-      "entropy": 5.961639919281006,
+      "entropy": 6.046922063827514,
       "epoch": 0.6332757628094415,
-      "grad_norm": 1.9077532291412354,
-      "learning_rate": 1.767268491555038e-05,
-      "loss": 5.7664,
-      "mean_token_accuracy": 0.26718012750148773,
-      "num_tokens": 1023882.0,
+      "grad_norm": 2.8049051761627197,
+      "learning_rate": 1.7335506816834617e-06,
+      "loss": 5.8011,
+      "mean_token_accuracy": 0.25127078920602797,
+      "num_tokens": 1153605.0,
       "step": 1100
     },
     {
-      "entropy": 5.889280087947846,
+      "entropy": 5.943600912094116,
       "epoch": 0.6620610247553252,
-      "grad_norm": 2.4254891872406006,
-      "learning_rate": 1.7556202679091442e-05,
-      "loss": 5.6952,
-      "mean_token_accuracy": 0.27529804170131683,
-      "num_tokens": 1068300.0,
+      "grad_norm": 4.063963890075684,
+      "learning_rate": 1.7187314759928866e-06,
+      "loss": 5.6855,
+      "mean_token_accuracy": 0.26265266716480257,
+      "num_tokens": 1204328.0,
       "step": 1150
     },
     {
-      "entropy": 6.085640063285828,
+      "entropy": 6.12883231639862,
       "epoch": 0.690846286701209,
-      "grad_norm": 2.35312557220459,
-      "learning_rate": 1.74397204426325e-05,
-      "loss": 5.8898,
-      "mean_token_accuracy": 0.25166562348604204,
-      "num_tokens": 1115425.0,
+      "grad_norm": 3.9440460205078125,
+      "learning_rate": 1.7039122703023117e-06,
+      "loss": 5.8578,
+      "mean_token_accuracy": 0.24439335912466048,
+      "num_tokens": 1257415.0,
       "step": 1200
     },
     {
-      "entropy": 6.146574058532715,
+      "entropy": 6.164987115859986,
       "epoch": 0.7196315486470927,
-      "grad_norm": 1.7730146646499634,
-      "learning_rate": 1.732323820617356e-05,
-      "loss": 5.9519,
-      "mean_token_accuracy": 0.24276195973157882,
-      "num_tokens": 1162319.0,
+      "grad_norm": 3.20070481300354,
+      "learning_rate": 1.6890930646117368e-06,
+      "loss": 5.8876,
+      "mean_token_accuracy": 0.24275501281023026,
+      "num_tokens": 1310049.0,
       "step": 1250
     },
     {
-      "entropy": 6.079372715950012,
+      "entropy": 6.080997190475464,
       "epoch": 0.7484168105929764,
-      "grad_norm": 1.7070863246917725,
-      "learning_rate": 1.720675596971462e-05,
-      "loss": 5.8922,
-      "mean_token_accuracy": 0.24961524546146394,
-      "num_tokens": 1208230.0,
+      "grad_norm": 2.8067362308502197,
+      "learning_rate": 1.6742738589211617e-06,
+      "loss": 5.8058,
+      "mean_token_accuracy": 0.25242207854986193,
+      "num_tokens": 1361794.0,
       "step": 1300
     },
     {
-      "entropy": 5.9683656406402585,
+      "entropy": 5.940848155021667,
       "epoch": 0.7772020725388601,
-      "grad_norm": 1.8790594339370728,
-      "learning_rate": 1.709027373325568e-05,
-      "loss": 5.7827,
-      "mean_token_accuracy": 0.2632122594118118,
-      "num_tokens": 1253074.0,
+      "grad_norm": 2.6375925540924072,
+      "learning_rate": 1.6594546532305868e-06,
+      "loss": 5.6718,
+      "mean_token_accuracy": 0.2665082859992981,
+      "num_tokens": 1412773.0,
       "step": 1350
     },
     {
-      "entropy": 6.107076721191406,
+      "entropy": 6.071129274368286,
       "epoch": 0.8059873344847438,
-      "grad_norm": 1.1745644807815552,
-      "learning_rate": 1.6973791496796742e-05,
-      "loss": 5.9211,
-      "mean_token_accuracy": 0.24564073830842972,
-      "num_tokens": 1300179.0,
+      "grad_norm": 3.951350212097168,
+      "learning_rate": 1.6446354475400117e-06,
+      "loss": 5.8012,
+      "mean_token_accuracy": 0.25434976994991304,
+      "num_tokens": 1465620.0,
       "step": 1400
     },
     {
-      "entropy": 6.141328382492065,
+      "entropy": 6.069429359436035,
       "epoch": 0.8347725964306275,
-      "grad_norm": 1.0346958637237549,
-      "learning_rate": 1.68573092603378e-05,
-      "loss": 5.9584,
-      "mean_token_accuracy": 0.23997059136629104,
-      "num_tokens": 1347539.0,
+      "grad_norm": 3.580608606338501,
+      "learning_rate": 1.6298162418494368e-06,
+      "loss": 5.8027,
+      "mean_token_accuracy": 0.25208072274923327,
+      "num_tokens": 1518899.0,
       "step": 1450
     },
     {
-      "entropy": 6.070010099411011,
+      "entropy": 6.005315380096436,
       "epoch": 0.8635578583765112,
-      "grad_norm": 1.6541163921356201,
-      "learning_rate": 1.674082702387886e-05,
-      "loss": 5.889,
-      "mean_token_accuracy": 0.24875166177749633,
-      "num_tokens": 1394157.0,
+      "grad_norm": 3.9580376148223877,
+      "learning_rate": 1.614997036158862e-06,
+      "loss": 5.7364,
+      "mean_token_accuracy": 0.25940640360116957,
+      "num_tokens": 1571304.0,
       "step": 1500
     },
     {
-      "entropy": 6.207450666427612,
+      "entropy": 6.0786464881896975,
       "epoch": 0.8923431203223949,
-      "grad_norm": 0.9742990732192993,
-      "learning_rate": 1.662434478741992e-05,
-      "loss": 6.0217,
-      "mean_token_accuracy": 0.23067249596118927,
-      "num_tokens": 1443892.0,
+      "grad_norm": 4.55721378326416,
+      "learning_rate": 1.6001778304682868e-06,
+      "loss": 5.8092,
+      "mean_token_accuracy": 0.2496869170665741,
+      "num_tokens": 1627369.0,
       "step": 1550
     },
     {
-      "entropy": 6.026197805404663,
+      "entropy": 5.939382014274597,
       "epoch": 0.9211283822682786,
-      "grad_norm": 1.4229531288146973,
-      "learning_rate": 1.650786255096098e-05,
-      "loss": 5.8455,
-      "mean_token_accuracy": 0.2537291014194489,
-      "num_tokens": 1491050.0,
+      "grad_norm": 2.330057144165039,
+      "learning_rate": 1.5853586247777117e-06,
+      "loss": 5.6604,
+      "mean_token_accuracy": 0.2686630353331566,
+      "num_tokens": 1680401.0,
       "step": 1600
     },
     {
-      "entropy": 6.210526428222656,
+      "entropy": 6.121775646209716,
       "epoch": 0.9499136442141624,
-      "grad_norm": 1.3555018901824951,
-      "learning_rate": 1.6391380314502038e-05,
-      "loss": 6.0279,
-      "mean_token_accuracy": 0.2308420208096504,
-      "num_tokens": 1540809.0,
+      "grad_norm": 2.9881200790405273,
+      "learning_rate": 1.5705394190871368e-06,
+      "loss": 5.8388,
+      "mean_token_accuracy": 0.2503683388233185,
+      "num_tokens": 1735745.0,
       "step": 1650
     },
     {
-      "entropy": 5.9872834014892575,
+      "entropy": 5.840040788650513,
       "epoch": 0.9786989061600461,
-      "grad_norm": 0.9893498420715332,
-      "learning_rate": 1.62748980780431e-05,
-      "loss": 5.8137,
-      "mean_token_accuracy": 0.2566875320672989,
-      "num_tokens": 1585876.0,
+      "grad_norm": 3.798994779586792,
+      "learning_rate": 1.555720213396562e-06,
+      "loss": 5.5635,
+      "mean_token_accuracy": 0.278279125392437,
+      "num_tokens": 1786896.0,
       "step": 1700
     },
     {
       "epoch": 1.0,
-      "eval_entropy": 6.322207130045386,
-      "eval_loss": 6.15173864364624,
-      "eval_mean_token_accuracy": 0.21116007946877985,
-      "eval_model_preparation_time": 0.0036,
-      "eval_num_tokens": 1619719.0,
-      "eval_runtime": 76.1297,
-      "eval_samples_per_second": 5.701,
-      "eval_steps_per_second": 2.85,
+      "eval_entropy": 6.139133475343203,
+      "eval_loss": 5.861395835876465,
+      "eval_mean_token_accuracy": 0.2402858340657801,
+      "eval_model_preparation_time": 0.0047,
+      "eval_num_tokens": 1825107.0,
+      "eval_runtime": 79.3994,
+      "eval_samples_per_second": 5.466,
+      "eval_steps_per_second": 2.733,
       "step": 1737
     }
   ],
   "logging_steps": 50,
-  "max_steps": 8685,
+  "max_steps": 6948,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 5,
+  "num_train_epochs": 4,
   "save_steps": 500,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -379,7 +379,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2.265889302609408e+16,
+  "total_flos": 2.5090142668416e+16,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null

checkpoint-1737/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:130d33149272782bd60306263c371036419926142b8999aad7806359168f8484
+oid sha256:8db5c304963110404ebb6947b83ba95bd9b8aad1f9b8b578cc33c46d601e13dc
 size 6225

checkpoint-3474/adapter_config.json

@@ -16,7 +16,7 @@
   "layers_pattern": null,
   "layers_to_transform": null,
   "loftq_config": {},
-  "lora_alpha": 16,
+  "lora_alpha": 32,
   "lora_bias": false,
   "lora_dropout": 0.1,
   "megatron_config": null,
@@ -25,12 +25,14 @@
   "peft_type": "LORA",
   "peft_version": "0.18.0",
   "qalora_group_size": 16,
-  "r": 8,
+  "r": 24,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
     "q_proj",
-    "v_proj"
+    "k_proj",
+    "v_proj",
+    "o_proj"
   ],
   "target_parameters": null,
   "task_type": "CAUSAL_LM",

checkpoint-3474/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b7979fe4ab41b842e564542d82ca738faea1a24cfcb2e3003501296353e2a240
-size 4374520
+oid sha256:a64c44cbe26eb26de9c868554476ac772a1101223d4511df741d375932e915d3
+size 26182176

checkpoint-3474/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:571f08123879a8157590252a0cd0abe24c345fd53c5c7a3b55bb8b256658f9c0
-size 8783179
+oid sha256:678993601594a7d04e501306f05a8d5de7ef3edaadbed87bc8a64e6f10f97582
+size 52486155

checkpoint-3474/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5f6c201154e30349ea924dac640f38cc7626e879caf89ba0aa995630585e3ea5
+oid sha256:388ebf23a81b449689f35e6de23bc7bbc9587bef795c318be18b9ce6620ad7a4
 size 14645

checkpoint-3474/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ecacb7697ae73257f39077a0e981cf0773317c0d0186dca0c24e0700ca53ab36
+oid sha256:9d0eb619e824095911c3281fa938e4204802f0a5951fcaf56996a5bc063db576
 size 1465

checkpoint-3474/trainer_state.json

@@ -1,6 +1,6 @@
 {
   "best_global_step": 3474,
-  "best_metric": 6.12472677230835,
+  "best_metric": 5.656307220458984,
   "best_model_checkpoint": "./output/checkpoint-3474",
   "epoch": 2.0,
   "eval_steps": 500,
@@ -10,724 +10,724 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "entropy": 3.864118957519531,
+      "entropy": 3.6583470726013183,
       "epoch": 0.028785261945883708,
-      "grad_norm": 2.7545533180236816,
-      "learning_rate": 9.800000000000001e-06,
-      "loss": 15.2997,
-      "mean_token_accuracy": 0.10086015284061432,
-      "num_tokens": 47319.0,
+      "grad_norm": 3.3817152976989746,
+      "learning_rate": 4.9e-07,
+      "loss": 13.8754,
+      "mean_token_accuracy": 0.15036460414528846,
+      "num_tokens": 53093.0,
       "step": 50
     },
     {
-      "entropy": 4.047076859474182,
+      "entropy": 3.669608063697815,
       "epoch": 0.057570523891767415,
-      "grad_norm": 5.0328264236450195,
-      "learning_rate": 1.98e-05,
-      "loss": 15.3264,
-      "mean_token_accuracy": 0.09582207053899765,
-      "num_tokens": 96809.0,
+      "grad_norm": 3.2541544437408447,
+      "learning_rate": 9.9e-07,
+      "loss": 14.2282,
+      "mean_token_accuracy": 0.14137721598148345,
+      "num_tokens": 108334.0,
       "step": 100
     },
     {
-      "entropy": 4.7578076648712155,
+      "entropy": 3.569736371040344,
       "epoch": 0.08635578583765112,
-      "grad_norm": 38.50589370727539,
-      "learning_rate": 1.988584740827024e-05,
-      "loss": 13.0056,
-      "mean_token_accuracy": 0.126854517608881,
-      "num_tokens": 139962.0,
+      "grad_norm": 3.6797454357147217,
+      "learning_rate": 1.49e-06,
+      "loss": 13.0735,
+      "mean_token_accuracy": 0.17473630651831626,
+      "num_tokens": 157491.0,
       "step": 150
     },
     {
-      "entropy": 6.80673882484436,
+      "entropy": 3.7253233194351196,
       "epoch": 0.11514104778353483,
-      "grad_norm": 12.030129432678223,
-      "learning_rate": 1.97693651718113e-05,
-      "loss": 9.2822,
-      "mean_token_accuracy": 0.11084575355052947,
-      "num_tokens": 188029.0,
+      "grad_norm": 4.297911643981934,
+      "learning_rate": 1.99e-06,
+      "loss": 13.7392,
+      "mean_token_accuracy": 0.1473099772632122,
+      "num_tokens": 211394.0,
       "step": 200
     },
     {
-      "entropy": 7.177925786972046,
+      "entropy": 3.8280500602722167,
       "epoch": 0.14392630972941853,
-      "grad_norm": 4.852536201477051,
-      "learning_rate": 1.965288293535236e-05,
-      "loss": 7.6333,
-      "mean_token_accuracy": 0.12398939326405525,
-      "num_tokens": 234425.0,
+      "grad_norm": 4.405268669128418,
+      "learning_rate": 1.9854771784232364e-06,
+      "loss": 13.0797,
+      "mean_token_accuracy": 0.16704789966344832,
+      "num_tokens": 263685.0,
       "step": 250
     },
     {
-      "entropy": 7.080496473312378,
+      "entropy": 4.066333084106446,
       "epoch": 0.17271157167530224,
-      "grad_norm": 4.10841178894043,
-      "learning_rate": 1.9536400698893422e-05,
-      "loss": 7.1632,
-      "mean_token_accuracy": 0.13563686355948448,
-      "num_tokens": 278885.0,
+      "grad_norm": 4.757556438446045,
+      "learning_rate": 1.9706579727326615e-06,
+      "loss": 12.6321,
+      "mean_token_accuracy": 0.1691790708899498,
+      "num_tokens": 314059.0,
       "step": 300
     },
     {
-      "entropy": 6.931579580307007,
+      "entropy": 4.257266030311585,
       "epoch": 0.20149683362118595,
-      "grad_norm": 14.636048316955566,
-      "learning_rate": 1.941991846243448e-05,
-      "loss": 6.8213,
-      "mean_token_accuracy": 0.16459846690297128,
-      "num_tokens": 325491.0,
+      "grad_norm": 6.406249523162842,
+      "learning_rate": 1.955838767042086e-06,
+      "loss": 12.2253,
+      "mean_token_accuracy": 0.17223650276660918,
+      "num_tokens": 367038.0,
       "step": 350
     },
     {
-      "entropy": 6.853660764694214,
+      "entropy": 4.694105777740479,
       "epoch": 0.23028209556706966,
-      "grad_norm": 5.966708183288574,
-      "learning_rate": 1.930343622597554e-05,
-      "loss": 6.6625,
-      "mean_token_accuracy": 0.17670693069696428,
-      "num_tokens": 372913.0,
+      "grad_norm": 12.57987117767334,
+      "learning_rate": 1.9410195613515113e-06,
+      "loss": 11.9714,
+      "mean_token_accuracy": 0.15997304677963256,
+      "num_tokens": 420327.0,
       "step": 400
     },
     {
-      "entropy": 6.684267387390137,
+      "entropy": 5.205010280609131,
       "epoch": 0.25906735751295334,
-      "grad_norm": 4.031010627746582,
-      "learning_rate": 1.91869539895166e-05,
-      "loss": 6.4505,
-      "mean_token_accuracy": 0.1943434515595436,
-      "num_tokens": 419159.0,
+      "grad_norm": 15.570313453674316,
+      "learning_rate": 1.9262003556609364e-06,
+      "loss": 10.8173,
+      "mean_token_accuracy": 0.16447648257017136,
+      "num_tokens": 472429.0,
       "step": 450
     },
     {
-      "entropy": 6.679989137649536,
+      "entropy": 5.917805089950561,
       "epoch": 0.28785261945883706,
-      "grad_norm": 6.251070022583008,
-      "learning_rate": 1.907047175305766e-05,
-      "loss": 6.4314,
-      "mean_token_accuracy": 0.19514557600021362,
-      "num_tokens": 466994.0,
+      "grad_norm": 23.61503791809082,
+      "learning_rate": 1.9113811499703615e-06,
+      "loss": 9.3196,
+      "mean_token_accuracy": 0.16179455041885377,
+      "num_tokens": 526315.0,
       "step": 500
     },
     {
-      "entropy": 6.477229623794556,
+      "entropy": 6.380368332862854,
       "epoch": 0.31663788140472077,
-      "grad_norm": 3.8656675815582275,
-      "learning_rate": 1.895398951659872e-05,
-      "loss": 6.2139,
-      "mean_token_accuracy": 0.21764743447303772,
-      "num_tokens": 513308.0,
+      "grad_norm": 13.846810340881348,
+      "learning_rate": 1.8965619442797864e-06,
+      "loss": 7.9636,
+      "mean_token_accuracy": 0.16881170988082886,
+      "num_tokens": 578511.0,
       "step": 550
     },
     {
-      "entropy": 6.408129243850708,
+      "entropy": 6.507339992523193,
       "epoch": 0.3454231433506045,
-      "grad_norm": 8.688581466674805,
-      "learning_rate": 1.883750728013978e-05,
-      "loss": 6.1224,
-      "mean_token_accuracy": 0.23438037544488907,
-      "num_tokens": 559679.0,
+      "grad_norm": 4.569090366363525,
+      "learning_rate": 1.8817427385892115e-06,
+      "loss": 7.4171,
+      "mean_token_accuracy": 0.16941152423620223,
+      "num_tokens": 630937.0,
       "step": 600
     },
     {
-      "entropy": 6.128518767356873,
+      "entropy": 6.392864561080932,
       "epoch": 0.3742084052964882,
-      "grad_norm": 5.419503688812256,
-      "learning_rate": 1.872102504368084e-05,
-      "loss": 5.8692,
-      "mean_token_accuracy": 0.26634690463542937,
-      "num_tokens": 603140.0,
+      "grad_norm": 4.594696521759033,
+      "learning_rate": 1.8669235328986366e-06,
+      "loss": 6.9389,
+      "mean_token_accuracy": 0.1844496901333332,
+      "num_tokens": 680501.0,
       "step": 650
     },
     {
-      "entropy": 6.322700729370117,
+      "entropy": 6.6726202869415285,
       "epoch": 0.4029936672423719,
-      "grad_norm": 2.2213082313537598,
-      "learning_rate": 1.86045428072219e-05,
-      "loss": 6.0717,
-      "mean_token_accuracy": 0.24038562417030335,
-      "num_tokens": 650179.0,
+      "grad_norm": 4.768734931945801,
+      "learning_rate": 1.8521043272080617e-06,
+      "loss": 6.9818,
+      "mean_token_accuracy": 0.16990411713719367,
+      "num_tokens": 733231.0,
       "step": 700
     },
     {
-      "entropy": 6.236415157318115,
+      "entropy": 6.592793455123902,
       "epoch": 0.4317789291882556,
-      "grad_norm": 4.804980278015137,
-      "learning_rate": 1.848806057076296e-05,
-      "loss": 5.9986,
-      "mean_token_accuracy": 0.24596781462430953,
-      "num_tokens": 696220.0,
+      "grad_norm": 3.253056764602661,
+      "learning_rate": 1.8372851215174864e-06,
+      "loss": 6.7105,
+      "mean_token_accuracy": 0.18250102579593658,
+      "num_tokens": 785373.0,
       "step": 750
     },
     {
-      "entropy": 6.269758443832398,
+      "entropy": 6.683582029342651,
       "epoch": 0.4605641911341393,
-      "grad_norm": 2.2888853549957275,
-      "learning_rate": 1.837157833430402e-05,
-      "loss": 6.0385,
-      "mean_token_accuracy": 0.24074893474578857,
-      "num_tokens": 743909.0,
+      "grad_norm": 2.1871063709259033,
+      "learning_rate": 1.8224659158269115e-06,
+      "loss": 6.6685,
+      "mean_token_accuracy": 0.17129646152257919,
+      "num_tokens": 838646.0,
       "step": 800
     },
     {
-      "entropy": 6.270364007949829,
+      "entropy": 6.636875295639038,
       "epoch": 0.48934945308002303,
-      "grad_norm": 3.0903279781341553,
-      "learning_rate": 1.825509609784508e-05,
-      "loss": 6.0481,
-      "mean_token_accuracy": 0.23740622967481614,
-      "num_tokens": 792015.0,
+      "grad_norm": 3.2284677028656006,
+      "learning_rate": 1.8076467101363366e-06,
+      "loss": 6.53,
+      "mean_token_accuracy": 0.18053789794445038,
+      "num_tokens": 892380.0,
       "step": 850
     },
     {
-      "entropy": 6.3037636184692385,
+      "entropy": 6.610673260688782,
       "epoch": 0.5181347150259067,
-      "grad_norm": 3.969320058822632,
-      "learning_rate": 1.813861386138614e-05,
-      "loss": 6.0855,
-      "mean_token_accuracy": 0.2309597587585449,
-      "num_tokens": 841802.0,
+      "grad_norm": 2.2088730335235596,
+      "learning_rate": 1.7928275044457617e-06,
+      "loss": 6.4429,
+      "mean_token_accuracy": 0.18492739230394364,
+      "num_tokens": 947971.0,
       "step": 900
     },
     {
-      "entropy": 6.038041458129883,
+      "entropy": 6.242899022102356,
       "epoch": 0.5469199769717904,
-      "grad_norm": 2.2712185382843018,
-      "learning_rate": 1.80221316249272e-05,
-      "loss": 5.8285,
-      "mean_token_accuracy": 0.26099125802516937,
-      "num_tokens": 886492.0,
+      "grad_norm": 2.3000030517578125,
+      "learning_rate": 1.7780082987551866e-06,
+      "loss": 6.047,
+      "mean_token_accuracy": 0.2291259828209877,
+      "num_tokens": 998810.0,
       "step": 950
     },
     {
-      "entropy": 6.142958383560181,
+      "entropy": 6.311488924026489,
       "epoch": 0.5757052389176741,
-      "grad_norm": 1.2311755418777466,
-      "learning_rate": 1.790564938846826e-05,
-      "loss": 5.9357,
-      "mean_token_accuracy": 0.24810438305139543,
-      "num_tokens": 932807.0,
+      "grad_norm": 2.1333675384521484,
+      "learning_rate": 1.7631890930646115e-06,
+      "loss": 6.0919,
+      "mean_token_accuracy": 0.22644571751356124,
+      "num_tokens": 1050860.0,
       "step": 1000
     },
     {
-      "entropy": 6.199834351539612,
+      "entropy": 6.3254336166381835,
       "epoch": 0.6044905008635578,
-      "grad_norm": 2.2788379192352295,
-      "learning_rate": 1.7789167152009318e-05,
-      "loss": 5.9964,
-      "mean_token_accuracy": 0.23942562609910964,
-      "num_tokens": 980541.0,
+      "grad_norm": 2.0400779247283936,
+      "learning_rate": 1.7483698873740366e-06,
+      "loss": 6.094,
+      "mean_token_accuracy": 0.2222653564810753,
+      "num_tokens": 1104304.0,
       "step": 1050
     },
     {
-      "entropy": 5.961639919281006,
+      "entropy": 6.046922063827514,
       "epoch": 0.6332757628094415,
-      "grad_norm": 1.9077532291412354,
-      "learning_rate": 1.767268491555038e-05,
-      "loss": 5.7664,
-      "mean_token_accuracy": 0.26718012750148773,
-      "num_tokens": 1023882.0,
+      "grad_norm": 2.8049051761627197,
+      "learning_rate": 1.7335506816834617e-06,
+      "loss": 5.8011,
+      "mean_token_accuracy": 0.25127078920602797,
+      "num_tokens": 1153605.0,
       "step": 1100
     },
     {
-      "entropy": 5.889280087947846,
+      "entropy": 5.943600912094116,
       "epoch": 0.6620610247553252,
-      "grad_norm": 2.4254891872406006,
-      "learning_rate": 1.7556202679091442e-05,
-      "loss": 5.6952,
-      "mean_token_accuracy": 0.27529804170131683,
-      "num_tokens": 1068300.0,
+      "grad_norm": 4.063963890075684,
+      "learning_rate": 1.7187314759928866e-06,
+      "loss": 5.6855,
+      "mean_token_accuracy": 0.26265266716480257,
+      "num_tokens": 1204328.0,
       "step": 1150
     },
     {
-      "entropy": 6.085640063285828,
+      "entropy": 6.12883231639862,
       "epoch": 0.690846286701209,
-      "grad_norm": 2.35312557220459,
-      "learning_rate": 1.74397204426325e-05,
-      "loss": 5.8898,
-      "mean_token_accuracy": 0.25166562348604204,
-      "num_tokens": 1115425.0,
+      "grad_norm": 3.9440460205078125,
+      "learning_rate": 1.7039122703023117e-06,
+      "loss": 5.8578,
+      "mean_token_accuracy": 0.24439335912466048,
+      "num_tokens": 1257415.0,
       "step": 1200
     },
     {
-      "entropy": 6.146574058532715,
+      "entropy": 6.164987115859986,
       "epoch": 0.7196315486470927,
-      "grad_norm": 1.7730146646499634,
-      "learning_rate": 1.732323820617356e-05,
-      "loss": 5.9519,
-      "mean_token_accuracy": 0.24276195973157882,
-      "num_tokens": 1162319.0,
+      "grad_norm": 3.20070481300354,
+      "learning_rate": 1.6890930646117368e-06,
+      "loss": 5.8876,
+      "mean_token_accuracy": 0.24275501281023026,
+      "num_tokens": 1310049.0,
       "step": 1250
     },
     {
-      "entropy": 6.079372715950012,
+      "entropy": 6.080997190475464,
       "epoch": 0.7484168105929764,
-      "grad_norm": 1.7070863246917725,
-      "learning_rate": 1.720675596971462e-05,
-      "loss": 5.8922,
-      "mean_token_accuracy": 0.24961524546146394,
-      "num_tokens": 1208230.0,
+      "grad_norm": 2.8067362308502197,
+      "learning_rate": 1.6742738589211617e-06,
+      "loss": 5.8058,
+      "mean_token_accuracy": 0.25242207854986193,
+      "num_tokens": 1361794.0,
       "step": 1300
     },
     {
-      "entropy": 5.9683656406402585,
+      "entropy": 5.940848155021667,
       "epoch": 0.7772020725388601,
-      "grad_norm": 1.8790594339370728,
-      "learning_rate": 1.709027373325568e-05,
-      "loss": 5.7827,
-      "mean_token_accuracy": 0.2632122594118118,
-      "num_tokens": 1253074.0,
+      "grad_norm": 2.6375925540924072,
+      "learning_rate": 1.6594546532305868e-06,
+      "loss": 5.6718,
+      "mean_token_accuracy": 0.2665082859992981,
+      "num_tokens": 1412773.0,
       "step": 1350
     },
     {
-      "entropy": 6.107076721191406,
+      "entropy": 6.071129274368286,
       "epoch": 0.8059873344847438,
-      "grad_norm": 1.1745644807815552,
-      "learning_rate": 1.6973791496796742e-05,
-      "loss": 5.9211,
-      "mean_token_accuracy": 0.24564073830842972,
-      "num_tokens": 1300179.0,
+      "grad_norm": 3.951350212097168,
+      "learning_rate": 1.6446354475400117e-06,
+      "loss": 5.8012,
+      "mean_token_accuracy": 0.25434976994991304,
+      "num_tokens": 1465620.0,
       "step": 1400
     },
     {
-      "entropy": 6.141328382492065,
+      "entropy": 6.069429359436035,
       "epoch": 0.8347725964306275,
-      "grad_norm": 1.0346958637237549,
-      "learning_rate": 1.68573092603378e-05,
-      "loss": 5.9584,
-      "mean_token_accuracy": 0.23997059136629104,
-      "num_tokens": 1347539.0,
+      "grad_norm": 3.580608606338501,
+      "learning_rate": 1.6298162418494368e-06,
+      "loss": 5.8027,
+      "mean_token_accuracy": 0.25208072274923327,
+      "num_tokens": 1518899.0,
       "step": 1450
     },
     {
-      "entropy": 6.070010099411011,
+      "entropy": 6.005315380096436,
       "epoch": 0.8635578583765112,
-      "grad_norm": 1.6541163921356201,
-      "learning_rate": 1.674082702387886e-05,
-      "loss": 5.889,
-      "mean_token_accuracy": 0.24875166177749633,
-      "num_tokens": 1394157.0,
+      "grad_norm": 3.9580376148223877,
+      "learning_rate": 1.614997036158862e-06,
+      "loss": 5.7364,
+      "mean_token_accuracy": 0.25940640360116957,
+      "num_tokens": 1571304.0,
       "step": 1500
     },
     {
-      "entropy": 6.207450666427612,
+      "entropy": 6.0786464881896975,
       "epoch": 0.8923431203223949,
-      "grad_norm": 0.9742990732192993,
-      "learning_rate": 1.662434478741992e-05,
-      "loss": 6.0217,
-      "mean_token_accuracy": 0.23067249596118927,
-      "num_tokens": 1443892.0,
+      "grad_norm": 4.55721378326416,
+      "learning_rate": 1.6001778304682868e-06,
+      "loss": 5.8092,
+      "mean_token_accuracy": 0.2496869170665741,
+      "num_tokens": 1627369.0,
       "step": 1550
     },
     {
-      "entropy": 6.026197805404663,
+      "entropy": 5.939382014274597,
       "epoch": 0.9211283822682786,
-      "grad_norm": 1.4229531288146973,
-      "learning_rate": 1.650786255096098e-05,
-      "loss": 5.8455,
-      "mean_token_accuracy": 0.2537291014194489,
-      "num_tokens": 1491050.0,
+      "grad_norm": 2.330057144165039,
+      "learning_rate": 1.5853586247777117e-06,
+      "loss": 5.6604,
+      "mean_token_accuracy": 0.2686630353331566,
+      "num_tokens": 1680401.0,
       "step": 1600
     },
     {
-      "entropy": 6.210526428222656,
+      "entropy": 6.121775646209716,
       "epoch": 0.9499136442141624,
-      "grad_norm": 1.3555018901824951,
-      "learning_rate": 1.6391380314502038e-05,
-      "loss": 6.0279,
-      "mean_token_accuracy": 0.2308420208096504,
-      "num_tokens": 1540809.0,
+      "grad_norm": 2.9881200790405273,
+      "learning_rate": 1.5705394190871368e-06,
+      "loss": 5.8388,
+      "mean_token_accuracy": 0.2503683388233185,
+      "num_tokens": 1735745.0,
       "step": 1650
     },
     {
-      "entropy": 5.9872834014892575,
+      "entropy": 5.840040788650513,
       "epoch": 0.9786989061600461,
-      "grad_norm": 0.9893498420715332,
-      "learning_rate": 1.62748980780431e-05,
-      "loss": 5.8137,
-      "mean_token_accuracy": 0.2566875320672989,
-      "num_tokens": 1585876.0,
+      "grad_norm": 3.798994779586792,
+      "learning_rate": 1.555720213396562e-06,
+      "loss": 5.5635,
+      "mean_token_accuracy": 0.278279125392437,
+      "num_tokens": 1786896.0,
       "step": 1700
     },
     {
       "epoch": 1.0,
-      "eval_entropy": 6.322207130045386,
-      "eval_loss": 6.15173864364624,
-      "eval_mean_token_accuracy": 0.21116007946877985,
-      "eval_model_preparation_time": 0.0036,
-      "eval_num_tokens": 1619719.0,
-      "eval_runtime": 76.1297,
-      "eval_samples_per_second": 5.701,
-      "eval_steps_per_second": 2.85,
+      "eval_entropy": 6.139133475343203,
+      "eval_loss": 5.861395835876465,
+      "eval_mean_token_accuracy": 0.2402858340657801,
+      "eval_model_preparation_time": 0.0047,
+      "eval_num_tokens": 1825107.0,
+      "eval_runtime": 79.3994,
+      "eval_samples_per_second": 5.466,
+      "eval_steps_per_second": 2.733,
       "step": 1737
     },
     {
-      "entropy": 6.038531675338745,
+      "entropy": 5.8970259666442875,
       "epoch": 1.0074841681059297,
-      "grad_norm": 0.8715208172798157,
-      "learning_rate": 1.615841584158416e-05,
-      "loss": 5.8628,
-      "mean_token_accuracy": 0.2510762655735016,
-      "num_tokens": 1632015.0,
+      "grad_norm": 2.6411802768707275,
+      "learning_rate": 1.540901007705987e-06,
+      "loss": 5.614,
+      "mean_token_accuracy": 0.273006406724453,
+      "num_tokens": 1838864.0,
       "step": 1750
     },
     {
-      "entropy": 6.164030771255494,
+      "entropy": 6.0111794090271,
       "epoch": 1.0362694300518134,
-      "grad_norm": 0.7344900965690613,
-      "learning_rate": 1.604193360512522e-05,
-      "loss": 5.9856,
-      "mean_token_accuracy": 0.2351543301343918,
-      "num_tokens": 1681154.0,
+      "grad_norm": 3.6491827964782715,
+      "learning_rate": 1.526081802015412e-06,
+      "loss": 5.7323,
+      "mean_token_accuracy": 0.26104256987571717,
+      "num_tokens": 1893816.0,
       "step": 1800
     },
     {
-      "entropy": 6.0731862354278565,
+      "entropy": 5.902219276428223,
       "epoch": 1.065054691997697,
-      "grad_norm": 1.0801328420639038,
-      "learning_rate": 1.592545136866628e-05,
-      "loss": 5.8976,
-      "mean_token_accuracy": 0.24701615989208223,
-      "num_tokens": 1728110.0,
+      "grad_norm": 2.593249559402466,
+      "learning_rate": 1.5112625963248368e-06,
+      "loss": 5.6187,
+      "mean_token_accuracy": 0.2746362566947937,
+      "num_tokens": 1946532.0,
       "step": 1850
     },
     {
-      "entropy": 6.079212121963501,
+      "entropy": 5.874705944061279,
       "epoch": 1.0938399539435808,
-      "grad_norm": 0.7876909375190735,
-      "learning_rate": 1.5808969132207338e-05,
-      "loss": 5.9056,
-      "mean_token_accuracy": 0.24457543224096298,
-      "num_tokens": 1775703.0,
+      "grad_norm": 2.554327964782715,
+      "learning_rate": 1.496443390634262e-06,
+      "loss": 5.6021,
+      "mean_token_accuracy": 0.2795292744040489,
+      "num_tokens": 2000184.0,
       "step": 1900
     },
     {
-      "entropy": 6.062467746734619,
+      "entropy": 5.850096368789673,
       "epoch": 1.1226252158894645,
-      "grad_norm": 0.5999078750610352,
-      "learning_rate": 1.56924868957484e-05,
-      "loss": 5.8899,
-      "mean_token_accuracy": 0.2469428673386574,
-      "num_tokens": 1821980.0,
+      "grad_norm": 3.6060993671417236,
+      "learning_rate": 1.481624184943687e-06,
+      "loss": 5.576,
+      "mean_token_accuracy": 0.28532547056674956,
+      "num_tokens": 2052250.0,
       "step": 1950
     },
     {
-      "entropy": 6.031774473190308,
+      "entropy": 5.802229671478272,
       "epoch": 1.1514104778353482,
-      "grad_norm": 1.6313235759735107,
-      "learning_rate": 1.557600465928946e-05,
-      "loss": 5.8593,
-      "mean_token_accuracy": 0.250918984413147,
-      "num_tokens": 1867547.0,
+      "grad_norm": 3.0913314819335938,
+      "learning_rate": 1.466804979253112e-06,
+      "loss": 5.53,
+      "mean_token_accuracy": 0.2916027933359146,
+      "num_tokens": 2103531.0,
       "step": 2000
     },
     {
-      "entropy": 6.122789564132691,
+      "entropy": 5.875646467208862,
       "epoch": 1.180195739781232,
-      "grad_norm": 2.562373161315918,
-      "learning_rate": 1.545952242283052e-05,
-      "loss": 5.9502,
-      "mean_token_accuracy": 0.23938885867595672,
-      "num_tokens": 1915411.0,
+      "grad_norm": 4.777045726776123,
+      "learning_rate": 1.451985773562537e-06,
+      "loss": 5.6146,
+      "mean_token_accuracy": 0.28063644528388976,
+      "num_tokens": 2157098.0,
       "step": 2050
     },
     {
-      "entropy": 6.067130417823791,
+      "entropy": 5.786596937179565,
       "epoch": 1.2089810017271156,
-      "grad_norm": 0.9762872457504272,
-      "learning_rate": 1.534304018637158e-05,
-      "loss": 5.8956,
-      "mean_token_accuracy": 0.2454381173849106,
-      "num_tokens": 1964009.0,
+      "grad_norm": 4.207762718200684,
+      "learning_rate": 1.437166567871962e-06,
+      "loss": 5.5417,
+      "mean_token_accuracy": 0.2870470091700554,
+      "num_tokens": 2211827.0,
       "step": 2100
     },
     {
-      "entropy": 5.9613511180877685,
+      "entropy": 5.672234449386597,
       "epoch": 1.2377662636729994,
-      "grad_norm": 0.8701547384262085,
-      "learning_rate": 1.5226557949912639e-05,
-      "loss": 5.7907,
-      "mean_token_accuracy": 0.25976367652416227,
-      "num_tokens": 2008595.0,
+      "grad_norm": 2.2771811485290527,
+      "learning_rate": 1.422347362181387e-06,
+      "loss": 5.4285,
+      "mean_token_accuracy": 0.30194485366344453,
+      "num_tokens": 2262174.0,
       "step": 2150
     },
     {
-      "entropy": 6.13505428314209,
+      "entropy": 5.862573285102844,
       "epoch": 1.266551525618883,
-      "grad_norm": 0.8511647582054138,
-      "learning_rate": 1.51100757134537e-05,
-      "loss": 5.9619,
-      "mean_token_accuracy": 0.23760781466960906,
-      "num_tokens": 2057229.0,
+      "grad_norm": 3.3273422718048096,
+      "learning_rate": 1.4075281564908121e-06,
+      "loss": 5.6169,
+      "mean_token_accuracy": 0.278145115673542,
+      "num_tokens": 2316440.0,
       "step": 2200
     },
     {
-      "entropy": 6.025254983901977,
+      "entropy": 5.734760231971741,
       "epoch": 1.2953367875647668,
-      "grad_norm": 0.7627406120300293,
-      "learning_rate": 1.4993593476994758e-05,
-      "loss": 5.8546,
-      "mean_token_accuracy": 0.2508662334084511,
-      "num_tokens": 2103631.0,
+      "grad_norm": 3.7049715518951416,
+      "learning_rate": 1.392708950800237e-06,
+      "loss": 5.493,
+      "mean_token_accuracy": 0.2941485676169395,
+      "num_tokens": 2368468.0,
       "step": 2250
     },
     {
-      "entropy": 5.981974196434021,
+      "entropy": 5.665819988250733,
       "epoch": 1.3241220495106505,
-      "grad_norm": 1.6922173500061035,
-      "learning_rate": 1.4877111240535819e-05,
-      "loss": 5.8119,
-      "mean_token_accuracy": 0.256170334815979,
-      "num_tokens": 2150369.0,
+      "grad_norm": 3.572636604309082,
+      "learning_rate": 1.3778897451096621e-06,
+      "loss": 5.4352,
+      "mean_token_accuracy": 0.3003745040297508,
+      "num_tokens": 2421180.0,
       "step": 2300
     },
     {
-      "entropy": 6.19903904914856,
+      "entropy": 5.890115032196045,
       "epoch": 1.3529073114565342,
-      "grad_norm": 0.40436601638793945,
-      "learning_rate": 1.4760629004076878e-05,
-      "loss": 6.0244,
-      "mean_token_accuracy": 0.22900927513837815,
-      "num_tokens": 2199724.0,
+      "grad_norm": 2.738203525543213,
+      "learning_rate": 1.3630705394190872e-06,
+      "loss": 5.6555,
+      "mean_token_accuracy": 0.2737997192144394,
+      "num_tokens": 2476255.0,
       "step": 2350
     },
     {
-      "entropy": 5.986697297096253,
+      "entropy": 5.66056040763855,
       "epoch": 1.381692573402418,
-      "grad_norm": 0.8481882214546204,
-      "learning_rate": 1.464414676761794e-05,
-      "loss": 5.8195,
-      "mean_token_accuracy": 0.2552035376429558,
-      "num_tokens": 2245341.0,
+      "grad_norm": 3.1416995525360107,
+      "learning_rate": 1.3482513337285121e-06,
+      "loss": 5.4302,
+      "mean_token_accuracy": 0.3000989046692848,
+      "num_tokens": 2527674.0,
       "step": 2400
     },
     {
-      "entropy": 6.1886044692993165,
+      "entropy": 5.861240615844727,
       "epoch": 1.4104778353483016,
-      "grad_norm": 0.7911505103111267,
-      "learning_rate": 1.4527664531159e-05,
-      "loss": 6.0148,
-      "mean_token_accuracy": 0.23026730984449387,
-      "num_tokens": 2294726.0,
+      "grad_norm": 2.7569284439086914,
+      "learning_rate": 1.333432128037937e-06,
+      "loss": 5.6304,
+      "mean_token_accuracy": 0.27707513481378554,
+      "num_tokens": 2582909.0,
       "step": 2450
     },
     {
-      "entropy": 5.974867792129516,
+      "entropy": 5.627686910629272,
       "epoch": 1.4392630972941853,
-      "grad_norm": 1.640499234199524,
-      "learning_rate": 1.441118229470006e-05,
-      "loss": 5.8111,
-      "mean_token_accuracy": 0.2554209426045418,
-      "num_tokens": 2342251.0,
+      "grad_norm": 1.7750262022018433,
+      "learning_rate": 1.3186129223473621e-06,
+      "loss": 5.4058,
+      "mean_token_accuracy": 0.3019809901714325,
+      "num_tokens": 2636579.0,
       "step": 2500
     },
     {
-      "entropy": 5.967635660171509,
+      "entropy": 5.607026796340943,
       "epoch": 1.468048359240069,
-      "grad_norm": 0.8022929430007935,
-      "learning_rate": 1.429470005824112e-05,
-      "loss": 5.8015,
-      "mean_token_accuracy": 0.2569852137565613,
-      "num_tokens": 2387469.0,
+      "grad_norm": 3.1005160808563232,
+      "learning_rate": 1.3037937166567872e-06,
+      "loss": 5.3836,
+      "mean_token_accuracy": 0.30584611505270004,
+      "num_tokens": 2687698.0,
       "step": 2550
     },
     {
-      "entropy": 6.047262029647827,
+      "entropy": 5.6909641885757445,
       "epoch": 1.4968336211859528,
-      "grad_norm": 0.9270678758621216,
-      "learning_rate": 1.417821782178218e-05,
-      "loss": 5.8782,
-      "mean_token_accuracy": 0.2467849862575531,
-      "num_tokens": 2434128.0,
+      "grad_norm": 1.6848654747009277,
+      "learning_rate": 1.2889745109662123e-06,
+      "loss": 5.4653,
+      "mean_token_accuracy": 0.296178964972496,
+      "num_tokens": 2740214.0,
       "step": 2600
     },
     {
-      "entropy": 6.00601068019867,
+      "entropy": 5.619450302124023,
       "epoch": 1.5256188831318365,
-      "grad_norm": 1.5378597974777222,
-      "learning_rate": 1.406173558532324e-05,
-      "loss": 5.839,
-      "mean_token_accuracy": 0.25216978013515473,
-      "num_tokens": 2480366.0,
+      "grad_norm": 2.469539165496826,
+      "learning_rate": 1.274155305275637e-06,
+      "loss": 5.4022,
+      "mean_token_accuracy": 0.3039679077267647,
+      "num_tokens": 2792574.0,
       "step": 2650
     },
     {
-      "entropy": 5.988714299201965,
+      "entropy": 5.61073097705841,
       "epoch": 1.5544041450777202,
-      "grad_norm": 0.819143533706665,
-      "learning_rate": 1.3945253348864299e-05,
-      "loss": 5.82,
-      "mean_token_accuracy": 0.254311783015728,
-      "num_tokens": 2527357.0,
+      "grad_norm": 2.367810010910034,
+      "learning_rate": 1.259336099585062e-06,
+      "loss": 5.3956,
+      "mean_token_accuracy": 0.3051413372159004,
+      "num_tokens": 2845597.0,
       "step": 2700
     },
     {
-      "entropy": 5.960293846130371,
+      "entropy": 5.5791136837005615,
       "epoch": 1.583189407023604,
-      "grad_norm": 0.8920449614524841,
-      "learning_rate": 1.382877111240536e-05,
-      "loss": 5.7946,
-      "mean_token_accuracy": 0.25750755161046984,
-      "num_tokens": 2574470.0,
+      "grad_norm": 2.3874764442443848,
+      "learning_rate": 1.2445168938944872e-06,
+      "loss": 5.3676,
+      "mean_token_accuracy": 0.3068238252401352,
+      "num_tokens": 2898683.0,
       "step": 2750
     },
     {
-      "entropy": 6.1214879322052,
+      "entropy": 5.735381307601929,
       "epoch": 1.6119746689694876,
-      "grad_norm": 0.5333890914916992,
-      "learning_rate": 1.371228887594642e-05,
-      "loss": 5.9513,
-      "mean_token_accuracy": 0.2377367687225342,
-      "num_tokens": 2622280.0,
+      "grad_norm": 2.2097349166870117,
+      "learning_rate": 1.2296976882039123e-06,
+      "loss": 5.5239,
+      "mean_token_accuracy": 0.28974882304668426,
+      "num_tokens": 2952290.0,
       "step": 2800
     },
     {
-      "entropy": 5.951769871711731,
+      "entropy": 5.55252691745758,
       "epoch": 1.6407599309153713,
-      "grad_norm": 0.5994665026664734,
-      "learning_rate": 1.3595806639487479e-05,
-      "loss": 5.7861,
-      "mean_token_accuracy": 0.25854207515716554,
-      "num_tokens": 2668624.0,
+      "grad_norm": 1.694831132888794,
+      "learning_rate": 1.2148784825133372e-06,
+      "loss": 5.351,
+      "mean_token_accuracy": 0.3091904193162918,
+      "num_tokens": 3004556.0,
       "step": 2850
     },
     {
-      "entropy": 5.927765312194825,
+      "entropy": 5.508773093223572,
       "epoch": 1.669545192861255,
-      "grad_norm": 0.4460087716579437,
-      "learning_rate": 1.347932440302854e-05,
-      "loss": 5.7661,
-      "mean_token_accuracy": 0.25973255425691605,
-      "num_tokens": 2714388.0,
+      "grad_norm": 1.8229279518127441,
+      "learning_rate": 1.200059276822762e-06,
+      "loss": 5.3164,
+      "mean_token_accuracy": 0.31158645361661913,
+      "num_tokens": 3056448.0,
       "step": 2900
     },
     {
-      "entropy": 6.097678365707398,
+      "entropy": 5.676794271469117,
       "epoch": 1.6983304548071387,
-      "grad_norm": 0.7125752568244934,
-      "learning_rate": 1.3362842166569598e-05,
-      "loss": 5.9284,
-      "mean_token_accuracy": 0.23995368272066117,
-      "num_tokens": 2761465.0,
+      "grad_norm": 1.7196234464645386,
+      "learning_rate": 1.1852400711321872e-06,
+      "loss": 5.4776,
+      "mean_token_accuracy": 0.2929128894209862,
+      "num_tokens": 3109539.0,
       "step": 2950
     },
     {
-      "entropy": 5.986212658882141,
+      "entropy": 5.551529383659362,
       "epoch": 1.7271157167530224,
-      "grad_norm": 1.5405049324035645,
-      "learning_rate": 1.3246359930110659e-05,
-      "loss": 5.8194,
-      "mean_token_accuracy": 0.25333445996046067,
-      "num_tokens": 2808066.0,
+      "grad_norm": 3.117525577545166,
+      "learning_rate": 1.1704208654416123e-06,
+      "loss": 5.3561,
+      "mean_token_accuracy": 0.30634030640125276,
+      "num_tokens": 3162421.0,
       "step": 3000
     },
     {
-      "entropy": 5.7968806195259095,
+      "entropy": 5.379635264873505,
       "epoch": 1.7559009786989062,
-      "grad_norm": 0.4532749652862549,
-      "learning_rate": 1.312987769365172e-05,
-      "loss": 5.6344,
-      "mean_token_accuracy": 0.2782411390542984,
-      "num_tokens": 2851822.0,
+      "grad_norm": 1.876755714416504,
+      "learning_rate": 1.1556016597510372e-06,
+      "loss": 5.1868,
+      "mean_token_accuracy": 0.32913618892431257,
+      "num_tokens": 3212079.0,
       "step": 3050
     },
     {
-      "entropy": 5.973708114624023,
+      "entropy": 5.538804936408996,
       "epoch": 1.7846862406447899,
-      "grad_norm": 1.4795438051223755,
-      "learning_rate": 1.3013395457192778e-05,
-      "loss": 5.8104,
-      "mean_token_accuracy": 0.25441971331834795,
-      "num_tokens": 2897737.0,
+      "grad_norm": 1.8670976161956787,
+      "learning_rate": 1.1407824540604623e-06,
+      "loss": 5.3494,
+      "mean_token_accuracy": 0.30661171555519107,
+      "num_tokens": 3264089.0,
       "step": 3100
     },
     {
-      "entropy": 5.70733567237854,
+      "entropy": 5.258263626098633,
       "epoch": 1.8134715025906736,
-      "grad_norm": 0.6216577887535095,
-      "learning_rate": 1.2896913220733839e-05,
-      "loss": 5.5523,
-      "mean_token_accuracy": 0.28787180870771406,
-      "num_tokens": 2939511.0,
+      "grad_norm": 2.748718023300171,
+      "learning_rate": 1.1259632483698874e-06,
+      "loss": 5.08,
+      "mean_token_accuracy": 0.3413010013103485,
+      "num_tokens": 3311881.0,
       "step": 3150
     },
     {
-      "entropy": 5.96826630115509,
+      "entropy": 5.54539008140564,
       "epoch": 1.8422567645365573,
-      "grad_norm": 0.9246350526809692,
-      "learning_rate": 1.2780430984274898e-05,
-      "loss": 5.8057,
-      "mean_token_accuracy": 0.25464902341365814,
-      "num_tokens": 2986368.0,
+      "grad_norm": 1.8556406497955322,
+      "learning_rate": 1.1111440426793123e-06,
+      "loss": 5.3614,
+      "mean_token_accuracy": 0.30550685405731204,
+      "num_tokens": 3364861.0,
       "step": 3200
     },
     {
-      "entropy": 5.950662693977356,
+      "entropy": 5.5433073282241825,
       "epoch": 1.871042026482441,
-      "grad_norm": 0.8141199946403503,
-      "learning_rate": 1.266394874781596e-05,
-      "loss": 5.7886,
-      "mean_token_accuracy": 0.25830793648958206,
-      "num_tokens": 3031770.0,
+      "grad_norm": 1.8386749029159546,
+      "learning_rate": 1.0963248369887374e-06,
+      "loss": 5.3543,
+      "mean_token_accuracy": 0.30875524014234546,
+      "num_tokens": 3415911.0,
       "step": 3250
     },
     {
-      "entropy": 6.00512773513794,
+      "entropy": 5.5769769477844235,
       "epoch": 1.8998272884283247,
-      "grad_norm": 0.4913998246192932,
-      "learning_rate": 1.2547466511357018e-05,
-      "loss": 5.838,
-      "mean_token_accuracy": 0.2512077575922012,
-      "num_tokens": 3078322.0,
+      "grad_norm": 1.922486662864685,
+      "learning_rate": 1.0815056312981623e-06,
+      "loss": 5.3834,
+      "mean_token_accuracy": 0.3035113242268562,
+      "num_tokens": 3468338.0,
       "step": 3300
     },
     {
-      "entropy": 6.090880632400513,
+      "entropy": 5.640013842582703,
       "epoch": 1.9286125503742084,
-      "grad_norm": 0.9893012046813965,
-      "learning_rate": 1.243098427489808e-05,
-      "loss": 5.9264,
-      "mean_token_accuracy": 0.2391783133149147,
-      "num_tokens": 3125572.0,
+      "grad_norm": 2.179500102996826,
+      "learning_rate": 1.0666864256075874e-06,
+      "loss": 5.4574,
+      "mean_token_accuracy": 0.2947095710039139,
+      "num_tokens": 3521693.0,
       "step": 3350
     },
     {
-      "entropy": 5.949693293571472,
+      "entropy": 5.506910061836242,
       "epoch": 1.9573978123200921,
-      "grad_norm": 0.5794200301170349,
-      "learning_rate": 1.231450203843914e-05,
-      "loss": 5.7861,
-      "mean_token_accuracy": 0.2568664598464966,
-      "num_tokens": 3171974.0,
+      "grad_norm": 1.4014379978179932,
+      "learning_rate": 1.0518672199170125e-06,
+      "loss": 5.3234,
+      "mean_token_accuracy": 0.3096472260355949,
+      "num_tokens": 3574206.0,
       "step": 3400
     },
     {
-      "entropy": 6.03591317653656,
+      "entropy": 5.607311015129089,
       "epoch": 1.9861830742659758,
-      "grad_norm": 0.8525373339653015,
-      "learning_rate": 1.21980198019802e-05,
-      "loss": 5.8741,
-      "mean_token_accuracy": 0.24642003327608109,
-      "num_tokens": 3219624.0,
+      "grad_norm": 1.41231107711792,
+      "learning_rate": 1.0370480142264374e-06,
+      "loss": 5.4226,
+      "mean_token_accuracy": 0.2979922544956207,
+      "num_tokens": 3627807.0,
       "step": 3450
     },
     {
       "epoch": 2.0,
-      "eval_entropy": 6.272298685416648,
-      "eval_loss": 6.12472677230835,
-      "eval_mean_token_accuracy": 0.21168697409091458,
-      "eval_model_preparation_time": 0.0036,
-      "eval_num_tokens": 3239438.0,
-      "eval_runtime": 76.2536,
-      "eval_samples_per_second": 5.692,
-      "eval_steps_per_second": 2.846,
+      "eval_entropy": 5.831721861790951,
+      "eval_loss": 5.656307220458984,
+      "eval_mean_token_accuracy": 0.2641724460685308,
+      "eval_model_preparation_time": 0.0047,
+      "eval_num_tokens": 3650214.0,
+      "eval_runtime": 79.7324,
+      "eval_samples_per_second": 5.443,
+      "eval_steps_per_second": 2.722,
       "step": 3474
     }
   ],
   "logging_steps": 50,
-  "max_steps": 8685,
+  "max_steps": 6948,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 5,
+  "num_train_epochs": 4,
   "save_steps": 500,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -741,7 +741,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 4.529454004325376e+16,
+  "total_flos": 5.014260864635904e+16,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null

checkpoint-3474/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:130d33149272782bd60306263c371036419926142b8999aad7806359168f8484
+oid sha256:8db5c304963110404ebb6947b83ba95bd9b8aad1f9b8b578cc33c46d601e13dc
 size 6225

checkpoint-5211/adapter_config.json

@@ -16,7 +16,7 @@
   "layers_pattern": null,
   "layers_to_transform": null,
   "loftq_config": {},
-  "lora_alpha": 16,
+  "lora_alpha": 32,
   "lora_bias": false,
   "lora_dropout": 0.1,
   "megatron_config": null,
@@ -25,12 +25,14 @@
   "peft_type": "LORA",
   "peft_version": "0.18.0",
   "qalora_group_size": 16,
-  "r": 8,
+  "r": 24,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
     "q_proj",
-    "v_proj"
+    "k_proj",
+    "v_proj",
+    "o_proj"
   ],
   "target_parameters": null,
   "task_type": "CAUSAL_LM",

checkpoint-5211/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0e6a7b22d63fd8741b839353cbaab150c0bd5f07d663ad8884bd3b4af58a9cce
-size 4374520
+oid sha256:96bed2a64089d15ba0d03e873c6ba43e222e9615622cb08853696f1bb3f72ed3
+size 26182176

checkpoint-5211/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0d7235486f7f068a0b9991bde7ca0b6a16106923b1cca53549a5bb621f15d218
-size 8783179
+oid sha256:bc97f69c9bd94b7be821d35593073dc08cf44ccce0203ce520c9a25dfcbc93d7
+size 52486155

checkpoint-5211/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:43cbafcbad7a00736ad4867a9fc18293a08b0b3d13acacb84d30cd8449539e81
+oid sha256:4bfc3867136ea1392d43912e26c993ff7e9d2c829e3cc938d41df7399c31116c
 size 14645

checkpoint-5211/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c82e157712778db9a1270de44d6dd5d35b469dbf5b63767059cabfb507d50c8a
+oid sha256:8a76316bf35b4ab1e089615992ceff4951bb9d24d95bfa6731e79f937bd9a30c
 size 1465

checkpoint-5211/trainer_state.json

@@ -1,6 +1,6 @@
 {
   "best_global_step": 5211,
-  "best_metric": 6.0980024337768555,
+  "best_metric": 5.628758430480957,
   "best_model_checkpoint": "./output/checkpoint-5211",
   "epoch": 3.0,
   "eval_steps": 500,
@@ -10,1086 +10,1086 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "entropy": 3.864118957519531,
+      "entropy": 3.6583470726013183,
       "epoch": 0.028785261945883708,
-      "grad_norm": 2.7545533180236816,
-      "learning_rate": 9.800000000000001e-06,
-      "loss": 15.2997,
-      "mean_token_accuracy": 0.10086015284061432,
-      "num_tokens": 47319.0,
+      "grad_norm": 3.3817152976989746,
+      "learning_rate": 4.9e-07,
+      "loss": 13.8754,
+      "mean_token_accuracy": 0.15036460414528846,
+      "num_tokens": 53093.0,
       "step": 50
     },
     {
-      "entropy": 4.047076859474182,
+      "entropy": 3.669608063697815,
       "epoch": 0.057570523891767415,
-      "grad_norm": 5.0328264236450195,
-      "learning_rate": 1.98e-05,
-      "loss": 15.3264,
-      "mean_token_accuracy": 0.09582207053899765,
-      "num_tokens": 96809.0,
+      "grad_norm": 3.2541544437408447,
+      "learning_rate": 9.9e-07,
+      "loss": 14.2282,
+      "mean_token_accuracy": 0.14137721598148345,
+      "num_tokens": 108334.0,
       "step": 100
     },
     {
-      "entropy": 4.7578076648712155,
+      "entropy": 3.569736371040344,
       "epoch": 0.08635578583765112,
-      "grad_norm": 38.50589370727539,
-      "learning_rate": 1.988584740827024e-05,
-      "loss": 13.0056,
-      "mean_token_accuracy": 0.126854517608881,
-      "num_tokens": 139962.0,
+      "grad_norm": 3.6797454357147217,
+      "learning_rate": 1.49e-06,
+      "loss": 13.0735,
+      "mean_token_accuracy": 0.17473630651831626,
+      "num_tokens": 157491.0,
       "step": 150
     },
     {
-      "entropy": 6.80673882484436,
+      "entropy": 3.7253233194351196,
       "epoch": 0.11514104778353483,
-      "grad_norm": 12.030129432678223,
-      "learning_rate": 1.97693651718113e-05,
-      "loss": 9.2822,
-      "mean_token_accuracy": 0.11084575355052947,
-      "num_tokens": 188029.0,
+      "grad_norm": 4.297911643981934,
+      "learning_rate": 1.99e-06,
+      "loss": 13.7392,
+      "mean_token_accuracy": 0.1473099772632122,
+      "num_tokens": 211394.0,
       "step": 200
     },
     {
-      "entropy": 7.177925786972046,
+      "entropy": 3.8280500602722167,
       "epoch": 0.14392630972941853,
-      "grad_norm": 4.852536201477051,
-      "learning_rate": 1.965288293535236e-05,
-      "loss": 7.6333,
-      "mean_token_accuracy": 0.12398939326405525,
-      "num_tokens": 234425.0,
+      "grad_norm": 4.405268669128418,
+      "learning_rate": 1.9854771784232364e-06,
+      "loss": 13.0797,
+      "mean_token_accuracy": 0.16704789966344832,
+      "num_tokens": 263685.0,
       "step": 250
     },
     {
-      "entropy": 7.080496473312378,
+      "entropy": 4.066333084106446,
       "epoch": 0.17271157167530224,
-      "grad_norm": 4.10841178894043,
-      "learning_rate": 1.9536400698893422e-05,
-      "loss": 7.1632,
-      "mean_token_accuracy": 0.13563686355948448,
-      "num_tokens": 278885.0,
+      "grad_norm": 4.757556438446045,
+      "learning_rate": 1.9706579727326615e-06,
+      "loss": 12.6321,
+      "mean_token_accuracy": 0.1691790708899498,
+      "num_tokens": 314059.0,
       "step": 300
     },
     {
-      "entropy": 6.931579580307007,
+      "entropy": 4.257266030311585,
       "epoch": 0.20149683362118595,
-      "grad_norm": 14.636048316955566,
-      "learning_rate": 1.941991846243448e-05,
-      "loss": 6.8213,
-      "mean_token_accuracy": 0.16459846690297128,
-      "num_tokens": 325491.0,
+      "grad_norm": 6.406249523162842,
+      "learning_rate": 1.955838767042086e-06,
+      "loss": 12.2253,
+      "mean_token_accuracy": 0.17223650276660918,
+      "num_tokens": 367038.0,
       "step": 350
     },
     {
-      "entropy": 6.853660764694214,
+      "entropy": 4.694105777740479,
       "epoch": 0.23028209556706966,
-      "grad_norm": 5.966708183288574,
-      "learning_rate": 1.930343622597554e-05,
-      "loss": 6.6625,
-      "mean_token_accuracy": 0.17670693069696428,
-      "num_tokens": 372913.0,
+      "grad_norm": 12.57987117767334,
+      "learning_rate": 1.9410195613515113e-06,
+      "loss": 11.9714,
+      "mean_token_accuracy": 0.15997304677963256,
+      "num_tokens": 420327.0,
       "step": 400
     },
     {
-      "entropy": 6.684267387390137,
+      "entropy": 5.205010280609131,
       "epoch": 0.25906735751295334,
-      "grad_norm": 4.031010627746582,
-      "learning_rate": 1.91869539895166e-05,
-      "loss": 6.4505,
-      "mean_token_accuracy": 0.1943434515595436,
-      "num_tokens": 419159.0,
+      "grad_norm": 15.570313453674316,
+      "learning_rate": 1.9262003556609364e-06,
+      "loss": 10.8173,
+      "mean_token_accuracy": 0.16447648257017136,
+      "num_tokens": 472429.0,
       "step": 450
     },
     {
-      "entropy": 6.679989137649536,
+      "entropy": 5.917805089950561,
       "epoch": 0.28785261945883706,
-      "grad_norm": 6.251070022583008,
-      "learning_rate": 1.907047175305766e-05,
-      "loss": 6.4314,
-      "mean_token_accuracy": 0.19514557600021362,
-      "num_tokens": 466994.0,
+      "grad_norm": 23.61503791809082,
+      "learning_rate": 1.9113811499703615e-06,
+      "loss": 9.3196,
+      "mean_token_accuracy": 0.16179455041885377,
+      "num_tokens": 526315.0,
       "step": 500
     },
     {
-      "entropy": 6.477229623794556,
+      "entropy": 6.380368332862854,
       "epoch": 0.31663788140472077,
-      "grad_norm": 3.8656675815582275,
-      "learning_rate": 1.895398951659872e-05,
-      "loss": 6.2139,
-      "mean_token_accuracy": 0.21764743447303772,
-      "num_tokens": 513308.0,
+      "grad_norm": 13.846810340881348,
+      "learning_rate": 1.8965619442797864e-06,
+      "loss": 7.9636,
+      "mean_token_accuracy": 0.16881170988082886,
+      "num_tokens": 578511.0,
       "step": 550
     },
     {
-      "entropy": 6.408129243850708,
+      "entropy": 6.507339992523193,
       "epoch": 0.3454231433506045,
-      "grad_norm": 8.688581466674805,
-      "learning_rate": 1.883750728013978e-05,
-      "loss": 6.1224,
-      "mean_token_accuracy": 0.23438037544488907,
-      "num_tokens": 559679.0,
+      "grad_norm": 4.569090366363525,
+      "learning_rate": 1.8817427385892115e-06,
+      "loss": 7.4171,
+      "mean_token_accuracy": 0.16941152423620223,
+      "num_tokens": 630937.0,
       "step": 600
     },
     {
-      "entropy": 6.128518767356873,
+      "entropy": 6.392864561080932,
       "epoch": 0.3742084052964882,
-      "grad_norm": 5.419503688812256,
-      "learning_rate": 1.872102504368084e-05,
-      "loss": 5.8692,
-      "mean_token_accuracy": 0.26634690463542937,
-      "num_tokens": 603140.0,
+      "grad_norm": 4.594696521759033,
+      "learning_rate": 1.8669235328986366e-06,
+      "loss": 6.9389,
+      "mean_token_accuracy": 0.1844496901333332,
+      "num_tokens": 680501.0,
       "step": 650
     },
     {
-      "entropy": 6.322700729370117,
+      "entropy": 6.6726202869415285,
       "epoch": 0.4029936672423719,
-      "grad_norm": 2.2213082313537598,
-      "learning_rate": 1.86045428072219e-05,
-      "loss": 6.0717,
-      "mean_token_accuracy": 0.24038562417030335,
-      "num_tokens": 650179.0,
+      "grad_norm": 4.768734931945801,
+      "learning_rate": 1.8521043272080617e-06,
+      "loss": 6.9818,
+      "mean_token_accuracy": 0.16990411713719367,
+      "num_tokens": 733231.0,
       "step": 700
     },
     {
-      "entropy": 6.236415157318115,
+      "entropy": 6.592793455123902,
       "epoch": 0.4317789291882556,
-      "grad_norm": 4.804980278015137,
-      "learning_rate": 1.848806057076296e-05,
-      "loss": 5.9986,
-      "mean_token_accuracy": 0.24596781462430953,
-      "num_tokens": 696220.0,
+      "grad_norm": 3.253056764602661,
+      "learning_rate": 1.8372851215174864e-06,
+      "loss": 6.7105,
+      "mean_token_accuracy": 0.18250102579593658,
+      "num_tokens": 785373.0,
       "step": 750
     },
     {
-      "entropy": 6.269758443832398,
+      "entropy": 6.683582029342651,
       "epoch": 0.4605641911341393,
-      "grad_norm": 2.2888853549957275,
-      "learning_rate": 1.837157833430402e-05,
-      "loss": 6.0385,
-      "mean_token_accuracy": 0.24074893474578857,
-      "num_tokens": 743909.0,
+      "grad_norm": 2.1871063709259033,
+      "learning_rate": 1.8224659158269115e-06,
+      "loss": 6.6685,
+      "mean_token_accuracy": 0.17129646152257919,
+      "num_tokens": 838646.0,
       "step": 800
     },
     {
-      "entropy": 6.270364007949829,
+      "entropy": 6.636875295639038,
       "epoch": 0.48934945308002303,
-      "grad_norm": 3.0903279781341553,
-      "learning_rate": 1.825509609784508e-05,
-      "loss": 6.0481,
-      "mean_token_accuracy": 0.23740622967481614,
-      "num_tokens": 792015.0,
+      "grad_norm": 3.2284677028656006,
+      "learning_rate": 1.8076467101363366e-06,
+      "loss": 6.53,
+      "mean_token_accuracy": 0.18053789794445038,
+      "num_tokens": 892380.0,
       "step": 850
     },
     {
-      "entropy": 6.3037636184692385,
+      "entropy": 6.610673260688782,
       "epoch": 0.5181347150259067,
-      "grad_norm": 3.969320058822632,
-      "learning_rate": 1.813861386138614e-05,
-      "loss": 6.0855,
-      "mean_token_accuracy": 0.2309597587585449,
-      "num_tokens": 841802.0,
+      "grad_norm": 2.2088730335235596,
+      "learning_rate": 1.7928275044457617e-06,
+      "loss": 6.4429,
+      "mean_token_accuracy": 0.18492739230394364,
+      "num_tokens": 947971.0,
       "step": 900
     },
     {
-      "entropy": 6.038041458129883,
+      "entropy": 6.242899022102356,
       "epoch": 0.5469199769717904,
-      "grad_norm": 2.2712185382843018,
-      "learning_rate": 1.80221316249272e-05,
-      "loss": 5.8285,
-      "mean_token_accuracy": 0.26099125802516937,
-      "num_tokens": 886492.0,
+      "grad_norm": 2.3000030517578125,
+      "learning_rate": 1.7780082987551866e-06,
+      "loss": 6.047,
+      "mean_token_accuracy": 0.2291259828209877,
+      "num_tokens": 998810.0,
       "step": 950
     },
     {
-      "entropy": 6.142958383560181,
+      "entropy": 6.311488924026489,
       "epoch": 0.5757052389176741,
-      "grad_norm": 1.2311755418777466,
-      "learning_rate": 1.790564938846826e-05,
-      "loss": 5.9357,
-      "mean_token_accuracy": 0.24810438305139543,
-      "num_tokens": 932807.0,
+      "grad_norm": 2.1333675384521484,
+      "learning_rate": 1.7631890930646115e-06,
+      "loss": 6.0919,
+      "mean_token_accuracy": 0.22644571751356124,
+      "num_tokens": 1050860.0,
       "step": 1000
     },
     {
-      "entropy": 6.199834351539612,
+      "entropy": 6.3254336166381835,
       "epoch": 0.6044905008635578,
-      "grad_norm": 2.2788379192352295,
-      "learning_rate": 1.7789167152009318e-05,
-      "loss": 5.9964,
-      "mean_token_accuracy": 0.23942562609910964,
-      "num_tokens": 980541.0,
+      "grad_norm": 2.0400779247283936,
+      "learning_rate": 1.7483698873740366e-06,
+      "loss": 6.094,
+      "mean_token_accuracy": 0.2222653564810753,
+      "num_tokens": 1104304.0,
       "step": 1050
     },
     {
-      "entropy": 5.961639919281006,
+      "entropy": 6.046922063827514,
       "epoch": 0.6332757628094415,
-      "grad_norm": 1.9077532291412354,
-      "learning_rate": 1.767268491555038e-05,
-      "loss": 5.7664,
-      "mean_token_accuracy": 0.26718012750148773,
-      "num_tokens": 1023882.0,
+      "grad_norm": 2.8049051761627197,
+      "learning_rate": 1.7335506816834617e-06,
+      "loss": 5.8011,
+      "mean_token_accuracy": 0.25127078920602797,
+      "num_tokens": 1153605.0,
       "step": 1100
     },
     {
-      "entropy": 5.889280087947846,
+      "entropy": 5.943600912094116,
       "epoch": 0.6620610247553252,
-      "grad_norm": 2.4254891872406006,
-      "learning_rate": 1.7556202679091442e-05,
-      "loss": 5.6952,
-      "mean_token_accuracy": 0.27529804170131683,
-      "num_tokens": 1068300.0,
+      "grad_norm": 4.063963890075684,
+      "learning_rate": 1.7187314759928866e-06,
+      "loss": 5.6855,
+      "mean_token_accuracy": 0.26265266716480257,
+      "num_tokens": 1204328.0,
       "step": 1150
     },
     {
-      "entropy": 6.085640063285828,
+      "entropy": 6.12883231639862,
       "epoch": 0.690846286701209,
-      "grad_norm": 2.35312557220459,
-      "learning_rate": 1.74397204426325e-05,
-      "loss": 5.8898,
-      "mean_token_accuracy": 0.25166562348604204,
-      "num_tokens": 1115425.0,
+      "grad_norm": 3.9440460205078125,
+      "learning_rate": 1.7039122703023117e-06,
+      "loss": 5.8578,
+      "mean_token_accuracy": 0.24439335912466048,
+      "num_tokens": 1257415.0,
       "step": 1200
     },
     {
-      "entropy": 6.146574058532715,
+      "entropy": 6.164987115859986,
       "epoch": 0.7196315486470927,
-      "grad_norm": 1.7730146646499634,
-      "learning_rate": 1.732323820617356e-05,
-      "loss": 5.9519,
-      "mean_token_accuracy": 0.24276195973157882,
-      "num_tokens": 1162319.0,
+      "grad_norm": 3.20070481300354,
+      "learning_rate": 1.6890930646117368e-06,
+      "loss": 5.8876,
+      "mean_token_accuracy": 0.24275501281023026,
+      "num_tokens": 1310049.0,
       "step": 1250
     },
     {
-      "entropy": 6.079372715950012,
+      "entropy": 6.080997190475464,
       "epoch": 0.7484168105929764,
-      "grad_norm": 1.7070863246917725,
-      "learning_rate": 1.720675596971462e-05,
-      "loss": 5.8922,
-      "mean_token_accuracy": 0.24961524546146394,
-      "num_tokens": 1208230.0,
+      "grad_norm": 2.8067362308502197,
+      "learning_rate": 1.6742738589211617e-06,
+      "loss": 5.8058,
+      "mean_token_accuracy": 0.25242207854986193,
+      "num_tokens": 1361794.0,
       "step": 1300
     },
     {
-      "entropy": 5.9683656406402585,
+      "entropy": 5.940848155021667,
       "epoch": 0.7772020725388601,
-      "grad_norm": 1.8790594339370728,
-      "learning_rate": 1.709027373325568e-05,
-      "loss": 5.7827,
-      "mean_token_accuracy": 0.2632122594118118,
-      "num_tokens": 1253074.0,
+      "grad_norm": 2.6375925540924072,
+      "learning_rate": 1.6594546532305868e-06,
+      "loss": 5.6718,
+      "mean_token_accuracy": 0.2665082859992981,
+      "num_tokens": 1412773.0,
       "step": 1350
     },
     {
-      "entropy": 6.107076721191406,
+      "entropy": 6.071129274368286,
       "epoch": 0.8059873344847438,
-      "grad_norm": 1.1745644807815552,
-      "learning_rate": 1.6973791496796742e-05,
-      "loss": 5.9211,
-      "mean_token_accuracy": 0.24564073830842972,
-      "num_tokens": 1300179.0,
+      "grad_norm": 3.951350212097168,
+      "learning_rate": 1.6446354475400117e-06,
+      "loss": 5.8012,
+      "mean_token_accuracy": 0.25434976994991304,
+      "num_tokens": 1465620.0,
       "step": 1400
     },
     {
-      "entropy": 6.141328382492065,
+      "entropy": 6.069429359436035,
       "epoch": 0.8347725964306275,
-      "grad_norm": 1.0346958637237549,
-      "learning_rate": 1.68573092603378e-05,
-      "loss": 5.9584,
-      "mean_token_accuracy": 0.23997059136629104,
-      "num_tokens": 1347539.0,
+      "grad_norm": 3.580608606338501,
+      "learning_rate": 1.6298162418494368e-06,
+      "loss": 5.8027,
+      "mean_token_accuracy": 0.25208072274923327,
+      "num_tokens": 1518899.0,
       "step": 1450
     },
     {
-      "entropy": 6.070010099411011,
+      "entropy": 6.005315380096436,
       "epoch": 0.8635578583765112,
-      "grad_norm": 1.6541163921356201,
-      "learning_rate": 1.674082702387886e-05,
-      "loss": 5.889,
-      "mean_token_accuracy": 0.24875166177749633,
-      "num_tokens": 1394157.0,
+      "grad_norm": 3.9580376148223877,
+      "learning_rate": 1.614997036158862e-06,
+      "loss": 5.7364,
+      "mean_token_accuracy": 0.25940640360116957,
+      "num_tokens": 1571304.0,
       "step": 1500
     },
     {
-      "entropy": 6.207450666427612,
+      "entropy": 6.0786464881896975,
       "epoch": 0.8923431203223949,
-      "grad_norm": 0.9742990732192993,
-      "learning_rate": 1.662434478741992e-05,
-      "loss": 6.0217,
-      "mean_token_accuracy": 0.23067249596118927,
-      "num_tokens": 1443892.0,
+      "grad_norm": 4.55721378326416,
+      "learning_rate": 1.6001778304682868e-06,
+      "loss": 5.8092,
+      "mean_token_accuracy": 0.2496869170665741,
+      "num_tokens": 1627369.0,
       "step": 1550
     },
     {
-      "entropy": 6.026197805404663,
+      "entropy": 5.939382014274597,
       "epoch": 0.9211283822682786,
-      "grad_norm": 1.4229531288146973,
-      "learning_rate": 1.650786255096098e-05,
-      "loss": 5.8455,
-      "mean_token_accuracy": 0.2537291014194489,
-      "num_tokens": 1491050.0,
+      "grad_norm": 2.330057144165039,
+      "learning_rate": 1.5853586247777117e-06,
+      "loss": 5.6604,
+      "mean_token_accuracy": 0.2686630353331566,
+      "num_tokens": 1680401.0,
       "step": 1600
     },
     {
-      "entropy": 6.210526428222656,
+      "entropy": 6.121775646209716,
       "epoch": 0.9499136442141624,
-      "grad_norm": 1.3555018901824951,
-      "learning_rate": 1.6391380314502038e-05,
-      "loss": 6.0279,
-      "mean_token_accuracy": 0.2308420208096504,
-      "num_tokens": 1540809.0,
+      "grad_norm": 2.9881200790405273,
+      "learning_rate": 1.5705394190871368e-06,
+      "loss": 5.8388,
+      "mean_token_accuracy": 0.2503683388233185,
+      "num_tokens": 1735745.0,
       "step": 1650
     },
     {
-      "entropy": 5.9872834014892575,
+      "entropy": 5.840040788650513,
       "epoch": 0.9786989061600461,
-      "grad_norm": 0.9893498420715332,
-      "learning_rate": 1.62748980780431e-05,
-      "loss": 5.8137,
-      "mean_token_accuracy": 0.2566875320672989,
-      "num_tokens": 1585876.0,
+      "grad_norm": 3.798994779586792,
+      "learning_rate": 1.555720213396562e-06,
+      "loss": 5.5635,
+      "mean_token_accuracy": 0.278279125392437,
+      "num_tokens": 1786896.0,
       "step": 1700
     },
     {
       "epoch": 1.0,
-      "eval_entropy": 6.322207130045386,
-      "eval_loss": 6.15173864364624,
-      "eval_mean_token_accuracy": 0.21116007946877985,
-      "eval_model_preparation_time": 0.0036,
-      "eval_num_tokens": 1619719.0,
-      "eval_runtime": 76.1297,
-      "eval_samples_per_second": 5.701,
-      "eval_steps_per_second": 2.85,
+      "eval_entropy": 6.139133475343203,
+      "eval_loss": 5.861395835876465,
+      "eval_mean_token_accuracy": 0.2402858340657801,
+      "eval_model_preparation_time": 0.0047,
+      "eval_num_tokens": 1825107.0,
+      "eval_runtime": 79.3994,
+      "eval_samples_per_second": 5.466,
+      "eval_steps_per_second": 2.733,
       "step": 1737
     },
     {
-      "entropy": 6.038531675338745,
+      "entropy": 5.8970259666442875,
       "epoch": 1.0074841681059297,
-      "grad_norm": 0.8715208172798157,
-      "learning_rate": 1.615841584158416e-05,
-      "loss": 5.8628,
-      "mean_token_accuracy": 0.2510762655735016,
-      "num_tokens": 1632015.0,
+      "grad_norm": 2.6411802768707275,
+      "learning_rate": 1.540901007705987e-06,
+      "loss": 5.614,
+      "mean_token_accuracy": 0.273006406724453,
+      "num_tokens": 1838864.0,
       "step": 1750
     },
     {
-      "entropy": 6.164030771255494,
+      "entropy": 6.0111794090271,
       "epoch": 1.0362694300518134,
-      "grad_norm": 0.7344900965690613,
-      "learning_rate": 1.604193360512522e-05,
-      "loss": 5.9856,
-      "mean_token_accuracy": 0.2351543301343918,
-      "num_tokens": 1681154.0,
+      "grad_norm": 3.6491827964782715,
+      "learning_rate": 1.526081802015412e-06,
+      "loss": 5.7323,
+      "mean_token_accuracy": 0.26104256987571717,
+      "num_tokens": 1893816.0,
       "step": 1800
     },
     {
-      "entropy": 6.0731862354278565,
+      "entropy": 5.902219276428223,
       "epoch": 1.065054691997697,
-      "grad_norm": 1.0801328420639038,
-      "learning_rate": 1.592545136866628e-05,
-      "loss": 5.8976,
-      "mean_token_accuracy": 0.24701615989208223,
-      "num_tokens": 1728110.0,
+      "grad_norm": 2.593249559402466,
+      "learning_rate": 1.5112625963248368e-06,
+      "loss": 5.6187,
+      "mean_token_accuracy": 0.2746362566947937,
+      "num_tokens": 1946532.0,
       "step": 1850
     },
     {
-      "entropy": 6.079212121963501,
+      "entropy": 5.874705944061279,
       "epoch": 1.0938399539435808,
-      "grad_norm": 0.7876909375190735,
-      "learning_rate": 1.5808969132207338e-05,
-      "loss": 5.9056,
-      "mean_token_accuracy": 0.24457543224096298,
-      "num_tokens": 1775703.0,
+      "grad_norm": 2.554327964782715,
+      "learning_rate": 1.496443390634262e-06,
+      "loss": 5.6021,
+      "mean_token_accuracy": 0.2795292744040489,
+      "num_tokens": 2000184.0,
       "step": 1900
     },
     {
-      "entropy": 6.062467746734619,
+      "entropy": 5.850096368789673,
       "epoch": 1.1226252158894645,
-      "grad_norm": 0.5999078750610352,
-      "learning_rate": 1.56924868957484e-05,
-      "loss": 5.8899,
-      "mean_token_accuracy": 0.2469428673386574,
-      "num_tokens": 1821980.0,
+      "grad_norm": 3.6060993671417236,
+      "learning_rate": 1.481624184943687e-06,
+      "loss": 5.576,
+      "mean_token_accuracy": 0.28532547056674956,
+      "num_tokens": 2052250.0,
       "step": 1950
     },
     {
-      "entropy": 6.031774473190308,
+      "entropy": 5.802229671478272,
       "epoch": 1.1514104778353482,
-      "grad_norm": 1.6313235759735107,
-      "learning_rate": 1.557600465928946e-05,
-      "loss": 5.8593,
-      "mean_token_accuracy": 0.250918984413147,
-      "num_tokens": 1867547.0,
+      "grad_norm": 3.0913314819335938,
+      "learning_rate": 1.466804979253112e-06,
+      "loss": 5.53,
+      "mean_token_accuracy": 0.2916027933359146,
+      "num_tokens": 2103531.0,
       "step": 2000
     },
     {
-      "entropy": 6.122789564132691,
+      "entropy": 5.875646467208862,
       "epoch": 1.180195739781232,
-      "grad_norm": 2.562373161315918,
-      "learning_rate": 1.545952242283052e-05,
-      "loss": 5.9502,
-      "mean_token_accuracy": 0.23938885867595672,
-      "num_tokens": 1915411.0,
+      "grad_norm": 4.777045726776123,
+      "learning_rate": 1.451985773562537e-06,
+      "loss": 5.6146,
+      "mean_token_accuracy": 0.28063644528388976,
+      "num_tokens": 2157098.0,
       "step": 2050
     },
     {
-      "entropy": 6.067130417823791,
+      "entropy": 5.786596937179565,
       "epoch": 1.2089810017271156,
-      "grad_norm": 0.9762872457504272,
-      "learning_rate": 1.534304018637158e-05,
-      "loss": 5.8956,
-      "mean_token_accuracy": 0.2454381173849106,
-      "num_tokens": 1964009.0,
+      "grad_norm": 4.207762718200684,
+      "learning_rate": 1.437166567871962e-06,
+      "loss": 5.5417,
+      "mean_token_accuracy": 0.2870470091700554,
+      "num_tokens": 2211827.0,
       "step": 2100
     },
     {
-      "entropy": 5.9613511180877685,
+      "entropy": 5.672234449386597,
       "epoch": 1.2377662636729994,
-      "grad_norm": 0.8701547384262085,
-      "learning_rate": 1.5226557949912639e-05,
-      "loss": 5.7907,
-      "mean_token_accuracy": 0.25976367652416227,
-      "num_tokens": 2008595.0,
+      "grad_norm": 2.2771811485290527,
+      "learning_rate": 1.422347362181387e-06,
+      "loss": 5.4285,
+      "mean_token_accuracy": 0.30194485366344453,
+      "num_tokens": 2262174.0,
       "step": 2150
     },
     {
-      "entropy": 6.13505428314209,
+      "entropy": 5.862573285102844,
       "epoch": 1.266551525618883,
-      "grad_norm": 0.8511647582054138,
-      "learning_rate": 1.51100757134537e-05,
-      "loss": 5.9619,
-      "mean_token_accuracy": 0.23760781466960906,
-      "num_tokens": 2057229.0,
+      "grad_norm": 3.3273422718048096,
+      "learning_rate": 1.4075281564908121e-06,
+      "loss": 5.6169,
+      "mean_token_accuracy": 0.278145115673542,
+      "num_tokens": 2316440.0,
       "step": 2200
     },
     {
-      "entropy": 6.025254983901977,
+      "entropy": 5.734760231971741,
       "epoch": 1.2953367875647668,
-      "grad_norm": 0.7627406120300293,
-      "learning_rate": 1.4993593476994758e-05,
-      "loss": 5.8546,
-      "mean_token_accuracy": 0.2508662334084511,
-      "num_tokens": 2103631.0,
+      "grad_norm": 3.7049715518951416,
+      "learning_rate": 1.392708950800237e-06,
+      "loss": 5.493,
+      "mean_token_accuracy": 0.2941485676169395,
+      "num_tokens": 2368468.0,
       "step": 2250
     },
     {
-      "entropy": 5.981974196434021,
+      "entropy": 5.665819988250733,
       "epoch": 1.3241220495106505,
-      "grad_norm": 1.6922173500061035,
-      "learning_rate": 1.4877111240535819e-05,
-      "loss": 5.8119,
-      "mean_token_accuracy": 0.256170334815979,
-      "num_tokens": 2150369.0,
+      "grad_norm": 3.572636604309082,
+      "learning_rate": 1.3778897451096621e-06,
+      "loss": 5.4352,
+      "mean_token_accuracy": 0.3003745040297508,
+      "num_tokens": 2421180.0,
       "step": 2300
     },
     {
-      "entropy": 6.19903904914856,
+      "entropy": 5.890115032196045,
       "epoch": 1.3529073114565342,
-      "grad_norm": 0.40436601638793945,
-      "learning_rate": 1.4760629004076878e-05,
-      "loss": 6.0244,
-      "mean_token_accuracy": 0.22900927513837815,
-      "num_tokens": 2199724.0,
+      "grad_norm": 2.738203525543213,
+      "learning_rate": 1.3630705394190872e-06,
+      "loss": 5.6555,
+      "mean_token_accuracy": 0.2737997192144394,
+      "num_tokens": 2476255.0,
       "step": 2350
     },
     {
-      "entropy": 5.986697297096253,
+      "entropy": 5.66056040763855,
       "epoch": 1.381692573402418,
-      "grad_norm": 0.8481882214546204,
-      "learning_rate": 1.464414676761794e-05,
-      "loss": 5.8195,
-      "mean_token_accuracy": 0.2552035376429558,
-      "num_tokens": 2245341.0,
+      "grad_norm": 3.1416995525360107,
+      "learning_rate": 1.3482513337285121e-06,
+      "loss": 5.4302,
+      "mean_token_accuracy": 0.3000989046692848,
+      "num_tokens": 2527674.0,
       "step": 2400
     },
     {
-      "entropy": 6.1886044692993165,
+      "entropy": 5.861240615844727,
       "epoch": 1.4104778353483016,
-      "grad_norm": 0.7911505103111267,
-      "learning_rate": 1.4527664531159e-05,
-      "loss": 6.0148,
-      "mean_token_accuracy": 0.23026730984449387,
-      "num_tokens": 2294726.0,
+      "grad_norm": 2.7569284439086914,
+      "learning_rate": 1.333432128037937e-06,
+      "loss": 5.6304,
+      "mean_token_accuracy": 0.27707513481378554,
+      "num_tokens": 2582909.0,
       "step": 2450
     },
     {
-      "entropy": 5.974867792129516,
+      "entropy": 5.627686910629272,
       "epoch": 1.4392630972941853,
-      "grad_norm": 1.640499234199524,
-      "learning_rate": 1.441118229470006e-05,
-      "loss": 5.8111,
-      "mean_token_accuracy": 0.2554209426045418,
-      "num_tokens": 2342251.0,
+      "grad_norm": 1.7750262022018433,
+      "learning_rate": 1.3186129223473621e-06,
+      "loss": 5.4058,
+      "mean_token_accuracy": 0.3019809901714325,
+      "num_tokens": 2636579.0,
       "step": 2500
     },
     {
-      "entropy": 5.967635660171509,
+      "entropy": 5.607026796340943,
       "epoch": 1.468048359240069,
-      "grad_norm": 0.8022929430007935,
-      "learning_rate": 1.429470005824112e-05,
-      "loss": 5.8015,
-      "mean_token_accuracy": 0.2569852137565613,
-      "num_tokens": 2387469.0,
+      "grad_norm": 3.1005160808563232,
+      "learning_rate": 1.3037937166567872e-06,
+      "loss": 5.3836,
+      "mean_token_accuracy": 0.30584611505270004,
+      "num_tokens": 2687698.0,
       "step": 2550
     },
     {
-      "entropy": 6.047262029647827,
+      "entropy": 5.6909641885757445,
       "epoch": 1.4968336211859528,
-      "grad_norm": 0.9270678758621216,
-      "learning_rate": 1.417821782178218e-05,
-      "loss": 5.8782,
-      "mean_token_accuracy": 0.2467849862575531,
-      "num_tokens": 2434128.0,
+      "grad_norm": 1.6848654747009277,
+      "learning_rate": 1.2889745109662123e-06,
+      "loss": 5.4653,
+      "mean_token_accuracy": 0.296178964972496,
+      "num_tokens": 2740214.0,
       "step": 2600
     },
     {
-      "entropy": 6.00601068019867,
+      "entropy": 5.619450302124023,
       "epoch": 1.5256188831318365,
-      "grad_norm": 1.5378597974777222,
-      "learning_rate": 1.406173558532324e-05,
-      "loss": 5.839,
-      "mean_token_accuracy": 0.25216978013515473,
-      "num_tokens": 2480366.0,
+      "grad_norm": 2.469539165496826,
+      "learning_rate": 1.274155305275637e-06,
+      "loss": 5.4022,
+      "mean_token_accuracy": 0.3039679077267647,
+      "num_tokens": 2792574.0,
       "step": 2650
     },
     {
-      "entropy": 5.988714299201965,
+      "entropy": 5.61073097705841,
       "epoch": 1.5544041450777202,
-      "grad_norm": 0.819143533706665,
-      "learning_rate": 1.3945253348864299e-05,
-      "loss": 5.82,
-      "mean_token_accuracy": 0.254311783015728,
-      "num_tokens": 2527357.0,
+      "grad_norm": 2.367810010910034,
+      "learning_rate": 1.259336099585062e-06,
+      "loss": 5.3956,
+      "mean_token_accuracy": 0.3051413372159004,
+      "num_tokens": 2845597.0,
       "step": 2700
     },
     {
-      "entropy": 5.960293846130371,
+      "entropy": 5.5791136837005615,
       "epoch": 1.583189407023604,
-      "grad_norm": 0.8920449614524841,
-      "learning_rate": 1.382877111240536e-05,
-      "loss": 5.7946,
-      "mean_token_accuracy": 0.25750755161046984,
-      "num_tokens": 2574470.0,
+      "grad_norm": 2.3874764442443848,
+      "learning_rate": 1.2445168938944872e-06,
+      "loss": 5.3676,
+      "mean_token_accuracy": 0.3068238252401352,
+      "num_tokens": 2898683.0,
       "step": 2750
     },
     {
-      "entropy": 6.1214879322052,
+      "entropy": 5.735381307601929,
       "epoch": 1.6119746689694876,
-      "grad_norm": 0.5333890914916992,
-      "learning_rate": 1.371228887594642e-05,
-      "loss": 5.9513,
-      "mean_token_accuracy": 0.2377367687225342,
-      "num_tokens": 2622280.0,
+      "grad_norm": 2.2097349166870117,
+      "learning_rate": 1.2296976882039123e-06,
+      "loss": 5.5239,
+      "mean_token_accuracy": 0.28974882304668426,
+      "num_tokens": 2952290.0,
       "step": 2800
     },
     {
-      "entropy": 5.951769871711731,
+      "entropy": 5.55252691745758,
       "epoch": 1.6407599309153713,
-      "grad_norm": 0.5994665026664734,
-      "learning_rate": 1.3595806639487479e-05,
-      "loss": 5.7861,
-      "mean_token_accuracy": 0.25854207515716554,
-      "num_tokens": 2668624.0,
+      "grad_norm": 1.694831132888794,
+      "learning_rate": 1.2148784825133372e-06,
+      "loss": 5.351,
+      "mean_token_accuracy": 0.3091904193162918,
+      "num_tokens": 3004556.0,
       "step": 2850
     },
     {
-      "entropy": 5.927765312194825,
+      "entropy": 5.508773093223572,
       "epoch": 1.669545192861255,
-      "grad_norm": 0.4460087716579437,
-      "learning_rate": 1.347932440302854e-05,
-      "loss": 5.7661,
-      "mean_token_accuracy": 0.25973255425691605,
-      "num_tokens": 2714388.0,
+      "grad_norm": 1.8229279518127441,
+      "learning_rate": 1.200059276822762e-06,
+      "loss": 5.3164,
+      "mean_token_accuracy": 0.31158645361661913,
+      "num_tokens": 3056448.0,
       "step": 2900
     },
     {
-      "entropy": 6.097678365707398,
+      "entropy": 5.676794271469117,
       "epoch": 1.6983304548071387,
-      "grad_norm": 0.7125752568244934,
-      "learning_rate": 1.3362842166569598e-05,
-      "loss": 5.9284,
-      "mean_token_accuracy": 0.23995368272066117,
-      "num_tokens": 2761465.0,
+      "grad_norm": 1.7196234464645386,
+      "learning_rate": 1.1852400711321872e-06,
+      "loss": 5.4776,
+      "mean_token_accuracy": 0.2929128894209862,
+      "num_tokens": 3109539.0,
       "step": 2950
     },
     {
-      "entropy": 5.986212658882141,
+      "entropy": 5.551529383659362,
       "epoch": 1.7271157167530224,
-      "grad_norm": 1.5405049324035645,
-      "learning_rate": 1.3246359930110659e-05,
-      "loss": 5.8194,
-      "mean_token_accuracy": 0.25333445996046067,
-      "num_tokens": 2808066.0,
+      "grad_norm": 3.117525577545166,
+      "learning_rate": 1.1704208654416123e-06,
+      "loss": 5.3561,
+      "mean_token_accuracy": 0.30634030640125276,
+      "num_tokens": 3162421.0,
       "step": 3000
     },
     {
-      "entropy": 5.7968806195259095,
+      "entropy": 5.379635264873505,
       "epoch": 1.7559009786989062,
-      "grad_norm": 0.4532749652862549,
-      "learning_rate": 1.312987769365172e-05,
-      "loss": 5.6344,
-      "mean_token_accuracy": 0.2782411390542984,
-      "num_tokens": 2851822.0,
+      "grad_norm": 1.876755714416504,
+      "learning_rate": 1.1556016597510372e-06,
+      "loss": 5.1868,
+      "mean_token_accuracy": 0.32913618892431257,
+      "num_tokens": 3212079.0,
       "step": 3050
     },
     {
-      "entropy": 5.973708114624023,
+      "entropy": 5.538804936408996,
       "epoch": 1.7846862406447899,
-      "grad_norm": 1.4795438051223755,
-      "learning_rate": 1.3013395457192778e-05,
-      "loss": 5.8104,
-      "mean_token_accuracy": 0.25441971331834795,
-      "num_tokens": 2897737.0,
+      "grad_norm": 1.8670976161956787,
+      "learning_rate": 1.1407824540604623e-06,
+      "loss": 5.3494,
+      "mean_token_accuracy": 0.30661171555519107,
+      "num_tokens": 3264089.0,
       "step": 3100
     },
     {
-      "entropy": 5.70733567237854,
+      "entropy": 5.258263626098633,
       "epoch": 1.8134715025906736,
-      "grad_norm": 0.6216577887535095,
-      "learning_rate": 1.2896913220733839e-05,
-      "loss": 5.5523,
-      "mean_token_accuracy": 0.28787180870771406,
-      "num_tokens": 2939511.0,
+      "grad_norm": 2.748718023300171,
+      "learning_rate": 1.1259632483698874e-06,
+      "loss": 5.08,
+      "mean_token_accuracy": 0.3413010013103485,
+      "num_tokens": 3311881.0,
       "step": 3150
     },
     {
-      "entropy": 5.96826630115509,
+      "entropy": 5.54539008140564,
       "epoch": 1.8422567645365573,
-      "grad_norm": 0.9246350526809692,
-      "learning_rate": 1.2780430984274898e-05,
-      "loss": 5.8057,
-      "mean_token_accuracy": 0.25464902341365814,
-      "num_tokens": 2986368.0,
+      "grad_norm": 1.8556406497955322,
+      "learning_rate": 1.1111440426793123e-06,
+      "loss": 5.3614,
+      "mean_token_accuracy": 0.30550685405731204,
+      "num_tokens": 3364861.0,
       "step": 3200
     },
     {
-      "entropy": 5.950662693977356,
+      "entropy": 5.5433073282241825,
       "epoch": 1.871042026482441,
-      "grad_norm": 0.8141199946403503,
-      "learning_rate": 1.266394874781596e-05,
-      "loss": 5.7886,
-      "mean_token_accuracy": 0.25830793648958206,
-      "num_tokens": 3031770.0,
+      "grad_norm": 1.8386749029159546,
+      "learning_rate": 1.0963248369887374e-06,
+      "loss": 5.3543,
+      "mean_token_accuracy": 0.30875524014234546,
+      "num_tokens": 3415911.0,
       "step": 3250
     },
     {
-      "entropy": 6.00512773513794,
+      "entropy": 5.5769769477844235,
       "epoch": 1.8998272884283247,
-      "grad_norm": 0.4913998246192932,
-      "learning_rate": 1.2547466511357018e-05,
-      "loss": 5.838,
-      "mean_token_accuracy": 0.2512077575922012,
-      "num_tokens": 3078322.0,
+      "grad_norm": 1.922486662864685,
+      "learning_rate": 1.0815056312981623e-06,
+      "loss": 5.3834,
+      "mean_token_accuracy": 0.3035113242268562,
+      "num_tokens": 3468338.0,
       "step": 3300
     },
     {
-      "entropy": 6.090880632400513,
+      "entropy": 5.640013842582703,
       "epoch": 1.9286125503742084,
-      "grad_norm": 0.9893012046813965,
-      "learning_rate": 1.243098427489808e-05,
-      "loss": 5.9264,
-      "mean_token_accuracy": 0.2391783133149147,
-      "num_tokens": 3125572.0,
+      "grad_norm": 2.179500102996826,
+      "learning_rate": 1.0666864256075874e-06,
+      "loss": 5.4574,
+      "mean_token_accuracy": 0.2947095710039139,
+      "num_tokens": 3521693.0,
       "step": 3350
     },
     {
-      "entropy": 5.949693293571472,
+      "entropy": 5.506910061836242,
       "epoch": 1.9573978123200921,
-      "grad_norm": 0.5794200301170349,
-      "learning_rate": 1.231450203843914e-05,
-      "loss": 5.7861,
-      "mean_token_accuracy": 0.2568664598464966,
-      "num_tokens": 3171974.0,
+      "grad_norm": 1.4014379978179932,
+      "learning_rate": 1.0518672199170125e-06,
+      "loss": 5.3234,
+      "mean_token_accuracy": 0.3096472260355949,
+      "num_tokens": 3574206.0,
       "step": 3400
     },
     {
-      "entropy": 6.03591317653656,
+      "entropy": 5.607311015129089,
       "epoch": 1.9861830742659758,
-      "grad_norm": 0.8525373339653015,
-      "learning_rate": 1.21980198019802e-05,
-      "loss": 5.8741,
-      "mean_token_accuracy": 0.24642003327608109,
-      "num_tokens": 3219624.0,
+      "grad_norm": 1.41231107711792,
+      "learning_rate": 1.0370480142264374e-06,
+      "loss": 5.4226,
+      "mean_token_accuracy": 0.2979922544956207,
+      "num_tokens": 3627807.0,
       "step": 3450
     },
     {
       "epoch": 2.0,
-      "eval_entropy": 6.272298685416648,
-      "eval_loss": 6.12472677230835,
-      "eval_mean_token_accuracy": 0.21168697409091458,
-      "eval_model_preparation_time": 0.0036,
-      "eval_num_tokens": 3239438.0,
-      "eval_runtime": 76.2536,
-      "eval_samples_per_second": 5.692,
-      "eval_steps_per_second": 2.846,
+      "eval_entropy": 5.831721861790951,
+      "eval_loss": 5.656307220458984,
+      "eval_mean_token_accuracy": 0.2641724460685308,
+      "eval_model_preparation_time": 0.0047,
+      "eval_num_tokens": 3650214.0,
+      "eval_runtime": 79.7324,
+      "eval_samples_per_second": 5.443,
+      "eval_steps_per_second": 2.722,
       "step": 3474
     },
     {
-      "entropy": 5.914763498306274,
+      "entropy": 5.477711625099182,
       "epoch": 2.0149683362118593,
-      "grad_norm": 0.5479806661605835,
-      "learning_rate": 1.208153756552126e-05,
-      "loss": 5.7559,
-      "mean_token_accuracy": 0.2624077323079109,
-      "num_tokens": 3263994.0,
+      "grad_norm": 3.0133137702941895,
+      "learning_rate": 1.0222288085358623e-06,
+      "loss": 5.2957,
+      "mean_token_accuracy": 0.31543311327695844,
+      "num_tokens": 3677883.0,
       "step": 3500
     },
     {
-      "entropy": 6.033470869064331,
+      "entropy": 5.599187393188476,
       "epoch": 2.043753598157743,
-      "grad_norm": 1.7186369895935059,
-      "learning_rate": 1.1965055329062319e-05,
-      "loss": 5.8677,
-      "mean_token_accuracy": 0.24745646148920059,
-      "num_tokens": 3311182.0,
+      "grad_norm": 1.885867714881897,
+      "learning_rate": 1.0074096028452874e-06,
+      "loss": 5.4142,
+      "mean_token_accuracy": 0.3004470923542976,
+      "num_tokens": 3730991.0,
       "step": 3550
     },
     {
-      "entropy": 5.962404427528381,
+      "entropy": 5.526448183059692,
       "epoch": 2.0725388601036268,
-      "grad_norm": 0.9068580269813538,
-      "learning_rate": 1.184857309260338e-05,
-      "loss": 5.8038,
-      "mean_token_accuracy": 0.25500513821840287,
-      "num_tokens": 3358036.0,
+      "grad_norm": 4.50788688659668,
+      "learning_rate": 9.925903971547125e-07,
+      "loss": 5.3517,
+      "mean_token_accuracy": 0.3069574165344238,
+      "num_tokens": 3783795.0,
       "step": 3600
     },
     {
-      "entropy": 5.995727968215943,
+      "entropy": 5.560557870864868,
       "epoch": 2.1013241220495105,
-      "grad_norm": 2.044490337371826,
-      "learning_rate": 1.1732090856144438e-05,
-      "loss": 5.8333,
-      "mean_token_accuracy": 0.2514388278126717,
-      "num_tokens": 3404058.0,
+      "grad_norm": 1.927862524986267,
+      "learning_rate": 9.777711914641374e-07,
+      "loss": 5.3815,
+      "mean_token_accuracy": 0.3045575937628746,
+      "num_tokens": 3835526.0,
       "step": 3650
     },
     {
-      "entropy": 5.981345901489258,
+      "entropy": 5.528058257102966,
       "epoch": 2.130109383995394,
-      "grad_norm": 0.5262818336486816,
-      "learning_rate": 1.1615608619685499e-05,
-      "loss": 5.8205,
-      "mean_token_accuracy": 0.2523340278863907,
-      "num_tokens": 3449834.0,
+      "grad_norm": 2.164687156677246,
+      "learning_rate": 9.629519857735625e-07,
+      "loss": 5.3501,
+      "mean_token_accuracy": 0.3071546205878258,
+      "num_tokens": 3887175.0,
       "step": 3700
     },
     {
-      "entropy": 5.848710675239563,
+      "entropy": 5.397617678642273,
       "epoch": 2.158894645941278,
-      "grad_norm": 0.726718544960022,
-      "learning_rate": 1.149912638322656e-05,
-      "loss": 5.6891,
-      "mean_token_accuracy": 0.2697497832775116,
-      "num_tokens": 3494740.0,
+      "grad_norm": 2.3098385334014893,
+      "learning_rate": 9.481327800829875e-07,
+      "loss": 5.2244,
+      "mean_token_accuracy": 0.3226669803261757,
+      "num_tokens": 3938003.0,
       "step": 3750
     },
     {
-      "entropy": 5.964878315925598,
+      "entropy": 5.529960298538208,
       "epoch": 2.1876799078871616,
-      "grad_norm": 0.6147393584251404,
-      "learning_rate": 1.1382644146767618e-05,
-      "loss": 5.8029,
-      "mean_token_accuracy": 0.2553535890579224,
-      "num_tokens": 3541342.0,
+      "grad_norm": 1.8144755363464355,
+      "learning_rate": 9.333135743924125e-07,
+      "loss": 5.3572,
+      "mean_token_accuracy": 0.306032218337059,
+      "num_tokens": 3990451.0,
       "step": 3800
     },
     {
-      "entropy": 6.045858116149902,
+      "entropy": 5.597109637260437,
       "epoch": 2.2164651698330453,
-      "grad_norm": 0.8283621072769165,
-      "learning_rate": 1.1266161910308679e-05,
-      "loss": 5.8802,
-      "mean_token_accuracy": 0.24544916599988936,
-      "num_tokens": 3588995.0,
+      "grad_norm": 2.7306935787200928,
+      "learning_rate": 9.184943687018375e-07,
+      "loss": 5.4162,
+      "mean_token_accuracy": 0.2985941395163536,
+      "num_tokens": 4044048.0,
       "step": 3850
     },
     {
-      "entropy": 5.909895505905151,
+      "entropy": 5.448684883117676,
       "epoch": 2.245250431778929,
-      "grad_norm": 0.9912867546081543,
-      "learning_rate": 1.1149679673849738e-05,
-      "loss": 5.7481,
-      "mean_token_accuracy": 0.2620398569107056,
-      "num_tokens": 3634252.0,
+      "grad_norm": 1.8199880123138428,
+      "learning_rate": 9.036751630112626e-07,
+      "loss": 5.2775,
+      "mean_token_accuracy": 0.31548845052719116,
+      "num_tokens": 4095276.0,
       "step": 3900
     },
     {
-      "entropy": 5.9534005498886104,
+      "entropy": 5.5008597612380985,
       "epoch": 2.2740356937248127,
-      "grad_norm": 1.2012401819229126,
-      "learning_rate": 1.1033197437390799e-05,
-      "loss": 5.788,
-      "mean_token_accuracy": 0.25642816990613937,
-      "num_tokens": 3681197.0,
+      "grad_norm": 1.755323052406311,
+      "learning_rate": 8.888559573206875e-07,
+      "loss": 5.3274,
+      "mean_token_accuracy": 0.309090721309185,
+      "num_tokens": 4148172.0,
       "step": 3950
     },
     {
-      "entropy": 6.155718851089477,
+      "entropy": 5.7040300464630125,
       "epoch": 2.3028209556706964,
-      "grad_norm": 1.4272509813308716,
-      "learning_rate": 1.0916715200931857e-05,
-      "loss": 5.9842,
-      "mean_token_accuracy": 0.23176315426826477,
-      "num_tokens": 3729955.0,
+      "grad_norm": 2.3154356479644775,
+      "learning_rate": 8.740367516301126e-07,
+      "loss": 5.5239,
+      "mean_token_accuracy": 0.28589318484067916,
+      "num_tokens": 4202733.0,
       "step": 4000
     },
     {
-      "entropy": 6.004842009544372,
+      "entropy": 5.549855670928955,
       "epoch": 2.33160621761658,
-      "grad_norm": 1.1919596195220947,
-      "learning_rate": 1.0800232964472918e-05,
-      "loss": 5.8332,
-      "mean_token_accuracy": 0.25039500594139097,
-      "num_tokens": 3777043.0,
+      "grad_norm": 1.9549669027328491,
+      "learning_rate": 8.592175459395375e-07,
+      "loss": 5.3755,
+      "mean_token_accuracy": 0.3029727828502655,
+      "num_tokens": 4255738.0,
       "step": 4050
     },
     {
-      "entropy": 6.045269584655761,
+      "entropy": 5.579690465927124,
       "epoch": 2.360391479562464,
-      "grad_norm": 0.6200748085975647,
-      "learning_rate": 1.068375072801398e-05,
-      "loss": 5.8641,
-      "mean_token_accuracy": 0.2466951721906662,
-      "num_tokens": 3824067.0,
+      "grad_norm": 1.7018866539001465,
+      "learning_rate": 8.443983402489626e-07,
+      "loss": 5.4036,
+      "mean_token_accuracy": 0.3001995691657066,
+      "num_tokens": 4308638.0,
       "step": 4100
     },
     {
-      "entropy": 6.105137758255005,
+      "entropy": 5.646504878997803,
       "epoch": 2.3891767415083476,
-      "grad_norm": 1.0185531377792358,
-      "learning_rate": 1.0567268491555038e-05,
-      "loss": 5.9181,
-      "mean_token_accuracy": 0.24000227689743042,
-      "num_tokens": 3872769.0,
+      "grad_norm": 1.4139262437820435,
+      "learning_rate": 8.295791345583877e-07,
+      "loss": 5.4733,
+      "mean_token_accuracy": 0.2912476986646652,
+      "num_tokens": 4363170.0,
       "step": 4150
     },
     {
-      "entropy": 6.013391451835632,
+      "entropy": 5.554990992546082,
       "epoch": 2.4179620034542313,
-      "grad_norm": 0.6188511848449707,
-      "learning_rate": 1.04507862550961e-05,
-      "loss": 5.8286,
-      "mean_token_accuracy": 0.25189226895570754,
-      "num_tokens": 3919379.0,
+      "grad_norm": 1.6886577606201172,
+      "learning_rate": 8.147599288678126e-07,
+      "loss": 5.3842,
+      "mean_token_accuracy": 0.302762059867382,
+      "num_tokens": 4415607.0,
       "step": 4200
     },
     {
-      "entropy": 5.972923498153687,
+      "entropy": 5.513420124053955,
       "epoch": 2.446747265400115,
-      "grad_norm": 0.7165982127189636,
-      "learning_rate": 1.0334304018637157e-05,
-      "loss": 5.7908,
-      "mean_token_accuracy": 0.2567197346687317,
-      "num_tokens": 3965593.0,
+      "grad_norm": 1.3537819385528564,
+      "learning_rate": 7.999407231772377e-07,
+      "loss": 5.3408,
+      "mean_token_accuracy": 0.30764526218175886,
+      "num_tokens": 4467608.0,
       "step": 4250
     },
     {
-      "entropy": 6.0378124713897705,
+      "entropy": 5.561378569602966,
       "epoch": 2.4755325273459987,
-      "grad_norm": 0.5278330445289612,
-      "learning_rate": 1.021782178217822e-05,
-      "loss": 5.8559,
-      "mean_token_accuracy": 0.2484271454811096,
-      "num_tokens": 4012300.0,
+      "grad_norm": 1.8514106273651123,
+      "learning_rate": 7.851215174866627e-07,
+      "loss": 5.3891,
+      "mean_token_accuracy": 0.301382859647274,
+      "num_tokens": 4520299.0,
       "step": 4300
     },
     {
-      "entropy": 5.984496111869812,
+      "entropy": 5.536689953804016,
       "epoch": 2.5043177892918824,
-      "grad_norm": 0.8995006680488586,
-      "learning_rate": 1.0101339545719278e-05,
-      "loss": 5.8092,
-      "mean_token_accuracy": 0.253717774450779,
-      "num_tokens": 4059323.0,
+      "grad_norm": 2.1830835342407227,
+      "learning_rate": 7.703023117960877e-07,
+      "loss": 5.3672,
+      "mean_token_accuracy": 0.3047756373882294,
+      "num_tokens": 4573065.0,
       "step": 4350
     },
     {
-      "entropy": 6.124767150878906,
+      "entropy": 5.69776873588562,
       "epoch": 2.533103051237766,
-      "grad_norm": 1.3810409307479858,
-      "learning_rate": 9.984857309260339e-06,
-      "loss": 5.9468,
-      "mean_token_accuracy": 0.23715158700942993,
-      "num_tokens": 4107616.0,
+      "grad_norm": 1.999536156654358,
+      "learning_rate": 7.554831061055127e-07,
+      "loss": 5.5236,
+      "mean_token_accuracy": 0.2868007507920265,
+      "num_tokens": 4626807.0,
       "step": 4400
     },
     {
-      "entropy": 5.8810745000839235,
+      "entropy": 5.3977436876297,
       "epoch": 2.56188831318365,
-      "grad_norm": 0.8794332146644592,
-      "learning_rate": 9.868375072801398e-06,
-      "loss": 5.7089,
-      "mean_token_accuracy": 0.2662400561571121,
-      "num_tokens": 4152400.0,
+      "grad_norm": 1.9608020782470703,
+      "learning_rate": 7.406639004149378e-07,
+      "loss": 5.2335,
+      "mean_token_accuracy": 0.3199601462483406,
+      "num_tokens": 4677663.0,
       "step": 4450
     },
     {
-      "entropy": 6.108017959594727,
+      "entropy": 5.6681678771972654,
       "epoch": 2.5906735751295336,
-      "grad_norm": 0.5132983922958374,
-      "learning_rate": 9.751892836342458e-06,
-      "loss": 5.9346,
-      "mean_token_accuracy": 0.23871887892484664,
-      "num_tokens": 4200994.0,
+      "grad_norm": 1.829047441482544,
+      "learning_rate": 7.258446947243627e-07,
+      "loss": 5.491,
+      "mean_token_accuracy": 0.2894612854719162,
+      "num_tokens": 4731830.0,
       "step": 4500
     },
     {
-      "entropy": 5.985005149841308,
+      "entropy": 5.49174174785614,
       "epoch": 2.6194588370754173,
-      "grad_norm": 0.6561470031738281,
-      "learning_rate": 9.635410599883519e-06,
-      "loss": 5.8111,
-      "mean_token_accuracy": 0.25315980523824694,
-      "num_tokens": 4247548.0,
+      "grad_norm": 1.3158719539642334,
+      "learning_rate": 7.110254890337878e-07,
+      "loss": 5.3225,
+      "mean_token_accuracy": 0.3084965732693672,
+      "num_tokens": 4784694.0,
       "step": 4550
     },
     {
-      "entropy": 6.050709452629089,
+      "entropy": 5.573234438896179,
       "epoch": 2.648244099021301,
-      "grad_norm": 0.8790570497512817,
-      "learning_rate": 9.51892836342458e-06,
-      "loss": 5.8789,
-      "mean_token_accuracy": 0.2440834751725197,
-      "num_tokens": 4295250.0,
+      "grad_norm": 1.562915325164795,
+      "learning_rate": 6.962062833432127e-07,
+      "loss": 5.4028,
+      "mean_token_accuracy": 0.2989520016312599,
+      "num_tokens": 4838534.0,
       "step": 4600
     },
     {
-      "entropy": 6.007251596450805,
+      "entropy": 5.550469598770142,
       "epoch": 2.6770293609671847,
-      "grad_norm": 0.6728562116622925,
-      "learning_rate": 9.402446126965639e-06,
-      "loss": 5.8338,
-      "mean_token_accuracy": 0.2509264424443245,
-      "num_tokens": 4341599.0,
+      "grad_norm": 2.114727735519409,
+      "learning_rate": 6.813870776526378e-07,
+      "loss": 5.3804,
+      "mean_token_accuracy": 0.30373542964458466,
+      "num_tokens": 4890611.0,
       "step": 4650
     },
     {
-      "entropy": 5.966628184318543,
+      "entropy": 5.523049550056458,
       "epoch": 2.7058146229130684,
-      "grad_norm": 0.5815795063972473,
-      "learning_rate": 9.285963890506699e-06,
-      "loss": 5.7961,
-      "mean_token_accuracy": 0.2559360232949257,
-      "num_tokens": 4388673.0,
+      "grad_norm": 2.5036823749542236,
+      "learning_rate": 6.665678719620628e-07,
+      "loss": 5.3542,
+      "mean_token_accuracy": 0.30681024432182313,
+      "num_tokens": 4943571.0,
       "step": 4700
     },
     {
-      "entropy": 5.7972593069076535,
+      "entropy": 5.323453049659729,
       "epoch": 2.734599884858952,
-      "grad_norm": 1.0610334873199463,
-      "learning_rate": 9.169481654047758e-06,
-      "loss": 5.6318,
-      "mean_token_accuracy": 0.27574603259563446,
-      "num_tokens": 4432959.0,
+      "grad_norm": 1.8069168329238892,
+      "learning_rate": 6.517486662714878e-07,
+      "loss": 5.1583,
+      "mean_token_accuracy": 0.32906652927398683,
+      "num_tokens": 4993871.0,
       "step": 4750
     },
     {
-      "entropy": 5.984181261062622,
+      "entropy": 5.504038324356079,
       "epoch": 2.763385146804836,
-      "grad_norm": 2.1847357749938965,
-      "learning_rate": 9.052999417588819e-06,
-      "loss": 5.8153,
-      "mean_token_accuracy": 0.2533784031867981,
-      "num_tokens": 4479190.0,
+      "grad_norm": 4.750283718109131,
+      "learning_rate": 6.369294605809128e-07,
+      "loss": 5.3366,
+      "mean_token_accuracy": 0.3087608867883682,
+      "num_tokens": 5046187.0,
       "step": 4800
     },
     {
-      "entropy": 5.959725599288941,
+      "entropy": 5.487624549865723,
       "epoch": 2.7921704087507195,
-      "grad_norm": 0.5671709179878235,
-      "learning_rate": 8.936517181129878e-06,
-      "loss": 5.7912,
-      "mean_token_accuracy": 0.2556650054454803,
-      "num_tokens": 4525674.0,
+      "grad_norm": 1.4186172485351562,
+      "learning_rate": 6.221102548903379e-07,
+      "loss": 5.3237,
+      "mean_token_accuracy": 0.3088638699054718,
+      "num_tokens": 5098644.0,
       "step": 4850
     },
     {
-      "entropy": 5.814929313659668,
+      "entropy": 5.346905107498169,
       "epoch": 2.8209556706966032,
-      "grad_norm": 0.9447108507156372,
-      "learning_rate": 8.820034944670938e-06,
-      "loss": 5.6478,
-      "mean_token_accuracy": 0.27417868226766584,
-      "num_tokens": 4570379.0,
+      "grad_norm": 1.5670177936553955,
+      "learning_rate": 6.072910491997628e-07,
+      "loss": 5.1849,
+      "mean_token_accuracy": 0.3265886321663857,
+      "num_tokens": 5149345.0,
       "step": 4900
     },
     {
-      "entropy": 5.96754421710968,
+      "entropy": 5.510410032272339,
       "epoch": 2.849740932642487,
-      "grad_norm": 2.009676218032837,
-      "learning_rate": 8.703552708211999e-06,
-      "loss": 5.795,
-      "mean_token_accuracy": 0.2556305864453316,
-      "num_tokens": 4617184.0,
+      "grad_norm": 7.489855766296387,
+      "learning_rate": 5.924718435091879e-07,
+      "loss": 5.3424,
+      "mean_token_accuracy": 0.30768151730299,
+      "num_tokens": 5202028.0,
       "step": 4950
     },
     {
-      "entropy": 6.008112049102783,
+      "entropy": 5.525181493759155,
       "epoch": 2.8785261945883707,
-      "grad_norm": 1.1977978944778442,
-      "learning_rate": 8.587070471753058e-06,
-      "loss": 5.8416,
-      "mean_token_accuracy": 0.2494604030251503,
-      "num_tokens": 4664180.0,
+      "grad_norm": 1.8829196691513062,
+      "learning_rate": 5.776526378186128e-07,
+      "loss": 5.3654,
+      "mean_token_accuracy": 0.30342737555503846,
+      "num_tokens": 5255082.0,
       "step": 5000
     },
     {
-      "entropy": 5.832320966720581,
+      "entropy": 5.374098634719848,
       "epoch": 2.9073114565342544,
-      "grad_norm": 0.4845636785030365,
-      "learning_rate": 8.470588235294118e-06,
-      "loss": 5.6672,
-      "mean_token_accuracy": 0.27187123566865923,
-      "num_tokens": 4708377.0,
+      "grad_norm": 1.3901060819625854,
+      "learning_rate": 5.628334321280379e-07,
+      "loss": 5.2103,
+      "mean_token_accuracy": 0.3233291879296303,
+      "num_tokens": 5305042.0,
       "step": 5050
     },
     {
-      "entropy": 5.84138514995575,
+      "entropy": 5.374619431495667,
       "epoch": 2.936096718480138,
-      "grad_norm": 0.8487229943275452,
-      "learning_rate": 8.354105998835179e-06,
-      "loss": 5.6769,
-      "mean_token_accuracy": 0.26995211571455,
-      "num_tokens": 4753587.0,
+      "grad_norm": 1.6586560010910034,
+      "learning_rate": 5.48014226437463e-07,
+      "loss": 5.2125,
+      "mean_token_accuracy": 0.322759662270546,
+      "num_tokens": 5356310.0,
       "step": 5100
     },
     {
-      "entropy": 6.016681690216064,
+      "entropy": 5.527479724884033,
       "epoch": 2.964881980426022,
-      "grad_norm": 0.9554332494735718,
-      "learning_rate": 8.237623762376238e-06,
-      "loss": 5.8479,
-      "mean_token_accuracy": 0.24785644590854644,
-      "num_tokens": 4800508.0,
+      "grad_norm": 1.6678485870361328,
+      "learning_rate": 5.331950207468879e-07,
+      "loss": 5.3627,
+      "mean_token_accuracy": 0.30430852621793747,
+      "num_tokens": 5409283.0,
       "step": 5150
     },
     {
-      "entropy": 6.103472499847412,
+      "entropy": 5.6171248292922975,
       "epoch": 2.9936672423719055,
-      "grad_norm": 0.6602863669395447,
-      "learning_rate": 8.121141525917298e-06,
-      "loss": 5.9305,
-      "mean_token_accuracy": 0.23794592499732972,
-      "num_tokens": 4849415.0,
+      "grad_norm": 1.50790274143219,
+      "learning_rate": 5.18375815056313e-07,
+      "loss": 5.4484,
+      "mean_token_accuracy": 0.29375598043203355,
+      "num_tokens": 5464332.0,
       "step": 5200
     },
     {
       "epoch": 3.0,
-      "eval_entropy": 6.254081044878278,
-      "eval_loss": 6.0980024337768555,
-      "eval_mean_token_accuracy": 0.21401402258103894,
-      "eval_model_preparation_time": 0.0036,
-      "eval_num_tokens": 4859157.0,
-      "eval_runtime": 75.9443,
-      "eval_samples_per_second": 5.715,
-      "eval_steps_per_second": 2.857,
+      "eval_entropy": 5.78779639186947,
+      "eval_loss": 5.628758430480957,
+      "eval_mean_token_accuracy": 0.2653660801698535,
+      "eval_model_preparation_time": 0.0047,
+      "eval_num_tokens": 5475321.0,
+      "eval_runtime": 80.3676,
+      "eval_samples_per_second": 5.4,
+      "eval_steps_per_second": 2.7,
       "step": 5211
     }
   ],
   "logging_steps": 50,
-  "max_steps": 8685,
+  "max_steps": 6948,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 5,
+  "num_train_epochs": 4,
   "save_steps": 500,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -1103,7 +1103,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 6.795785692717056e+16,
+  "total_flos": 7.520890606086144e+16,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null

checkpoint-5211/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:130d33149272782bd60306263c371036419926142b8999aad7806359168f8484
+oid sha256:8db5c304963110404ebb6947b83ba95bd9b8aad1f9b8b578cc33c46d601e13dc
 size 6225

checkpoint-6948/adapter_config.json

@@ -16,7 +16,7 @@
   "layers_pattern": null,
   "layers_to_transform": null,
   "loftq_config": {},
-  "lora_alpha": 16,
+  "lora_alpha": 32,
   "lora_bias": false,
   "lora_dropout": 0.1,
   "megatron_config": null,
@@ -25,12 +25,14 @@
   "peft_type": "LORA",
   "peft_version": "0.18.0",
   "qalora_group_size": 16,
-  "r": 8,
+  "r": 24,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
     "q_proj",
-    "v_proj"
+    "k_proj",
+    "v_proj",
+    "o_proj"
   ],
   "target_parameters": null,
   "task_type": "CAUSAL_LM",

checkpoint-6948/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d98bdffcaf94c61a7d4f2d6e4effa1765874d8fc8f97c30cd420626b72320c3b
-size 4374520
+oid sha256:a42655e5c5bf5a17388c99c67741b81d97a904a649f92d5298361717c78abaac
+size 26182176

checkpoint-6948/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:00bc515ace55234a59210394255dbfc391f61f9f697c5ca151b3d2dde3e16426
-size 8783179
+oid sha256:3f4dc67fd123c4a9f8eb45bc8894cccfeeb5a7766daf44f4ca97786db172fd5f
+size 52486155

checkpoint-6948/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6d29771f79d36c2441adbe71a52a34256493ea9dc339b022adf52f9bd8969a78
+oid sha256:de23a91d8efb3b92e132a49e237b78926ed9acaded7b594b358633abace10591
 size 14645

checkpoint-6948/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:78dfb562bc2784af18bf3113adae6543329adb31bab3face56cd71f730439d6d
+oid sha256:ced24601208e373e591e4ce71c0d860f568ef5205374f58c5db9ee9e78232103
 size 1465

checkpoint-6948/trainer_state.json

@@ -1,6 +1,6 @@
 {
   "best_global_step": 6948,
-  "best_metric": 6.0967888832092285,
+  "best_metric": 5.622366428375244,
   "best_model_checkpoint": "./output/checkpoint-6948",
   "epoch": 4.0,
   "eval_steps": 500,
@@ -10,1438 +10,1438 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "entropy": 3.864118957519531,
+      "entropy": 3.6583470726013183,
       "epoch": 0.028785261945883708,
-      "grad_norm": 2.7545533180236816,
-      "learning_rate": 9.800000000000001e-06,
-      "loss": 15.2997,
-      "mean_token_accuracy": 0.10086015284061432,
-      "num_tokens": 47319.0,
+      "grad_norm": 3.3817152976989746,
+      "learning_rate": 4.9e-07,
+      "loss": 13.8754,
+      "mean_token_accuracy": 0.15036460414528846,
+      "num_tokens": 53093.0,
       "step": 50
     },
     {
-      "entropy": 4.047076859474182,
+      "entropy": 3.669608063697815,
       "epoch": 0.057570523891767415,
-      "grad_norm": 5.0328264236450195,
-      "learning_rate": 1.98e-05,
-      "loss": 15.3264,
-      "mean_token_accuracy": 0.09582207053899765,
-      "num_tokens": 96809.0,
+      "grad_norm": 3.2541544437408447,
+      "learning_rate": 9.9e-07,
+      "loss": 14.2282,
+      "mean_token_accuracy": 0.14137721598148345,
+      "num_tokens": 108334.0,
       "step": 100
     },
     {
-      "entropy": 4.7578076648712155,
+      "entropy": 3.569736371040344,
       "epoch": 0.08635578583765112,
-      "grad_norm": 38.50589370727539,
-      "learning_rate": 1.988584740827024e-05,
-      "loss": 13.0056,
-      "mean_token_accuracy": 0.126854517608881,
-      "num_tokens": 139962.0,
+      "grad_norm": 3.6797454357147217,
+      "learning_rate": 1.49e-06,
+      "loss": 13.0735,
+      "mean_token_accuracy": 0.17473630651831626,
+      "num_tokens": 157491.0,
       "step": 150
     },
     {
-      "entropy": 6.80673882484436,
+      "entropy": 3.7253233194351196,
       "epoch": 0.11514104778353483,
-      "grad_norm": 12.030129432678223,
-      "learning_rate": 1.97693651718113e-05,
-      "loss": 9.2822,
-      "mean_token_accuracy": 0.11084575355052947,
-      "num_tokens": 188029.0,
+      "grad_norm": 4.297911643981934,
+      "learning_rate": 1.99e-06,
+      "loss": 13.7392,
+      "mean_token_accuracy": 0.1473099772632122,
+      "num_tokens": 211394.0,
       "step": 200
     },
     {
-      "entropy": 7.177925786972046,
+      "entropy": 3.8280500602722167,
       "epoch": 0.14392630972941853,
-      "grad_norm": 4.852536201477051,
-      "learning_rate": 1.965288293535236e-05,
-      "loss": 7.6333,
-      "mean_token_accuracy": 0.12398939326405525,
-      "num_tokens": 234425.0,
+      "grad_norm": 4.405268669128418,
+      "learning_rate": 1.9854771784232364e-06,
+      "loss": 13.0797,
+      "mean_token_accuracy": 0.16704789966344832,
+      "num_tokens": 263685.0,
       "step": 250
     },
     {
-      "entropy": 7.080496473312378,
+      "entropy": 4.066333084106446,
       "epoch": 0.17271157167530224,
-      "grad_norm": 4.10841178894043,
-      "learning_rate": 1.9536400698893422e-05,
-      "loss": 7.1632,
-      "mean_token_accuracy": 0.13563686355948448,
-      "num_tokens": 278885.0,
+      "grad_norm": 4.757556438446045,
+      "learning_rate": 1.9706579727326615e-06,
+      "loss": 12.6321,
+      "mean_token_accuracy": 0.1691790708899498,
+      "num_tokens": 314059.0,
       "step": 300
     },
     {
-      "entropy": 6.931579580307007,
+      "entropy": 4.257266030311585,
       "epoch": 0.20149683362118595,
-      "grad_norm": 14.636048316955566,
-      "learning_rate": 1.941991846243448e-05,
-      "loss": 6.8213,
-      "mean_token_accuracy": 0.16459846690297128,
-      "num_tokens": 325491.0,
+      "grad_norm": 6.406249523162842,
+      "learning_rate": 1.955838767042086e-06,
+      "loss": 12.2253,
+      "mean_token_accuracy": 0.17223650276660918,
+      "num_tokens": 367038.0,
       "step": 350
     },
     {
-      "entropy": 6.853660764694214,
+      "entropy": 4.694105777740479,
       "epoch": 0.23028209556706966,
-      "grad_norm": 5.966708183288574,
-      "learning_rate": 1.930343622597554e-05,
-      "loss": 6.6625,
-      "mean_token_accuracy": 0.17670693069696428,
-      "num_tokens": 372913.0,
+      "grad_norm": 12.57987117767334,
+      "learning_rate": 1.9410195613515113e-06,
+      "loss": 11.9714,
+      "mean_token_accuracy": 0.15997304677963256,
+      "num_tokens": 420327.0,
       "step": 400
     },
     {
-      "entropy": 6.684267387390137,
+      "entropy": 5.205010280609131,
       "epoch": 0.25906735751295334,
-      "grad_norm": 4.031010627746582,
-      "learning_rate": 1.91869539895166e-05,
-      "loss": 6.4505,
-      "mean_token_accuracy": 0.1943434515595436,
-      "num_tokens": 419159.0,
+      "grad_norm": 15.570313453674316,
+      "learning_rate": 1.9262003556609364e-06,
+      "loss": 10.8173,
+      "mean_token_accuracy": 0.16447648257017136,
+      "num_tokens": 472429.0,
       "step": 450
     },
     {
-      "entropy": 6.679989137649536,
+      "entropy": 5.917805089950561,
       "epoch": 0.28785261945883706,
-      "grad_norm": 6.251070022583008,
-      "learning_rate": 1.907047175305766e-05,
-      "loss": 6.4314,
-      "mean_token_accuracy": 0.19514557600021362,
-      "num_tokens": 466994.0,
+      "grad_norm": 23.61503791809082,
+      "learning_rate": 1.9113811499703615e-06,
+      "loss": 9.3196,
+      "mean_token_accuracy": 0.16179455041885377,
+      "num_tokens": 526315.0,
       "step": 500
     },
     {
-      "entropy": 6.477229623794556,
+      "entropy": 6.380368332862854,
       "epoch": 0.31663788140472077,
-      "grad_norm": 3.8656675815582275,
-      "learning_rate": 1.895398951659872e-05,
-      "loss": 6.2139,
-      "mean_token_accuracy": 0.21764743447303772,
-      "num_tokens": 513308.0,
+      "grad_norm": 13.846810340881348,
+      "learning_rate": 1.8965619442797864e-06,
+      "loss": 7.9636,
+      "mean_token_accuracy": 0.16881170988082886,
+      "num_tokens": 578511.0,
       "step": 550
     },
     {
-      "entropy": 6.408129243850708,
+      "entropy": 6.507339992523193,
       "epoch": 0.3454231433506045,
-      "grad_norm": 8.688581466674805,
-      "learning_rate": 1.883750728013978e-05,
-      "loss": 6.1224,
-      "mean_token_accuracy": 0.23438037544488907,
-      "num_tokens": 559679.0,
+      "grad_norm": 4.569090366363525,
+      "learning_rate": 1.8817427385892115e-06,
+      "loss": 7.4171,
+      "mean_token_accuracy": 0.16941152423620223,
+      "num_tokens": 630937.0,
       "step": 600
     },
     {
-      "entropy": 6.128518767356873,
+      "entropy": 6.392864561080932,
       "epoch": 0.3742084052964882,
-      "grad_norm": 5.419503688812256,
-      "learning_rate": 1.872102504368084e-05,
-      "loss": 5.8692,
-      "mean_token_accuracy": 0.26634690463542937,
-      "num_tokens": 603140.0,
+      "grad_norm": 4.594696521759033,
+      "learning_rate": 1.8669235328986366e-06,
+      "loss": 6.9389,
+      "mean_token_accuracy": 0.1844496901333332,
+      "num_tokens": 680501.0,
       "step": 650
     },
     {
-      "entropy": 6.322700729370117,
+      "entropy": 6.6726202869415285,
       "epoch": 0.4029936672423719,
-      "grad_norm": 2.2213082313537598,
-      "learning_rate": 1.86045428072219e-05,
-      "loss": 6.0717,
-      "mean_token_accuracy": 0.24038562417030335,
-      "num_tokens": 650179.0,
+      "grad_norm": 4.768734931945801,
+      "learning_rate": 1.8521043272080617e-06,
+      "loss": 6.9818,
+      "mean_token_accuracy": 0.16990411713719367,
+      "num_tokens": 733231.0,
       "step": 700
     },
     {
-      "entropy": 6.236415157318115,
+      "entropy": 6.592793455123902,
       "epoch": 0.4317789291882556,
-      "grad_norm": 4.804980278015137,
-      "learning_rate": 1.848806057076296e-05,
-      "loss": 5.9986,
-      "mean_token_accuracy": 0.24596781462430953,
-      "num_tokens": 696220.0,
+      "grad_norm": 3.253056764602661,
+      "learning_rate": 1.8372851215174864e-06,
+      "loss": 6.7105,
+      "mean_token_accuracy": 0.18250102579593658,
+      "num_tokens": 785373.0,
       "step": 750
     },
     {
-      "entropy": 6.269758443832398,
+      "entropy": 6.683582029342651,
       "epoch": 0.4605641911341393,
-      "grad_norm": 2.2888853549957275,
-      "learning_rate": 1.837157833430402e-05,
-      "loss": 6.0385,
-      "mean_token_accuracy": 0.24074893474578857,
-      "num_tokens": 743909.0,
+      "grad_norm": 2.1871063709259033,
+      "learning_rate": 1.8224659158269115e-06,
+      "loss": 6.6685,
+      "mean_token_accuracy": 0.17129646152257919,
+      "num_tokens": 838646.0,
       "step": 800
     },
     {
-      "entropy": 6.270364007949829,
+      "entropy": 6.636875295639038,
       "epoch": 0.48934945308002303,
-      "grad_norm": 3.0903279781341553,
-      "learning_rate": 1.825509609784508e-05,
-      "loss": 6.0481,
-      "mean_token_accuracy": 0.23740622967481614,
-      "num_tokens": 792015.0,
+      "grad_norm": 3.2284677028656006,
+      "learning_rate": 1.8076467101363366e-06,
+      "loss": 6.53,
+      "mean_token_accuracy": 0.18053789794445038,
+      "num_tokens": 892380.0,
       "step": 850
     },
     {
-      "entropy": 6.3037636184692385,
+      "entropy": 6.610673260688782,
       "epoch": 0.5181347150259067,
-      "grad_norm": 3.969320058822632,
-      "learning_rate": 1.813861386138614e-05,
-      "loss": 6.0855,
-      "mean_token_accuracy": 0.2309597587585449,
-      "num_tokens": 841802.0,
+      "grad_norm": 2.2088730335235596,
+      "learning_rate": 1.7928275044457617e-06,
+      "loss": 6.4429,
+      "mean_token_accuracy": 0.18492739230394364,
+      "num_tokens": 947971.0,
       "step": 900
     },
     {
-      "entropy": 6.038041458129883,
+      "entropy": 6.242899022102356,
       "epoch": 0.5469199769717904,
-      "grad_norm": 2.2712185382843018,
-      "learning_rate": 1.80221316249272e-05,
-      "loss": 5.8285,
-      "mean_token_accuracy": 0.26099125802516937,
-      "num_tokens": 886492.0,
+      "grad_norm": 2.3000030517578125,
+      "learning_rate": 1.7780082987551866e-06,
+      "loss": 6.047,
+      "mean_token_accuracy": 0.2291259828209877,
+      "num_tokens": 998810.0,
       "step": 950
     },
     {
-      "entropy": 6.142958383560181,
+      "entropy": 6.311488924026489,
       "epoch": 0.5757052389176741,
-      "grad_norm": 1.2311755418777466,
-      "learning_rate": 1.790564938846826e-05,
-      "loss": 5.9357,
-      "mean_token_accuracy": 0.24810438305139543,
-      "num_tokens": 932807.0,
+      "grad_norm": 2.1333675384521484,
+      "learning_rate": 1.7631890930646115e-06,
+      "loss": 6.0919,
+      "mean_token_accuracy": 0.22644571751356124,
+      "num_tokens": 1050860.0,
       "step": 1000
     },
     {
-      "entropy": 6.199834351539612,
+      "entropy": 6.3254336166381835,
       "epoch": 0.6044905008635578,
-      "grad_norm": 2.2788379192352295,
-      "learning_rate": 1.7789167152009318e-05,
-      "loss": 5.9964,
-      "mean_token_accuracy": 0.23942562609910964,
-      "num_tokens": 980541.0,
+      "grad_norm": 2.0400779247283936,
+      "learning_rate": 1.7483698873740366e-06,
+      "loss": 6.094,
+      "mean_token_accuracy": 0.2222653564810753,
+      "num_tokens": 1104304.0,
       "step": 1050
     },
     {
-      "entropy": 5.961639919281006,
+      "entropy": 6.046922063827514,
       "epoch": 0.6332757628094415,
-      "grad_norm": 1.9077532291412354,
-      "learning_rate": 1.767268491555038e-05,
-      "loss": 5.7664,
-      "mean_token_accuracy": 0.26718012750148773,
-      "num_tokens": 1023882.0,
+      "grad_norm": 2.8049051761627197,
+      "learning_rate": 1.7335506816834617e-06,
+      "loss": 5.8011,
+      "mean_token_accuracy": 0.25127078920602797,
+      "num_tokens": 1153605.0,
       "step": 1100
     },
     {
-      "entropy": 5.889280087947846,
+      "entropy": 5.943600912094116,
       "epoch": 0.6620610247553252,
-      "grad_norm": 2.4254891872406006,
-      "learning_rate": 1.7556202679091442e-05,
-      "loss": 5.6952,
-      "mean_token_accuracy": 0.27529804170131683,
-      "num_tokens": 1068300.0,
+      "grad_norm": 4.063963890075684,
+      "learning_rate": 1.7187314759928866e-06,
+      "loss": 5.6855,
+      "mean_token_accuracy": 0.26265266716480257,
+      "num_tokens": 1204328.0,
       "step": 1150
     },
     {
-      "entropy": 6.085640063285828,
+      "entropy": 6.12883231639862,
       "epoch": 0.690846286701209,
-      "grad_norm": 2.35312557220459,
-      "learning_rate": 1.74397204426325e-05,
-      "loss": 5.8898,
-      "mean_token_accuracy": 0.25166562348604204,
-      "num_tokens": 1115425.0,
+      "grad_norm": 3.9440460205078125,
+      "learning_rate": 1.7039122703023117e-06,
+      "loss": 5.8578,
+      "mean_token_accuracy": 0.24439335912466048,
+      "num_tokens": 1257415.0,
       "step": 1200
     },
     {
-      "entropy": 6.146574058532715,
+      "entropy": 6.164987115859986,
       "epoch": 0.7196315486470927,
-      "grad_norm": 1.7730146646499634,
-      "learning_rate": 1.732323820617356e-05,
-      "loss": 5.9519,
-      "mean_token_accuracy": 0.24276195973157882,
-      "num_tokens": 1162319.0,
+      "grad_norm": 3.20070481300354,
+      "learning_rate": 1.6890930646117368e-06,
+      "loss": 5.8876,
+      "mean_token_accuracy": 0.24275501281023026,
+      "num_tokens": 1310049.0,
       "step": 1250
     },
     {
-      "entropy": 6.079372715950012,
+      "entropy": 6.080997190475464,
       "epoch": 0.7484168105929764,
-      "grad_norm": 1.7070863246917725,
-      "learning_rate": 1.720675596971462e-05,
-      "loss": 5.8922,
-      "mean_token_accuracy": 0.24961524546146394,
-      "num_tokens": 1208230.0,
+      "grad_norm": 2.8067362308502197,
+      "learning_rate": 1.6742738589211617e-06,
+      "loss": 5.8058,
+      "mean_token_accuracy": 0.25242207854986193,
+      "num_tokens": 1361794.0,
       "step": 1300
     },
     {
-      "entropy": 5.9683656406402585,
+      "entropy": 5.940848155021667,
       "epoch": 0.7772020725388601,
-      "grad_norm": 1.8790594339370728,
-      "learning_rate": 1.709027373325568e-05,
-      "loss": 5.7827,
-      "mean_token_accuracy": 0.2632122594118118,
-      "num_tokens": 1253074.0,
+      "grad_norm": 2.6375925540924072,
+      "learning_rate": 1.6594546532305868e-06,
+      "loss": 5.6718,
+      "mean_token_accuracy": 0.2665082859992981,
+      "num_tokens": 1412773.0,
       "step": 1350
     },
     {
-      "entropy": 6.107076721191406,
+      "entropy": 6.071129274368286,
       "epoch": 0.8059873344847438,
-      "grad_norm": 1.1745644807815552,
-      "learning_rate": 1.6973791496796742e-05,
-      "loss": 5.9211,
-      "mean_token_accuracy": 0.24564073830842972,
-      "num_tokens": 1300179.0,
+      "grad_norm": 3.951350212097168,
+      "learning_rate": 1.6446354475400117e-06,
+      "loss": 5.8012,
+      "mean_token_accuracy": 0.25434976994991304,
+      "num_tokens": 1465620.0,
       "step": 1400
     },
     {
-      "entropy": 6.141328382492065,
+      "entropy": 6.069429359436035,
       "epoch": 0.8347725964306275,
-      "grad_norm": 1.0346958637237549,
-      "learning_rate": 1.68573092603378e-05,
-      "loss": 5.9584,
-      "mean_token_accuracy": 0.23997059136629104,
-      "num_tokens": 1347539.0,
+      "grad_norm": 3.580608606338501,
+      "learning_rate": 1.6298162418494368e-06,
+      "loss": 5.8027,
+      "mean_token_accuracy": 0.25208072274923327,
+      "num_tokens": 1518899.0,
       "step": 1450
     },
     {
-      "entropy": 6.070010099411011,
+      "entropy": 6.005315380096436,
       "epoch": 0.8635578583765112,
-      "grad_norm": 1.6541163921356201,
-      "learning_rate": 1.674082702387886e-05,
-      "loss": 5.889,
-      "mean_token_accuracy": 0.24875166177749633,
-      "num_tokens": 1394157.0,
+      "grad_norm": 3.9580376148223877,
+      "learning_rate": 1.614997036158862e-06,
+      "loss": 5.7364,
+      "mean_token_accuracy": 0.25940640360116957,
+      "num_tokens": 1571304.0,
       "step": 1500
     },
     {
-      "entropy": 6.207450666427612,
+      "entropy": 6.0786464881896975,
       "epoch": 0.8923431203223949,
-      "grad_norm": 0.9742990732192993,
-      "learning_rate": 1.662434478741992e-05,
-      "loss": 6.0217,
-      "mean_token_accuracy": 0.23067249596118927,
-      "num_tokens": 1443892.0,
+      "grad_norm": 4.55721378326416,
+      "learning_rate": 1.6001778304682868e-06,
+      "loss": 5.8092,
+      "mean_token_accuracy": 0.2496869170665741,
+      "num_tokens": 1627369.0,
       "step": 1550
     },
     {
-      "entropy": 6.026197805404663,
+      "entropy": 5.939382014274597,
       "epoch": 0.9211283822682786,
-      "grad_norm": 1.4229531288146973,
-      "learning_rate": 1.650786255096098e-05,
-      "loss": 5.8455,
-      "mean_token_accuracy": 0.2537291014194489,
-      "num_tokens": 1491050.0,
+      "grad_norm": 2.330057144165039,
+      "learning_rate": 1.5853586247777117e-06,
+      "loss": 5.6604,
+      "mean_token_accuracy": 0.2686630353331566,
+      "num_tokens": 1680401.0,
       "step": 1600
     },
     {
-      "entropy": 6.210526428222656,
+      "entropy": 6.121775646209716,
       "epoch": 0.9499136442141624,
-      "grad_norm": 1.3555018901824951,
-      "learning_rate": 1.6391380314502038e-05,
-      "loss": 6.0279,
-      "mean_token_accuracy": 0.2308420208096504,
-      "num_tokens": 1540809.0,
+      "grad_norm": 2.9881200790405273,
+      "learning_rate": 1.5705394190871368e-06,
+      "loss": 5.8388,
+      "mean_token_accuracy": 0.2503683388233185,
+      "num_tokens": 1735745.0,
       "step": 1650
     },
     {
-      "entropy": 5.9872834014892575,
+      "entropy": 5.840040788650513,
       "epoch": 0.9786989061600461,
-      "grad_norm": 0.9893498420715332,
-      "learning_rate": 1.62748980780431e-05,
-      "loss": 5.8137,
-      "mean_token_accuracy": 0.2566875320672989,
-      "num_tokens": 1585876.0,
+      "grad_norm": 3.798994779586792,
+      "learning_rate": 1.555720213396562e-06,
+      "loss": 5.5635,
+      "mean_token_accuracy": 0.278279125392437,
+      "num_tokens": 1786896.0,
       "step": 1700
     },
     {
       "epoch": 1.0,
-      "eval_entropy": 6.322207130045386,
-      "eval_loss": 6.15173864364624,
-      "eval_mean_token_accuracy": 0.21116007946877985,
-      "eval_model_preparation_time": 0.0036,
-      "eval_num_tokens": 1619719.0,
-      "eval_runtime": 76.1297,
-      "eval_samples_per_second": 5.701,
-      "eval_steps_per_second": 2.85,
+      "eval_entropy": 6.139133475343203,
+      "eval_loss": 5.861395835876465,
+      "eval_mean_token_accuracy": 0.2402858340657801,
+      "eval_model_preparation_time": 0.0047,
+      "eval_num_tokens": 1825107.0,
+      "eval_runtime": 79.3994,
+      "eval_samples_per_second": 5.466,
+      "eval_steps_per_second": 2.733,
       "step": 1737
     },
     {
-      "entropy": 6.038531675338745,
+      "entropy": 5.8970259666442875,
       "epoch": 1.0074841681059297,
-      "grad_norm": 0.8715208172798157,
-      "learning_rate": 1.615841584158416e-05,
-      "loss": 5.8628,
-      "mean_token_accuracy": 0.2510762655735016,
-      "num_tokens": 1632015.0,
+      "grad_norm": 2.6411802768707275,
+      "learning_rate": 1.540901007705987e-06,
+      "loss": 5.614,
+      "mean_token_accuracy": 0.273006406724453,
+      "num_tokens": 1838864.0,
       "step": 1750
     },
     {
-      "entropy": 6.164030771255494,
+      "entropy": 6.0111794090271,
       "epoch": 1.0362694300518134,
-      "grad_norm": 0.7344900965690613,
-      "learning_rate": 1.604193360512522e-05,
-      "loss": 5.9856,
-      "mean_token_accuracy": 0.2351543301343918,
-      "num_tokens": 1681154.0,
+      "grad_norm": 3.6491827964782715,
+      "learning_rate": 1.526081802015412e-06,
+      "loss": 5.7323,
+      "mean_token_accuracy": 0.26104256987571717,
+      "num_tokens": 1893816.0,
       "step": 1800
     },
     {
-      "entropy": 6.0731862354278565,
+      "entropy": 5.902219276428223,
       "epoch": 1.065054691997697,
-      "grad_norm": 1.0801328420639038,
-      "learning_rate": 1.592545136866628e-05,
-      "loss": 5.8976,
-      "mean_token_accuracy": 0.24701615989208223,
-      "num_tokens": 1728110.0,
+      "grad_norm": 2.593249559402466,
+      "learning_rate": 1.5112625963248368e-06,
+      "loss": 5.6187,
+      "mean_token_accuracy": 0.2746362566947937,
+      "num_tokens": 1946532.0,
       "step": 1850
     },
     {
-      "entropy": 6.079212121963501,
+      "entropy": 5.874705944061279,
       "epoch": 1.0938399539435808,
-      "grad_norm": 0.7876909375190735,
-      "learning_rate": 1.5808969132207338e-05,
-      "loss": 5.9056,
-      "mean_token_accuracy": 0.24457543224096298,
-      "num_tokens": 1775703.0,
+      "grad_norm": 2.554327964782715,
+      "learning_rate": 1.496443390634262e-06,
+      "loss": 5.6021,
+      "mean_token_accuracy": 0.2795292744040489,
+      "num_tokens": 2000184.0,
       "step": 1900
     },
     {
-      "entropy": 6.062467746734619,
+      "entropy": 5.850096368789673,
       "epoch": 1.1226252158894645,
-      "grad_norm": 0.5999078750610352,
-      "learning_rate": 1.56924868957484e-05,
-      "loss": 5.8899,
-      "mean_token_accuracy": 0.2469428673386574,
-      "num_tokens": 1821980.0,
+      "grad_norm": 3.6060993671417236,
+      "learning_rate": 1.481624184943687e-06,
+      "loss": 5.576,
+      "mean_token_accuracy": 0.28532547056674956,
+      "num_tokens": 2052250.0,
       "step": 1950
     },
     {
-      "entropy": 6.031774473190308,
+      "entropy": 5.802229671478272,
       "epoch": 1.1514104778353482,
-      "grad_norm": 1.6313235759735107,
-      "learning_rate": 1.557600465928946e-05,
-      "loss": 5.8593,
-      "mean_token_accuracy": 0.250918984413147,
-      "num_tokens": 1867547.0,
+      "grad_norm": 3.0913314819335938,
+      "learning_rate": 1.466804979253112e-06,
+      "loss": 5.53,
+      "mean_token_accuracy": 0.2916027933359146,
+      "num_tokens": 2103531.0,
       "step": 2000
     },
     {
-      "entropy": 6.122789564132691,
+      "entropy": 5.875646467208862,
       "epoch": 1.180195739781232,
-      "grad_norm": 2.562373161315918,
-      "learning_rate": 1.545952242283052e-05,
-      "loss": 5.9502,
-      "mean_token_accuracy": 0.23938885867595672,
-      "num_tokens": 1915411.0,
+      "grad_norm": 4.777045726776123,
+      "learning_rate": 1.451985773562537e-06,
+      "loss": 5.6146,
+      "mean_token_accuracy": 0.28063644528388976,
+      "num_tokens": 2157098.0,
       "step": 2050
     },
     {
-      "entropy": 6.067130417823791,
+      "entropy": 5.786596937179565,
       "epoch": 1.2089810017271156,
-      "grad_norm": 0.9762872457504272,
-      "learning_rate": 1.534304018637158e-05,
-      "loss": 5.8956,
-      "mean_token_accuracy": 0.2454381173849106,
-      "num_tokens": 1964009.0,
+      "grad_norm": 4.207762718200684,
+      "learning_rate": 1.437166567871962e-06,
+      "loss": 5.5417,
+      "mean_token_accuracy": 0.2870470091700554,
+      "num_tokens": 2211827.0,
       "step": 2100
     },
     {
-      "entropy": 5.9613511180877685,
+      "entropy": 5.672234449386597,
       "epoch": 1.2377662636729994,
-      "grad_norm": 0.8701547384262085,
-      "learning_rate": 1.5226557949912639e-05,
-      "loss": 5.7907,
-      "mean_token_accuracy": 0.25976367652416227,
-      "num_tokens": 2008595.0,
+      "grad_norm": 2.2771811485290527,
+      "learning_rate": 1.422347362181387e-06,
+      "loss": 5.4285,
+      "mean_token_accuracy": 0.30194485366344453,
+      "num_tokens": 2262174.0,
       "step": 2150
     },
     {
-      "entropy": 6.13505428314209,
+      "entropy": 5.862573285102844,
       "epoch": 1.266551525618883,
-      "grad_norm": 0.8511647582054138,
-      "learning_rate": 1.51100757134537e-05,
-      "loss": 5.9619,
-      "mean_token_accuracy": 0.23760781466960906,
-      "num_tokens": 2057229.0,
+      "grad_norm": 3.3273422718048096,
+      "learning_rate": 1.4075281564908121e-06,
+      "loss": 5.6169,
+      "mean_token_accuracy": 0.278145115673542,
+      "num_tokens": 2316440.0,
       "step": 2200
     },
     {
-      "entropy": 6.025254983901977,
+      "entropy": 5.734760231971741,
       "epoch": 1.2953367875647668,
-      "grad_norm": 0.7627406120300293,
-      "learning_rate": 1.4993593476994758e-05,
-      "loss": 5.8546,
-      "mean_token_accuracy": 0.2508662334084511,
-      "num_tokens": 2103631.0,
+      "grad_norm": 3.7049715518951416,
+      "learning_rate": 1.392708950800237e-06,
+      "loss": 5.493,
+      "mean_token_accuracy": 0.2941485676169395,
+      "num_tokens": 2368468.0,
       "step": 2250
     },
     {
-      "entropy": 5.981974196434021,
+      "entropy": 5.665819988250733,
       "epoch": 1.3241220495106505,
-      "grad_norm": 1.6922173500061035,
-      "learning_rate": 1.4877111240535819e-05,
-      "loss": 5.8119,
-      "mean_token_accuracy": 0.256170334815979,
-      "num_tokens": 2150369.0,
+      "grad_norm": 3.572636604309082,
+      "learning_rate": 1.3778897451096621e-06,
+      "loss": 5.4352,
+      "mean_token_accuracy": 0.3003745040297508,
+      "num_tokens": 2421180.0,
       "step": 2300
     },
     {
-      "entropy": 6.19903904914856,
+      "entropy": 5.890115032196045,
       "epoch": 1.3529073114565342,
-      "grad_norm": 0.40436601638793945,
-      "learning_rate": 1.4760629004076878e-05,
-      "loss": 6.0244,
-      "mean_token_accuracy": 0.22900927513837815,
-      "num_tokens": 2199724.0,
+      "grad_norm": 2.738203525543213,
+      "learning_rate": 1.3630705394190872e-06,
+      "loss": 5.6555,
+      "mean_token_accuracy": 0.2737997192144394,
+      "num_tokens": 2476255.0,
       "step": 2350
     },
     {
-      "entropy": 5.986697297096253,
+      "entropy": 5.66056040763855,
       "epoch": 1.381692573402418,
-      "grad_norm": 0.8481882214546204,
-      "learning_rate": 1.464414676761794e-05,
-      "loss": 5.8195,
-      "mean_token_accuracy": 0.2552035376429558,
-      "num_tokens": 2245341.0,
+      "grad_norm": 3.1416995525360107,
+      "learning_rate": 1.3482513337285121e-06,
+      "loss": 5.4302,
+      "mean_token_accuracy": 0.3000989046692848,
+      "num_tokens": 2527674.0,
       "step": 2400
     },
     {
-      "entropy": 6.1886044692993165,
+      "entropy": 5.861240615844727,
       "epoch": 1.4104778353483016,
-      "grad_norm": 0.7911505103111267,
-      "learning_rate": 1.4527664531159e-05,
-      "loss": 6.0148,
-      "mean_token_accuracy": 0.23026730984449387,
-      "num_tokens": 2294726.0,
+      "grad_norm": 2.7569284439086914,
+      "learning_rate": 1.333432128037937e-06,
+      "loss": 5.6304,
+      "mean_token_accuracy": 0.27707513481378554,
+      "num_tokens": 2582909.0,
       "step": 2450
     },
     {
-      "entropy": 5.974867792129516,
+      "entropy": 5.627686910629272,
       "epoch": 1.4392630972941853,
-      "grad_norm": 1.640499234199524,
-      "learning_rate": 1.441118229470006e-05,
-      "loss": 5.8111,
-      "mean_token_accuracy": 0.2554209426045418,
-      "num_tokens": 2342251.0,
+      "grad_norm": 1.7750262022018433,
+      "learning_rate": 1.3186129223473621e-06,
+      "loss": 5.4058,
+      "mean_token_accuracy": 0.3019809901714325,
+      "num_tokens": 2636579.0,
       "step": 2500
     },
     {
-      "entropy": 5.967635660171509,
+      "entropy": 5.607026796340943,
       "epoch": 1.468048359240069,
-      "grad_norm": 0.8022929430007935,
-      "learning_rate": 1.429470005824112e-05,
-      "loss": 5.8015,
-      "mean_token_accuracy": 0.2569852137565613,
-      "num_tokens": 2387469.0,
+      "grad_norm": 3.1005160808563232,
+      "learning_rate": 1.3037937166567872e-06,
+      "loss": 5.3836,
+      "mean_token_accuracy": 0.30584611505270004,
+      "num_tokens": 2687698.0,
       "step": 2550
     },
     {
-      "entropy": 6.047262029647827,
+      "entropy": 5.6909641885757445,
       "epoch": 1.4968336211859528,
-      "grad_norm": 0.9270678758621216,
-      "learning_rate": 1.417821782178218e-05,
-      "loss": 5.8782,
-      "mean_token_accuracy": 0.2467849862575531,
-      "num_tokens": 2434128.0,
+      "grad_norm": 1.6848654747009277,
+      "learning_rate": 1.2889745109662123e-06,
+      "loss": 5.4653,
+      "mean_token_accuracy": 0.296178964972496,
+      "num_tokens": 2740214.0,
       "step": 2600
     },
     {
-      "entropy": 6.00601068019867,
+      "entropy": 5.619450302124023,
       "epoch": 1.5256188831318365,
-      "grad_norm": 1.5378597974777222,
-      "learning_rate": 1.406173558532324e-05,
-      "loss": 5.839,
-      "mean_token_accuracy": 0.25216978013515473,
-      "num_tokens": 2480366.0,
+      "grad_norm": 2.469539165496826,
+      "learning_rate": 1.274155305275637e-06,
+      "loss": 5.4022,
+      "mean_token_accuracy": 0.3039679077267647,
+      "num_tokens": 2792574.0,
       "step": 2650
     },
     {
-      "entropy": 5.988714299201965,
+      "entropy": 5.61073097705841,
       "epoch": 1.5544041450777202,
-      "grad_norm": 0.819143533706665,
-      "learning_rate": 1.3945253348864299e-05,
-      "loss": 5.82,
-      "mean_token_accuracy": 0.254311783015728,
-      "num_tokens": 2527357.0,
+      "grad_norm": 2.367810010910034,
+      "learning_rate": 1.259336099585062e-06,
+      "loss": 5.3956,
+      "mean_token_accuracy": 0.3051413372159004,
+      "num_tokens": 2845597.0,
       "step": 2700
     },
     {
-      "entropy": 5.960293846130371,
+      "entropy": 5.5791136837005615,
       "epoch": 1.583189407023604,
-      "grad_norm": 0.8920449614524841,
-      "learning_rate": 1.382877111240536e-05,
-      "loss": 5.7946,
-      "mean_token_accuracy": 0.25750755161046984,
-      "num_tokens": 2574470.0,
+      "grad_norm": 2.3874764442443848,
+      "learning_rate": 1.2445168938944872e-06,
+      "loss": 5.3676,
+      "mean_token_accuracy": 0.3068238252401352,
+      "num_tokens": 2898683.0,
       "step": 2750
     },
     {
-      "entropy": 6.1214879322052,
+      "entropy": 5.735381307601929,
       "epoch": 1.6119746689694876,
-      "grad_norm": 0.5333890914916992,
-      "learning_rate": 1.371228887594642e-05,
-      "loss": 5.9513,
-      "mean_token_accuracy": 0.2377367687225342,
-      "num_tokens": 2622280.0,
+      "grad_norm": 2.2097349166870117,
+      "learning_rate": 1.2296976882039123e-06,
+      "loss": 5.5239,
+      "mean_token_accuracy": 0.28974882304668426,
+      "num_tokens": 2952290.0,
       "step": 2800
     },
     {
-      "entropy": 5.951769871711731,
+      "entropy": 5.55252691745758,
       "epoch": 1.6407599309153713,
-      "grad_norm": 0.5994665026664734,
-      "learning_rate": 1.3595806639487479e-05,
-      "loss": 5.7861,
-      "mean_token_accuracy": 0.25854207515716554,
-      "num_tokens": 2668624.0,
+      "grad_norm": 1.694831132888794,
+      "learning_rate": 1.2148784825133372e-06,
+      "loss": 5.351,
+      "mean_token_accuracy": 0.3091904193162918,
+      "num_tokens": 3004556.0,
       "step": 2850
     },
     {
-      "entropy": 5.927765312194825,
+      "entropy": 5.508773093223572,
       "epoch": 1.669545192861255,
-      "grad_norm": 0.4460087716579437,
-      "learning_rate": 1.347932440302854e-05,
-      "loss": 5.7661,
-      "mean_token_accuracy": 0.25973255425691605,
-      "num_tokens": 2714388.0,
+      "grad_norm": 1.8229279518127441,
+      "learning_rate": 1.200059276822762e-06,
+      "loss": 5.3164,
+      "mean_token_accuracy": 0.31158645361661913,
+      "num_tokens": 3056448.0,
       "step": 2900
     },
     {
-      "entropy": 6.097678365707398,
+      "entropy": 5.676794271469117,
       "epoch": 1.6983304548071387,
-      "grad_norm": 0.7125752568244934,
-      "learning_rate": 1.3362842166569598e-05,
-      "loss": 5.9284,
-      "mean_token_accuracy": 0.23995368272066117,
-      "num_tokens": 2761465.0,
+      "grad_norm": 1.7196234464645386,
+      "learning_rate": 1.1852400711321872e-06,
+      "loss": 5.4776,
+      "mean_token_accuracy": 0.2929128894209862,
+      "num_tokens": 3109539.0,
       "step": 2950
     },
     {
-      "entropy": 5.986212658882141,
+      "entropy": 5.551529383659362,
       "epoch": 1.7271157167530224,
-      "grad_norm": 1.5405049324035645,
-      "learning_rate": 1.3246359930110659e-05,
-      "loss": 5.8194,
-      "mean_token_accuracy": 0.25333445996046067,
-      "num_tokens": 2808066.0,
+      "grad_norm": 3.117525577545166,
+      "learning_rate": 1.1704208654416123e-06,
+      "loss": 5.3561,
+      "mean_token_accuracy": 0.30634030640125276,
+      "num_tokens": 3162421.0,
       "step": 3000
     },
     {
-      "entropy": 5.7968806195259095,
+      "entropy": 5.379635264873505,
       "epoch": 1.7559009786989062,
-      "grad_norm": 0.4532749652862549,
-      "learning_rate": 1.312987769365172e-05,
-      "loss": 5.6344,
-      "mean_token_accuracy": 0.2782411390542984,
-      "num_tokens": 2851822.0,
+      "grad_norm": 1.876755714416504,
+      "learning_rate": 1.1556016597510372e-06,
+      "loss": 5.1868,
+      "mean_token_accuracy": 0.32913618892431257,
+      "num_tokens": 3212079.0,
       "step": 3050
     },
     {
-      "entropy": 5.973708114624023,
+      "entropy": 5.538804936408996,
       "epoch": 1.7846862406447899,
-      "grad_norm": 1.4795438051223755,
-      "learning_rate": 1.3013395457192778e-05,
-      "loss": 5.8104,
-      "mean_token_accuracy": 0.25441971331834795,
-      "num_tokens": 2897737.0,
+      "grad_norm": 1.8670976161956787,
+      "learning_rate": 1.1407824540604623e-06,
+      "loss": 5.3494,
+      "mean_token_accuracy": 0.30661171555519107,
+      "num_tokens": 3264089.0,
       "step": 3100
     },
     {
-      "entropy": 5.70733567237854,
+      "entropy": 5.258263626098633,
       "epoch": 1.8134715025906736,
-      "grad_norm": 0.6216577887535095,
-      "learning_rate": 1.2896913220733839e-05,
-      "loss": 5.5523,
-      "mean_token_accuracy": 0.28787180870771406,
-      "num_tokens": 2939511.0,
+      "grad_norm": 2.748718023300171,
+      "learning_rate": 1.1259632483698874e-06,
+      "loss": 5.08,
+      "mean_token_accuracy": 0.3413010013103485,
+      "num_tokens": 3311881.0,
       "step": 3150
     },
     {
-      "entropy": 5.96826630115509,
+      "entropy": 5.54539008140564,
       "epoch": 1.8422567645365573,
-      "grad_norm": 0.9246350526809692,
-      "learning_rate": 1.2780430984274898e-05,
-      "loss": 5.8057,
-      "mean_token_accuracy": 0.25464902341365814,
-      "num_tokens": 2986368.0,
+      "grad_norm": 1.8556406497955322,
+      "learning_rate": 1.1111440426793123e-06,
+      "loss": 5.3614,
+      "mean_token_accuracy": 0.30550685405731204,
+      "num_tokens": 3364861.0,
       "step": 3200
     },
     {
-      "entropy": 5.950662693977356,
+      "entropy": 5.5433073282241825,
       "epoch": 1.871042026482441,
-      "grad_norm": 0.8141199946403503,
-      "learning_rate": 1.266394874781596e-05,
-      "loss": 5.7886,
-      "mean_token_accuracy": 0.25830793648958206,
-      "num_tokens": 3031770.0,
+      "grad_norm": 1.8386749029159546,
+      "learning_rate": 1.0963248369887374e-06,
+      "loss": 5.3543,
+      "mean_token_accuracy": 0.30875524014234546,
+      "num_tokens": 3415911.0,
       "step": 3250
     },
     {
-      "entropy": 6.00512773513794,
+      "entropy": 5.5769769477844235,
       "epoch": 1.8998272884283247,
-      "grad_norm": 0.4913998246192932,
-      "learning_rate": 1.2547466511357018e-05,
-      "loss": 5.838,
-      "mean_token_accuracy": 0.2512077575922012,
-      "num_tokens": 3078322.0,
+      "grad_norm": 1.922486662864685,
+      "learning_rate": 1.0815056312981623e-06,
+      "loss": 5.3834,
+      "mean_token_accuracy": 0.3035113242268562,
+      "num_tokens": 3468338.0,
       "step": 3300
     },
     {
-      "entropy": 6.090880632400513,
+      "entropy": 5.640013842582703,
       "epoch": 1.9286125503742084,
-      "grad_norm": 0.9893012046813965,
-      "learning_rate": 1.243098427489808e-05,
-      "loss": 5.9264,
-      "mean_token_accuracy": 0.2391783133149147,
-      "num_tokens": 3125572.0,
+      "grad_norm": 2.179500102996826,
+      "learning_rate": 1.0666864256075874e-06,
+      "loss": 5.4574,
+      "mean_token_accuracy": 0.2947095710039139,
+      "num_tokens": 3521693.0,
       "step": 3350
     },
     {
-      "entropy": 5.949693293571472,
+      "entropy": 5.506910061836242,
       "epoch": 1.9573978123200921,
-      "grad_norm": 0.5794200301170349,
-      "learning_rate": 1.231450203843914e-05,
-      "loss": 5.7861,
-      "mean_token_accuracy": 0.2568664598464966,
-      "num_tokens": 3171974.0,
+      "grad_norm": 1.4014379978179932,
+      "learning_rate": 1.0518672199170125e-06,
+      "loss": 5.3234,
+      "mean_token_accuracy": 0.3096472260355949,
+      "num_tokens": 3574206.0,
       "step": 3400
     },
     {
-      "entropy": 6.03591317653656,
+      "entropy": 5.607311015129089,
       "epoch": 1.9861830742659758,
-      "grad_norm": 0.8525373339653015,
-      "learning_rate": 1.21980198019802e-05,
-      "loss": 5.8741,
-      "mean_token_accuracy": 0.24642003327608109,
-      "num_tokens": 3219624.0,
+      "grad_norm": 1.41231107711792,
+      "learning_rate": 1.0370480142264374e-06,
+      "loss": 5.4226,
+      "mean_token_accuracy": 0.2979922544956207,
+      "num_tokens": 3627807.0,
       "step": 3450
     },
     {
       "epoch": 2.0,
-      "eval_entropy": 6.272298685416648,
-      "eval_loss": 6.12472677230835,
-      "eval_mean_token_accuracy": 0.21168697409091458,
-      "eval_model_preparation_time": 0.0036,
-      "eval_num_tokens": 3239438.0,
-      "eval_runtime": 76.2536,
-      "eval_samples_per_second": 5.692,
-      "eval_steps_per_second": 2.846,
+      "eval_entropy": 5.831721861790951,
+      "eval_loss": 5.656307220458984,
+      "eval_mean_token_accuracy": 0.2641724460685308,
+      "eval_model_preparation_time": 0.0047,
+      "eval_num_tokens": 3650214.0,
+      "eval_runtime": 79.7324,
+      "eval_samples_per_second": 5.443,
+      "eval_steps_per_second": 2.722,
       "step": 3474
     },
     {
-      "entropy": 5.914763498306274,
+      "entropy": 5.477711625099182,
       "epoch": 2.0149683362118593,
-      "grad_norm": 0.5479806661605835,
-      "learning_rate": 1.208153756552126e-05,
-      "loss": 5.7559,
-      "mean_token_accuracy": 0.2624077323079109,
-      "num_tokens": 3263994.0,
+      "grad_norm": 3.0133137702941895,
+      "learning_rate": 1.0222288085358623e-06,
+      "loss": 5.2957,
+      "mean_token_accuracy": 0.31543311327695844,
+      "num_tokens": 3677883.0,
       "step": 3500
     },
     {
-      "entropy": 6.033470869064331,
+      "entropy": 5.599187393188476,
       "epoch": 2.043753598157743,
-      "grad_norm": 1.7186369895935059,
-      "learning_rate": 1.1965055329062319e-05,
-      "loss": 5.8677,
-      "mean_token_accuracy": 0.24745646148920059,
-      "num_tokens": 3311182.0,
+      "grad_norm": 1.885867714881897,
+      "learning_rate": 1.0074096028452874e-06,
+      "loss": 5.4142,
+      "mean_token_accuracy": 0.3004470923542976,
+      "num_tokens": 3730991.0,
       "step": 3550
     },
     {
-      "entropy": 5.962404427528381,
+      "entropy": 5.526448183059692,
       "epoch": 2.0725388601036268,
-      "grad_norm": 0.9068580269813538,
-      "learning_rate": 1.184857309260338e-05,
-      "loss": 5.8038,
-      "mean_token_accuracy": 0.25500513821840287,
-      "num_tokens": 3358036.0,
+      "grad_norm": 4.50788688659668,
+      "learning_rate": 9.925903971547125e-07,
+      "loss": 5.3517,
+      "mean_token_accuracy": 0.3069574165344238,
+      "num_tokens": 3783795.0,
       "step": 3600
     },
     {
-      "entropy": 5.995727968215943,
+      "entropy": 5.560557870864868,
       "epoch": 2.1013241220495105,
-      "grad_norm": 2.044490337371826,
-      "learning_rate": 1.1732090856144438e-05,
-      "loss": 5.8333,
-      "mean_token_accuracy": 0.2514388278126717,
-      "num_tokens": 3404058.0,
+      "grad_norm": 1.927862524986267,
+      "learning_rate": 9.777711914641374e-07,
+      "loss": 5.3815,
+      "mean_token_accuracy": 0.3045575937628746,
+      "num_tokens": 3835526.0,
       "step": 3650
     },
     {
-      "entropy": 5.981345901489258,
+      "entropy": 5.528058257102966,
       "epoch": 2.130109383995394,
-      "grad_norm": 0.5262818336486816,
-      "learning_rate": 1.1615608619685499e-05,
-      "loss": 5.8205,
-      "mean_token_accuracy": 0.2523340278863907,
-      "num_tokens": 3449834.0,
+      "grad_norm": 2.164687156677246,
+      "learning_rate": 9.629519857735625e-07,
+      "loss": 5.3501,
+      "mean_token_accuracy": 0.3071546205878258,
+      "num_tokens": 3887175.0,
       "step": 3700
     },
     {
-      "entropy": 5.848710675239563,
+      "entropy": 5.397617678642273,
       "epoch": 2.158894645941278,
-      "grad_norm": 0.726718544960022,
-      "learning_rate": 1.149912638322656e-05,
-      "loss": 5.6891,
-      "mean_token_accuracy": 0.2697497832775116,
-      "num_tokens": 3494740.0,
+      "grad_norm": 2.3098385334014893,
+      "learning_rate": 9.481327800829875e-07,
+      "loss": 5.2244,
+      "mean_token_accuracy": 0.3226669803261757,
+      "num_tokens": 3938003.0,
       "step": 3750
     },
     {
-      "entropy": 5.964878315925598,
+      "entropy": 5.529960298538208,
       "epoch": 2.1876799078871616,
-      "grad_norm": 0.6147393584251404,
-      "learning_rate": 1.1382644146767618e-05,
-      "loss": 5.8029,
-      "mean_token_accuracy": 0.2553535890579224,
-      "num_tokens": 3541342.0,
+      "grad_norm": 1.8144755363464355,
+      "learning_rate": 9.333135743924125e-07,
+      "loss": 5.3572,
+      "mean_token_accuracy": 0.306032218337059,
+      "num_tokens": 3990451.0,
       "step": 3800
     },
     {
-      "entropy": 6.045858116149902,
+      "entropy": 5.597109637260437,
       "epoch": 2.2164651698330453,
-      "grad_norm": 0.8283621072769165,
-      "learning_rate": 1.1266161910308679e-05,
-      "loss": 5.8802,
-      "mean_token_accuracy": 0.24544916599988936,
-      "num_tokens": 3588995.0,
+      "grad_norm": 2.7306935787200928,
+      "learning_rate": 9.184943687018375e-07,
+      "loss": 5.4162,
+      "mean_token_accuracy": 0.2985941395163536,
+      "num_tokens": 4044048.0,
       "step": 3850
     },
     {
-      "entropy": 5.909895505905151,
+      "entropy": 5.448684883117676,
       "epoch": 2.245250431778929,
-      "grad_norm": 0.9912867546081543,
-      "learning_rate": 1.1149679673849738e-05,
-      "loss": 5.7481,
-      "mean_token_accuracy": 0.2620398569107056,
-      "num_tokens": 3634252.0,
+      "grad_norm": 1.8199880123138428,
+      "learning_rate": 9.036751630112626e-07,
+      "loss": 5.2775,
+      "mean_token_accuracy": 0.31548845052719116,
+      "num_tokens": 4095276.0,
       "step": 3900
     },
     {
-      "entropy": 5.9534005498886104,
+      "entropy": 5.5008597612380985,
       "epoch": 2.2740356937248127,
-      "grad_norm": 1.2012401819229126,
-      "learning_rate": 1.1033197437390799e-05,
-      "loss": 5.788,
-      "mean_token_accuracy": 0.25642816990613937,
-      "num_tokens": 3681197.0,
+      "grad_norm": 1.755323052406311,
+      "learning_rate": 8.888559573206875e-07,
+      "loss": 5.3274,
+      "mean_token_accuracy": 0.309090721309185,
+      "num_tokens": 4148172.0,
       "step": 3950
     },
     {
-      "entropy": 6.155718851089477,
+      "entropy": 5.7040300464630125,
       "epoch": 2.3028209556706964,
-      "grad_norm": 1.4272509813308716,
-      "learning_rate": 1.0916715200931857e-05,
-      "loss": 5.9842,
-      "mean_token_accuracy": 0.23176315426826477,
-      "num_tokens": 3729955.0,
+      "grad_norm": 2.3154356479644775,
+      "learning_rate": 8.740367516301126e-07,
+      "loss": 5.5239,
+      "mean_token_accuracy": 0.28589318484067916,
+      "num_tokens": 4202733.0,
       "step": 4000
     },
     {
-      "entropy": 6.004842009544372,
+      "entropy": 5.549855670928955,
       "epoch": 2.33160621761658,
-      "grad_norm": 1.1919596195220947,
-      "learning_rate": 1.0800232964472918e-05,
-      "loss": 5.8332,
-      "mean_token_accuracy": 0.25039500594139097,
-      "num_tokens": 3777043.0,
+      "grad_norm": 1.9549669027328491,
+      "learning_rate": 8.592175459395375e-07,
+      "loss": 5.3755,
+      "mean_token_accuracy": 0.3029727828502655,
+      "num_tokens": 4255738.0,
       "step": 4050
     },
     {
-      "entropy": 6.045269584655761,
+      "entropy": 5.579690465927124,
       "epoch": 2.360391479562464,
-      "grad_norm": 0.6200748085975647,
-      "learning_rate": 1.068375072801398e-05,
-      "loss": 5.8641,
-      "mean_token_accuracy": 0.2466951721906662,
-      "num_tokens": 3824067.0,
+      "grad_norm": 1.7018866539001465,
+      "learning_rate": 8.443983402489626e-07,
+      "loss": 5.4036,
+      "mean_token_accuracy": 0.3001995691657066,
+      "num_tokens": 4308638.0,
       "step": 4100
     },
     {
-      "entropy": 6.105137758255005,
+      "entropy": 5.646504878997803,
       "epoch": 2.3891767415083476,
-      "grad_norm": 1.0185531377792358,
-      "learning_rate": 1.0567268491555038e-05,
-      "loss": 5.9181,
-      "mean_token_accuracy": 0.24000227689743042,
-      "num_tokens": 3872769.0,
+      "grad_norm": 1.4139262437820435,
+      "learning_rate": 8.295791345583877e-07,
+      "loss": 5.4733,
+      "mean_token_accuracy": 0.2912476986646652,
+      "num_tokens": 4363170.0,
       "step": 4150
     },
     {
-      "entropy": 6.013391451835632,
+      "entropy": 5.554990992546082,
       "epoch": 2.4179620034542313,
-      "grad_norm": 0.6188511848449707,
-      "learning_rate": 1.04507862550961e-05,
-      "loss": 5.8286,
-      "mean_token_accuracy": 0.25189226895570754,
-      "num_tokens": 3919379.0,
+      "grad_norm": 1.6886577606201172,
+      "learning_rate": 8.147599288678126e-07,
+      "loss": 5.3842,
+      "mean_token_accuracy": 0.302762059867382,
+      "num_tokens": 4415607.0,
       "step": 4200
     },
     {
-      "entropy": 5.972923498153687,
+      "entropy": 5.513420124053955,
       "epoch": 2.446747265400115,
-      "grad_norm": 0.7165982127189636,
-      "learning_rate": 1.0334304018637157e-05,
-      "loss": 5.7908,
-      "mean_token_accuracy": 0.2567197346687317,
-      "num_tokens": 3965593.0,
+      "grad_norm": 1.3537819385528564,
+      "learning_rate": 7.999407231772377e-07,
+      "loss": 5.3408,
+      "mean_token_accuracy": 0.30764526218175886,
+      "num_tokens": 4467608.0,
       "step": 4250
     },
     {
-      "entropy": 6.0378124713897705,
+      "entropy": 5.561378569602966,
       "epoch": 2.4755325273459987,
-      "grad_norm": 0.5278330445289612,
-      "learning_rate": 1.021782178217822e-05,
-      "loss": 5.8559,
-      "mean_token_accuracy": 0.2484271454811096,
-      "num_tokens": 4012300.0,
+      "grad_norm": 1.8514106273651123,
+      "learning_rate": 7.851215174866627e-07,
+      "loss": 5.3891,
+      "mean_token_accuracy": 0.301382859647274,
+      "num_tokens": 4520299.0,
       "step": 4300
     },
     {
-      "entropy": 5.984496111869812,
+      "entropy": 5.536689953804016,
       "epoch": 2.5043177892918824,
-      "grad_norm": 0.8995006680488586,
-      "learning_rate": 1.0101339545719278e-05,
-      "loss": 5.8092,
-      "mean_token_accuracy": 0.253717774450779,
-      "num_tokens": 4059323.0,
+      "grad_norm": 2.1830835342407227,
+      "learning_rate": 7.703023117960877e-07,
+      "loss": 5.3672,
+      "mean_token_accuracy": 0.3047756373882294,
+      "num_tokens": 4573065.0,
       "step": 4350
     },
     {
-      "entropy": 6.124767150878906,
+      "entropy": 5.69776873588562,
       "epoch": 2.533103051237766,
-      "grad_norm": 1.3810409307479858,
-      "learning_rate": 9.984857309260339e-06,
-      "loss": 5.9468,
-      "mean_token_accuracy": 0.23715158700942993,
-      "num_tokens": 4107616.0,
+      "grad_norm": 1.999536156654358,
+      "learning_rate": 7.554831061055127e-07,
+      "loss": 5.5236,
+      "mean_token_accuracy": 0.2868007507920265,
+      "num_tokens": 4626807.0,
       "step": 4400
     },
     {
-      "entropy": 5.8810745000839235,
+      "entropy": 5.3977436876297,
       "epoch": 2.56188831318365,
-      "grad_norm": 0.8794332146644592,
-      "learning_rate": 9.868375072801398e-06,
-      "loss": 5.7089,
-      "mean_token_accuracy": 0.2662400561571121,
-      "num_tokens": 4152400.0,
+      "grad_norm": 1.9608020782470703,
+      "learning_rate": 7.406639004149378e-07,
+      "loss": 5.2335,
+      "mean_token_accuracy": 0.3199601462483406,
+      "num_tokens": 4677663.0,
       "step": 4450
     },
     {
-      "entropy": 6.108017959594727,
+      "entropy": 5.6681678771972654,
       "epoch": 2.5906735751295336,
-      "grad_norm": 0.5132983922958374,
-      "learning_rate": 9.751892836342458e-06,
-      "loss": 5.9346,
-      "mean_token_accuracy": 0.23871887892484664,
-      "num_tokens": 4200994.0,
+      "grad_norm": 1.829047441482544,
+      "learning_rate": 7.258446947243627e-07,
+      "loss": 5.491,
+      "mean_token_accuracy": 0.2894612854719162,
+      "num_tokens": 4731830.0,
       "step": 4500
     },
     {
-      "entropy": 5.985005149841308,
+      "entropy": 5.49174174785614,
       "epoch": 2.6194588370754173,
-      "grad_norm": 0.6561470031738281,
-      "learning_rate": 9.635410599883519e-06,
-      "loss": 5.8111,
-      "mean_token_accuracy": 0.25315980523824694,
-      "num_tokens": 4247548.0,
+      "grad_norm": 1.3158719539642334,
+      "learning_rate": 7.110254890337878e-07,
+      "loss": 5.3225,
+      "mean_token_accuracy": 0.3084965732693672,
+      "num_tokens": 4784694.0,
       "step": 4550
     },
     {
-      "entropy": 6.050709452629089,
+      "entropy": 5.573234438896179,
       "epoch": 2.648244099021301,
-      "grad_norm": 0.8790570497512817,
-      "learning_rate": 9.51892836342458e-06,
-      "loss": 5.8789,
-      "mean_token_accuracy": 0.2440834751725197,
-      "num_tokens": 4295250.0,
+      "grad_norm": 1.562915325164795,
+      "learning_rate": 6.962062833432127e-07,
+      "loss": 5.4028,
+      "mean_token_accuracy": 0.2989520016312599,
+      "num_tokens": 4838534.0,
       "step": 4600
     },
     {
-      "entropy": 6.007251596450805,
+      "entropy": 5.550469598770142,
       "epoch": 2.6770293609671847,
-      "grad_norm": 0.6728562116622925,
-      "learning_rate": 9.402446126965639e-06,
-      "loss": 5.8338,
-      "mean_token_accuracy": 0.2509264424443245,
-      "num_tokens": 4341599.0,
+      "grad_norm": 2.114727735519409,
+      "learning_rate": 6.813870776526378e-07,
+      "loss": 5.3804,
+      "mean_token_accuracy": 0.30373542964458466,
+      "num_tokens": 4890611.0,
       "step": 4650
     },
     {
-      "entropy": 5.966628184318543,
+      "entropy": 5.523049550056458,
       "epoch": 2.7058146229130684,
-      "grad_norm": 0.5815795063972473,
-      "learning_rate": 9.285963890506699e-06,
-      "loss": 5.7961,
-      "mean_token_accuracy": 0.2559360232949257,
-      "num_tokens": 4388673.0,
+      "grad_norm": 2.5036823749542236,
+      "learning_rate": 6.665678719620628e-07,
+      "loss": 5.3542,
+      "mean_token_accuracy": 0.30681024432182313,
+      "num_tokens": 4943571.0,
       "step": 4700
     },
     {
-      "entropy": 5.7972593069076535,
+      "entropy": 5.323453049659729,
       "epoch": 2.734599884858952,
-      "grad_norm": 1.0610334873199463,
-      "learning_rate": 9.169481654047758e-06,
-      "loss": 5.6318,
-      "mean_token_accuracy": 0.27574603259563446,
-      "num_tokens": 4432959.0,
+      "grad_norm": 1.8069168329238892,
+      "learning_rate": 6.517486662714878e-07,
+      "loss": 5.1583,
+      "mean_token_accuracy": 0.32906652927398683,
+      "num_tokens": 4993871.0,
       "step": 4750
     },
     {
-      "entropy": 5.984181261062622,
+      "entropy": 5.504038324356079,
       "epoch": 2.763385146804836,
-      "grad_norm": 2.1847357749938965,
-      "learning_rate": 9.052999417588819e-06,
-      "loss": 5.8153,
-      "mean_token_accuracy": 0.2533784031867981,
-      "num_tokens": 4479190.0,
+      "grad_norm": 4.750283718109131,
+      "learning_rate": 6.369294605809128e-07,
+      "loss": 5.3366,
+      "mean_token_accuracy": 0.3087608867883682,
+      "num_tokens": 5046187.0,
       "step": 4800
     },
     {
-      "entropy": 5.959725599288941,
+      "entropy": 5.487624549865723,
       "epoch": 2.7921704087507195,
-      "grad_norm": 0.5671709179878235,
-      "learning_rate": 8.936517181129878e-06,
-      "loss": 5.7912,
-      "mean_token_accuracy": 0.2556650054454803,
-      "num_tokens": 4525674.0,
+      "grad_norm": 1.4186172485351562,
+      "learning_rate": 6.221102548903379e-07,
+      "loss": 5.3237,
+      "mean_token_accuracy": 0.3088638699054718,
+      "num_tokens": 5098644.0,
       "step": 4850
     },
     {
-      "entropy": 5.814929313659668,
+      "entropy": 5.346905107498169,
       "epoch": 2.8209556706966032,
-      "grad_norm": 0.9447108507156372,
-      "learning_rate": 8.820034944670938e-06,
-      "loss": 5.6478,
-      "mean_token_accuracy": 0.27417868226766584,
-      "num_tokens": 4570379.0,
+      "grad_norm": 1.5670177936553955,
+      "learning_rate": 6.072910491997628e-07,
+      "loss": 5.1849,
+      "mean_token_accuracy": 0.3265886321663857,
+      "num_tokens": 5149345.0,
       "step": 4900
     },
     {
-      "entropy": 5.96754421710968,
+      "entropy": 5.510410032272339,
       "epoch": 2.849740932642487,
-      "grad_norm": 2.009676218032837,
-      "learning_rate": 8.703552708211999e-06,
-      "loss": 5.795,
-      "mean_token_accuracy": 0.2556305864453316,
-      "num_tokens": 4617184.0,
+      "grad_norm": 7.489855766296387,
+      "learning_rate": 5.924718435091879e-07,
+      "loss": 5.3424,
+      "mean_token_accuracy": 0.30768151730299,
+      "num_tokens": 5202028.0,
       "step": 4950
     },
     {
-      "entropy": 6.008112049102783,
+      "entropy": 5.525181493759155,
       "epoch": 2.8785261945883707,
-      "grad_norm": 1.1977978944778442,
-      "learning_rate": 8.587070471753058e-06,
-      "loss": 5.8416,
-      "mean_token_accuracy": 0.2494604030251503,
-      "num_tokens": 4664180.0,
+      "grad_norm": 1.8829196691513062,
+      "learning_rate": 5.776526378186128e-07,
+      "loss": 5.3654,
+      "mean_token_accuracy": 0.30342737555503846,
+      "num_tokens": 5255082.0,
       "step": 5000
     },
     {
-      "entropy": 5.832320966720581,
+      "entropy": 5.374098634719848,
       "epoch": 2.9073114565342544,
-      "grad_norm": 0.4845636785030365,
-      "learning_rate": 8.470588235294118e-06,
-      "loss": 5.6672,
-      "mean_token_accuracy": 0.27187123566865923,
-      "num_tokens": 4708377.0,
+      "grad_norm": 1.3901060819625854,
+      "learning_rate": 5.628334321280379e-07,
+      "loss": 5.2103,
+      "mean_token_accuracy": 0.3233291879296303,
+      "num_tokens": 5305042.0,
       "step": 5050
     },
     {
-      "entropy": 5.84138514995575,
+      "entropy": 5.374619431495667,
       "epoch": 2.936096718480138,
-      "grad_norm": 0.8487229943275452,
-      "learning_rate": 8.354105998835179e-06,
-      "loss": 5.6769,
-      "mean_token_accuracy": 0.26995211571455,
-      "num_tokens": 4753587.0,
+      "grad_norm": 1.6586560010910034,
+      "learning_rate": 5.48014226437463e-07,
+      "loss": 5.2125,
+      "mean_token_accuracy": 0.322759662270546,
+      "num_tokens": 5356310.0,
       "step": 5100
     },
     {
-      "entropy": 6.016681690216064,
+      "entropy": 5.527479724884033,
       "epoch": 2.964881980426022,
-      "grad_norm": 0.9554332494735718,
-      "learning_rate": 8.237623762376238e-06,
-      "loss": 5.8479,
-      "mean_token_accuracy": 0.24785644590854644,
-      "num_tokens": 4800508.0,
+      "grad_norm": 1.6678485870361328,
+      "learning_rate": 5.331950207468879e-07,
+      "loss": 5.3627,
+      "mean_token_accuracy": 0.30430852621793747,
+      "num_tokens": 5409283.0,
       "step": 5150
     },
     {
-      "entropy": 6.103472499847412,
+      "entropy": 5.6171248292922975,
       "epoch": 2.9936672423719055,
-      "grad_norm": 0.6602863669395447,
-      "learning_rate": 8.121141525917298e-06,
-      "loss": 5.9305,
-      "mean_token_accuracy": 0.23794592499732972,
-      "num_tokens": 4849415.0,
+      "grad_norm": 1.50790274143219,
+      "learning_rate": 5.18375815056313e-07,
+      "loss": 5.4484,
+      "mean_token_accuracy": 0.29375598043203355,
+      "num_tokens": 5464332.0,
       "step": 5200
     },
     {
       "epoch": 3.0,
-      "eval_entropy": 6.254081044878278,
-      "eval_loss": 6.0980024337768555,
-      "eval_mean_token_accuracy": 0.21401402258103894,
-      "eval_model_preparation_time": 0.0036,
-      "eval_num_tokens": 4859157.0,
-      "eval_runtime": 75.9443,
-      "eval_samples_per_second": 5.715,
-      "eval_steps_per_second": 2.857,
+      "eval_entropy": 5.78779639186947,
+      "eval_loss": 5.628758430480957,
+      "eval_mean_token_accuracy": 0.2653660801698535,
+      "eval_model_preparation_time": 0.0047,
+      "eval_num_tokens": 5475321.0,
+      "eval_runtime": 80.3676,
+      "eval_samples_per_second": 5.4,
+      "eval_steps_per_second": 2.7,
       "step": 5211
     },
     {
-      "entropy": 5.829766502380371,
+      "entropy": 5.323350539207459,
       "epoch": 3.0224525043177892,
-      "grad_norm": 0.5663251280784607,
-      "learning_rate": 8.004659289458359e-06,
-      "loss": 5.6641,
-      "mean_token_accuracy": 0.27141522347927094,
-      "num_tokens": 4893297.0,
+      "grad_norm": 2.033228635787964,
+      "learning_rate": 5.03556609365738e-07,
+      "loss": 5.1623,
+      "mean_token_accuracy": 0.32844111531972886,
+      "num_tokens": 5514450.0,
       "step": 5250
     },
     {
-      "entropy": 5.987464437484741,
+      "entropy": 5.509175033569336,
       "epoch": 3.051237766263673,
-      "grad_norm": 0.7494759559631348,
-      "learning_rate": 7.888177052999418e-06,
-      "loss": 5.8163,
-      "mean_token_accuracy": 0.2527216270565987,
-      "num_tokens": 4940190.0,
+      "grad_norm": 1.4281281232833862,
+      "learning_rate": 4.88737403675163e-07,
+      "loss": 5.3403,
+      "mean_token_accuracy": 0.30768867909908293,
+      "num_tokens": 5567345.0,
       "step": 5300
     },
     {
-      "entropy": 5.922745175361634,
+      "entropy": 5.4536163854599,
       "epoch": 3.0800230282095566,
-      "grad_norm": 1.0836946964263916,
-      "learning_rate": 7.771694816540478e-06,
-      "loss": 5.7536,
-      "mean_token_accuracy": 0.26056944400072096,
-      "num_tokens": 4986555.0,
+      "grad_norm": 2.0320699214935303,
+      "learning_rate": 4.73918197984588e-07,
+      "loss": 5.2898,
+      "mean_token_accuracy": 0.31407355904579165,
+      "num_tokens": 5619654.0,
       "step": 5350
     },
     {
-      "entropy": 5.940353560447693,
+      "entropy": 5.487306084632873,
       "epoch": 3.1088082901554404,
-      "grad_norm": 0.9733691811561584,
-      "learning_rate": 7.655212580081537e-06,
-      "loss": 5.7726,
-      "mean_token_accuracy": 0.25769122928380966,
-      "num_tokens": 5033343.0,
+      "grad_norm": 1.2829618453979492,
+      "learning_rate": 4.59098992294013e-07,
+      "loss": 5.3204,
+      "mean_token_accuracy": 0.30913869380950926,
+      "num_tokens": 5672269.0,
       "step": 5400
     },
     {
-      "entropy": 6.020898208618164,
+      "entropy": 5.569495844841003,
       "epoch": 3.137593552101324,
-      "grad_norm": 0.7032522559165955,
-      "learning_rate": 7.538730343622598e-06,
-      "loss": 5.8525,
-      "mean_token_accuracy": 0.2485825625061989,
-      "num_tokens": 5080498.0,
+      "grad_norm": 2.231628656387329,
+      "learning_rate": 4.44279786603438e-07,
+      "loss": 5.4045,
+      "mean_token_accuracy": 0.30076681196689603,
+      "num_tokens": 5725059.0,
       "step": 5450
     },
     {
-      "entropy": 5.953487596511841,
+      "entropy": 5.499957413673401,
       "epoch": 3.166378814047208,
-      "grad_norm": 0.34246090054512024,
-      "learning_rate": 7.422248107163658e-06,
-      "loss": 5.7907,
-      "mean_token_accuracy": 0.25604957044124604,
-      "num_tokens": 5126524.0,
+      "grad_norm": 1.549865484237671,
+      "learning_rate": 4.2946058091286305e-07,
+      "loss": 5.3415,
+      "mean_token_accuracy": 0.30755339056253433,
+      "num_tokens": 5776784.0,
       "step": 5500
     },
     {
-      "entropy": 6.124787425994873,
+      "entropy": 5.664071002006531,
       "epoch": 3.1951640759930915,
-      "grad_norm": 0.8139322400093079,
-      "learning_rate": 7.305765870704718e-06,
-      "loss": 5.9528,
-      "mean_token_accuracy": 0.23523027300834656,
-      "num_tokens": 5176187.0,
+      "grad_norm": 1.2153443098068237,
+      "learning_rate": 4.146413752222881e-07,
+      "loss": 5.4948,
+      "mean_token_accuracy": 0.28785294711589815,
+      "num_tokens": 5832296.0,
       "step": 5550
     },
     {
-      "entropy": 5.990889682769775,
+      "entropy": 5.516234860420227,
       "epoch": 3.223949337938975,
-      "grad_norm": 0.47230103611946106,
-      "learning_rate": 7.189283634245778e-06,
-      "loss": 5.8239,
-      "mean_token_accuracy": 0.2525310072302818,
-      "num_tokens": 5223101.0,
+      "grad_norm": 1.0542709827423096,
+      "learning_rate": 3.998221695317131e-07,
+      "loss": 5.3465,
+      "mean_token_accuracy": 0.3083792108297348,
+      "num_tokens": 5885122.0,
       "step": 5600
     },
     {
-      "entropy": 5.9849296569824215,
+      "entropy": 5.500826091766357,
       "epoch": 3.252734599884859,
-      "grad_norm": 1.0184181928634644,
-      "learning_rate": 7.072801397786838e-06,
-      "loss": 5.8195,
-      "mean_token_accuracy": 0.25241400361061095,
-      "num_tokens": 5270200.0,
+      "grad_norm": 2.2477681636810303,
+      "learning_rate": 3.850029638411381e-07,
+      "loss": 5.3385,
+      "mean_token_accuracy": 0.30737883657217024,
+      "num_tokens": 5938386.0,
       "step": 5650
     },
     {
-      "entropy": 5.9844825649261475,
+      "entropy": 5.517533864974975,
       "epoch": 3.2815198618307426,
-      "grad_norm": 0.8858366012573242,
-      "learning_rate": 6.956319161327898e-06,
-      "loss": 5.8149,
-      "mean_token_accuracy": 0.2527842208743095,
-      "num_tokens": 5315872.0,
+      "grad_norm": 1.03904128074646,
+      "learning_rate": 3.7018375815056315e-07,
+      "loss": 5.3533,
+      "mean_token_accuracy": 0.3064529225230217,
+      "num_tokens": 5989784.0,
       "step": 5700
     },
     {
-      "entropy": 6.0137806224823,
+      "entropy": 5.543709697723389,
       "epoch": 3.3103051237766263,
-      "grad_norm": 1.8156790733337402,
-      "learning_rate": 6.839836924868957e-06,
-      "loss": 5.8448,
-      "mean_token_accuracy": 0.24918658077716827,
-      "num_tokens": 5362860.0,
+      "grad_norm": 1.562757134437561,
+      "learning_rate": 3.5536455245998815e-07,
+      "loss": 5.3766,
+      "mean_token_accuracy": 0.3036728450655937,
+      "num_tokens": 6042646.0,
       "step": 5750
     },
     {
-      "entropy": 5.8676600885391235,
+      "entropy": 5.389412899017334,
       "epoch": 3.33909038572251,
-      "grad_norm": 0.5497516393661499,
-      "learning_rate": 6.723354688410018e-06,
-      "loss": 5.7041,
-      "mean_token_accuracy": 0.2672875428199768,
-      "num_tokens": 5407854.0,
+      "grad_norm": 2.2124178409576416,
+      "learning_rate": 3.4054534676941315e-07,
+      "loss": 5.2287,
+      "mean_token_accuracy": 0.32173423111438754,
+      "num_tokens": 6093550.0,
       "step": 5800
     },
     {
-      "entropy": 5.702701902389526,
+      "entropy": 5.236968355178833,
       "epoch": 3.3678756476683938,
-      "grad_norm": 1.8813326358795166,
-      "learning_rate": 6.606872451951079e-06,
-      "loss": 5.5403,
-      "mean_token_accuracy": 0.2881160417199135,
-      "num_tokens": 5450830.0,
+      "grad_norm": 2.146965503692627,
+      "learning_rate": 3.2572614107883814e-07,
+      "loss": 5.0793,
+      "mean_token_accuracy": 0.3410212889313698,
+      "num_tokens": 6142299.0,
       "step": 5850
     },
     {
-      "entropy": 5.931481714248657,
+      "entropy": 5.459367966651916,
       "epoch": 3.3966609096142775,
-      "grad_norm": 0.664723813533783,
-      "learning_rate": 6.490390215492138e-06,
-      "loss": 5.7643,
-      "mean_token_accuracy": 0.2589978861808777,
-      "num_tokens": 5497011.0,
+      "grad_norm": 1.0992231369018555,
+      "learning_rate": 3.109069353882632e-07,
+      "loss": 5.2978,
+      "mean_token_accuracy": 0.31258249312639236,
+      "num_tokens": 6194315.0,
       "step": 5900
     },
     {
-      "entropy": 5.998486938476563,
+      "entropy": 5.526850900650024,
       "epoch": 3.425446171560161,
-      "grad_norm": 0.8670396208763123,
-      "learning_rate": 6.373907979033198e-06,
-      "loss": 5.8296,
-      "mean_token_accuracy": 0.2514311093091965,
-      "num_tokens": 5542914.0,
+      "grad_norm": 2.137270212173462,
+      "learning_rate": 2.960877296976882e-07,
+      "loss": 5.3598,
+      "mean_token_accuracy": 0.3052875977754593,
+      "num_tokens": 6246032.0,
       "step": 5950
     },
     {
-      "entropy": 6.03099499464035,
+      "entropy": 5.573816101551056,
       "epoch": 3.454231433506045,
-      "grad_norm": 0.6441876292228699,
-      "learning_rate": 6.257425742574258e-06,
-      "loss": 5.865,
-      "mean_token_accuracy": 0.2457648393511772,
-      "num_tokens": 5591031.0,
+      "grad_norm": 1.5624985694885254,
+      "learning_rate": 2.812685240071132e-07,
+      "loss": 5.4081,
+      "mean_token_accuracy": 0.2992635017633438,
+      "num_tokens": 6300018.0,
       "step": 6000
     },
     {
-      "entropy": 5.982430481910706,
+      "entropy": 5.514087476730347,
       "epoch": 3.4830166954519286,
-      "grad_norm": 0.8924009799957275,
-      "learning_rate": 6.140943506115318e-06,
-      "loss": 5.8162,
-      "mean_token_accuracy": 0.2524935993552208,
-      "num_tokens": 5638071.0,
+      "grad_norm": 1.2660338878631592,
+      "learning_rate": 2.664493183165382e-07,
+      "loss": 5.3472,
+      "mean_token_accuracy": 0.3070674228668213,
+      "num_tokens": 6352988.0,
       "step": 6050
     },
     {
-      "entropy": 5.9037020778656,
+      "entropy": 5.430188207626343,
       "epoch": 3.5118019573978123,
-      "grad_norm": 0.6898691654205322,
-      "learning_rate": 6.024461269656377e-06,
-      "loss": 5.7369,
-      "mean_token_accuracy": 0.2629904666543007,
-      "num_tokens": 5684251.0,
+      "grad_norm": 1.2666460275650024,
+      "learning_rate": 2.5163011262596324e-07,
+      "loss": 5.2645,
+      "mean_token_accuracy": 0.31776045858860014,
+      "num_tokens": 6405116.0,
       "step": 6100
     },
     {
-      "entropy": 6.049537987709045,
+      "entropy": 5.5897090005874634,
       "epoch": 3.540587219343696,
-      "grad_norm": 0.7176857590675354,
-      "learning_rate": 5.907979033197437e-06,
-      "loss": 5.8796,
-      "mean_token_accuracy": 0.24504777789115906,
-      "num_tokens": 5732144.0,
+      "grad_norm": 1.275363802909851,
+      "learning_rate": 2.3681090693538824e-07,
+      "loss": 5.4265,
+      "mean_token_accuracy": 0.297469447851181,
+      "num_tokens": 6458789.0,
       "step": 6150
     },
     {
-      "entropy": 5.896630597114563,
+      "entropy": 5.422791337966919,
       "epoch": 3.5693724812895797,
-      "grad_norm": 0.2607983350753784,
-      "learning_rate": 5.7914967967384986e-06,
-      "loss": 5.7322,
-      "mean_token_accuracy": 0.2642005959153175,
-      "num_tokens": 5777711.0,
+      "grad_norm": 2.2392683029174805,
+      "learning_rate": 2.2199170124481327e-07,
+      "loss": 5.2608,
+      "mean_token_accuracy": 0.3180572906136513,
+      "num_tokens": 6510168.0,
       "step": 6200
     },
     {
-      "entropy": 5.914587182998657,
+      "entropy": 5.408909387588501,
       "epoch": 3.5981577432354634,
-      "grad_norm": 1.745258092880249,
-      "learning_rate": 5.675014560279558e-06,
-      "loss": 5.75,
-      "mean_token_accuracy": 0.26022892773151396,
-      "num_tokens": 5823662.0,
+      "grad_norm": 2.821279525756836,
+      "learning_rate": 2.071724955542383e-07,
+      "loss": 5.2455,
+      "mean_token_accuracy": 0.316647432744503,
+      "num_tokens": 6562528.0,
       "step": 6250
     },
     {
-      "entropy": 6.1133457374572755,
+      "entropy": 5.657666215896606,
       "epoch": 3.626943005181347,
-      "grad_norm": 1.2074909210205078,
-      "learning_rate": 5.558532323820618e-06,
-      "loss": 5.9456,
-      "mean_token_accuracy": 0.2363065341114998,
-      "num_tokens": 5872695.0,
+      "grad_norm": 3.261878490447998,
+      "learning_rate": 1.9235328986366332e-07,
+      "loss": 5.4941,
+      "mean_token_accuracy": 0.28845800429582596,
+      "num_tokens": 6617308.0,
       "step": 6300
     },
     {
-      "entropy": 5.927231726646423,
+      "entropy": 5.446933870315552,
       "epoch": 3.655728267127231,
-      "grad_norm": 0.4929303824901581,
-      "learning_rate": 5.442050087361678e-06,
-      "loss": 5.7627,
-      "mean_token_accuracy": 0.2595584252476692,
-      "num_tokens": 5919280.0,
+      "grad_norm": 1.1171406507492065,
+      "learning_rate": 1.7753408417308832e-07,
+      "loss": 5.2848,
+      "mean_token_accuracy": 0.31402444154024123,
+      "num_tokens": 6669969.0,
       "step": 6350
     },
     {
-      "entropy": 6.085461645126343,
+      "entropy": 5.605754513740539,
       "epoch": 3.6845135290731146,
-      "grad_norm": 0.6310611367225647,
-      "learning_rate": 5.325567850902738e-06,
-      "loss": 5.9152,
-      "mean_token_accuracy": 0.23941247612237931,
-      "num_tokens": 5967702.0,
+      "grad_norm": 2.066650152206421,
+      "learning_rate": 1.6271487848251334e-07,
+      "loss": 5.4447,
+      "mean_token_accuracy": 0.2945487481355667,
+      "num_tokens": 6724425.0,
       "step": 6400
     },
     {
-      "entropy": 5.884929132461548,
+      "entropy": 5.39195601940155,
       "epoch": 3.7132987910189983,
-      "grad_norm": 1.1140141487121582,
-      "learning_rate": 5.209085614443797e-06,
-      "loss": 5.7183,
-      "mean_token_accuracy": 0.26445025473833084,
-      "num_tokens": 6012476.0,
+      "grad_norm": 1.6908842325210571,
+      "learning_rate": 1.4789567279193834e-07,
+      "loss": 5.2298,
+      "mean_token_accuracy": 0.3206364804506302,
+      "num_tokens": 6775236.0,
       "step": 6450
     },
     {
-      "entropy": 5.980639338493347,
+      "entropy": 5.514347395896912,
       "epoch": 3.742084052964882,
-      "grad_norm": 0.8079864978790283,
-      "learning_rate": 5.092603377984858e-06,
-      "loss": 5.8145,
-      "mean_token_accuracy": 0.2531423449516296,
-      "num_tokens": 6059915.0,
+      "grad_norm": 1.166090726852417,
+      "learning_rate": 1.3307646710136337e-07,
+      "loss": 5.3517,
+      "mean_token_accuracy": 0.30615471601486205,
+      "num_tokens": 6828545.0,
       "step": 6500
     },
     {
-      "entropy": 6.113835816383362,
+      "entropy": 5.6728374910354615,
       "epoch": 3.7708693149107657,
-      "grad_norm": 1.0994234085083008,
-      "learning_rate": 4.976121141525918e-06,
-      "loss": 5.9438,
-      "mean_token_accuracy": 0.2360466265678406,
-      "num_tokens": 6109703.0,
+      "grad_norm": 2.3615996837615967,
+      "learning_rate": 1.1825726141078837e-07,
+      "loss": 5.5058,
+      "mean_token_accuracy": 0.28638383001089096,
+      "num_tokens": 6884005.0,
       "step": 6550
     },
     {
-      "entropy": 5.891526069641113,
+      "entropy": 5.4262278175354,
       "epoch": 3.7996545768566494,
-      "grad_norm": 0.6459522843360901,
-      "learning_rate": 4.859638905066978e-06,
-      "loss": 5.7234,
-      "mean_token_accuracy": 0.264646929204464,
-      "num_tokens": 6155107.0,
+      "grad_norm": 1.7658995389938354,
+      "learning_rate": 1.0343805572021339e-07,
+      "loss": 5.2617,
+      "mean_token_accuracy": 0.31743784427642824,
+      "num_tokens": 6935209.0,
       "step": 6600
     },
     {
-      "entropy": 5.878629055023193,
+      "entropy": 5.436288638114929,
       "epoch": 3.828439838802533,
-      "grad_norm": 2.0741031169891357,
-      "learning_rate": 4.743156668608038e-06,
-      "loss": 5.7132,
-      "mean_token_accuracy": 0.26615093410015106,
-      "num_tokens": 6201565.0,
+      "grad_norm": 3.455641269683838,
+      "learning_rate": 8.861885002963842e-08,
+      "loss": 5.2706,
+      "mean_token_accuracy": 0.31677050977945326,
+      "num_tokens": 6987396.0,
       "step": 6650
     },
     {
-      "entropy": 6.0427888488769534,
+      "entropy": 5.586358890533448,
       "epoch": 3.857225100748417,
-      "grad_norm": 0.6295380592346191,
-      "learning_rate": 4.626674432149098e-06,
-      "loss": 5.8755,
-      "mean_token_accuracy": 0.24501585960388184,
-      "num_tokens": 6249569.0,
+      "grad_norm": 1.981423020362854,
+      "learning_rate": 7.379964433906343e-08,
+      "loss": 5.4191,
+      "mean_token_accuracy": 0.2982942935824394,
+      "num_tokens": 7041132.0,
       "step": 6700
     },
     {
-      "entropy": 5.9715061330795285,
+      "entropy": 5.494750590324402,
       "epoch": 3.8860103626943006,
-      "grad_norm": 0.5263471007347107,
-      "learning_rate": 4.5101921956901576e-06,
-      "loss": 5.8028,
-      "mean_token_accuracy": 0.25363644570112226,
-      "num_tokens": 6296537.0,
+      "grad_norm": 1.7962652444839478,
+      "learning_rate": 5.8980438648488434e-08,
+      "loss": 5.3306,
+      "mean_token_accuracy": 0.3082431614398956,
+      "num_tokens": 7094059.0,
       "step": 6750
     },
     {
-      "entropy": 5.886189122200012,
+      "entropy": 5.393875141143798,
       "epoch": 3.9147956246401843,
-      "grad_norm": 0.8449739217758179,
-      "learning_rate": 4.393709959231217e-06,
-      "loss": 5.7206,
-      "mean_token_accuracy": 0.26399643808603285,
-      "num_tokens": 6341373.0,
+      "grad_norm": 1.8328484296798706,
+      "learning_rate": 4.416123295791346e-08,
+      "loss": 5.2351,
+      "mean_token_accuracy": 0.3187332367897034,
+      "num_tokens": 7144964.0,
       "step": 6800
     },
     {
-      "entropy": 6.144029655456543,
+      "entropy": 5.660646886825561,
       "epoch": 3.943580886586068,
-      "grad_norm": 0.9823312163352966,
-      "learning_rate": 4.277227722772277e-06,
-      "loss": 5.9724,
-      "mean_token_accuracy": 0.23244859367609025,
-      "num_tokens": 6391079.0,
+      "grad_norm": 0.8133105039596558,
+      "learning_rate": 2.934202726733847e-08,
+      "loss": 5.4946,
+      "mean_token_accuracy": 0.2876924830675125,
+      "num_tokens": 7200805.0,
       "step": 6850
     },
     {
-      "entropy": 5.7569769096374515,
+      "entropy": 5.239456839561463,
       "epoch": 3.9723661485319517,
-      "grad_norm": 1.98943030834198,
-      "learning_rate": 4.160745486313338e-06,
-      "loss": 5.5963,
-      "mean_token_accuracy": 0.2798686361312866,
-      "num_tokens": 6434857.0,
+      "grad_norm": 7.838026523590088,
+      "learning_rate": 1.4522821576763486e-08,
+      "loss": 5.0866,
+      "mean_token_accuracy": 0.33811178654432295,
+      "num_tokens": 7250918.0,
       "step": 6900
     },
     {
       "epoch": 4.0,
-      "eval_entropy": 6.247500973363077,
-      "eval_loss": 6.0967888832092285,
-      "eval_mean_token_accuracy": 0.2140663956304849,
-      "eval_model_preparation_time": 0.0036,
-      "eval_num_tokens": 6478876.0,
-      "eval_runtime": 76.4612,
-      "eval_samples_per_second": 5.676,
-      "eval_steps_per_second": 2.838,
+      "eval_entropy": 5.780879339314826,
+      "eval_loss": 5.622366428375244,
+      "eval_mean_token_accuracy": 0.26563407995733795,
+      "eval_model_preparation_time": 0.0047,
+      "eval_num_tokens": 7300428.0,
+      "eval_runtime": 80.4424,
+      "eval_samples_per_second": 5.395,
+      "eval_steps_per_second": 2.698,
       "step": 6948
     }
   ],
   "logging_steps": 50,
-  "max_steps": 8685,
+  "max_steps": 6948,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 5,
+  "num_train_epochs": 4,
   "save_steps": 500,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -1450,12 +1450,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 9.054713052854784e+16,
+  "total_flos": 1.0021019691282432e+17,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null

checkpoint-6948/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:130d33149272782bd60306263c371036419926142b8999aad7806359168f8484
+oid sha256:8db5c304963110404ebb6947b83ba95bd9b8aad1f9b8b578cc33c46d601e13dc
 size 6225

checkpoint-8685/adapter_config.json

@@ -16,7 +16,7 @@
   "layers_pattern": null,
   "layers_to_transform": null,
   "loftq_config": {},
-  "lora_alpha": 16,
+  "lora_alpha": 32,
   "lora_bias": false,
   "lora_dropout": 0.1,
   "megatron_config": null,
@@ -29,8 +29,10 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
+    "k_proj",
+    "v_proj",
     "q_proj",
-    "v_proj"
+    "o_proj"
   ],
   "target_parameters": null,
   "task_type": "CAUSAL_LM",

checkpoint-8685/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:76b5201211b5dac5150a2b3a87809a5671a1239a76fdfafed2618f15a157a612
-size 4374520
+oid sha256:2381d61542c1032294bdfd8d93b87c507ec0307a2bd423dfa1c90ac19f153434
+size 8749064

checkpoint-8685/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:01d27e4d4e2843533494998d1773c9af8797769c8e1900f756f6eb1a61546355
-size 8783179
+oid sha256:36cfb0e2c01a3583f649b7157010998f7cfe60c81f2d8dd9f8a236e6ac0ea717
+size 17621003