Upload folder using huggingface_hub

README.md

@@ -1,3 +1,62 @@
----
-license: cc-by-nc-4.0
----
+---
+base_model: HuggingFaceTB/SmolLM2-135M-Instruct
+library_name: peft
+model_name: smol-code-finetuned
+tags:
+- base_model:adapter:HuggingFaceTB/SmolLM2-135M-Instruct
+- lora
+- sft
+- transformers
+- trl
+licence: license
+pipeline_tag: text-generation
+---
+
+# Model Card for smol-code-finetuned
+
+This model is a fine-tuned version of [HuggingFaceTB/SmolLM2-135M-Instruct](https://huggingface.co/HuggingFaceTB/SmolLM2-135M-Instruct).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="None", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+ 
+
+
+
+This model was trained with SFT.
+
+### Framework versions
+
+- PEFT 0.18.1
+- TRL: 0.29.0
+- Transformers: 5.2.0
+- Pytorch: 2.8.0+cu128
+- Datasets: 4.6.0
+- Tokenizers: 0.22.2
+
+## Citations
+
+
+
+Cite TRL as:
+    
+```bibtex
+@software{vonwerra2020trl,
+  title   = {{TRL: Transformers Reinforcement Learning}},
+  author  = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+  license = {Apache-2.0},
+  url     = {https://github.com/huggingface/trl},
+  year    = {2020}
+}
+```

adapter_config.json

@@ -0,0 +1,43 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "HuggingFaceTB/SmolLM2-135M-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "k_proj",
+    "q_proj",
+    "v_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7fc9ea20edd42cffd8c5b32349900876962b89d0cf0b52aa2d24603f2324ff2a
+size 7404368

chat_template.jinja

@@ -0,0 +1,6 @@
+{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system
+You are a helpful AI assistant named SmolLM, trained by Hugging Face<|im_end|>
+' }}{% endif %}{{'<|im_start|>' + message['role'] + '
+' + message['content'] + '<|im_end|>' + '
+'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
+' }}{% endif %}

checkpoint-1000/README.md

@@ -0,0 +1,209 @@
+---
+base_model: HuggingFaceTB/SmolLM2-135M-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:HuggingFaceTB/SmolLM2-135M-Instruct
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

checkpoint-1000/adapter_config.json

@@ -0,0 +1,43 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "HuggingFaceTB/SmolLM2-135M-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "k_proj",
+    "q_proj",
+    "v_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-1000/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1fb14f59fc57fecf96ea885954c9cb2822207733ba43670e1ff021fc31e68975
+size 7404368

checkpoint-1000/chat_template.jinja

@@ -0,0 +1,6 @@
+{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system
+You are a helpful AI assistant named SmolLM, trained by Hugging Face<|im_end|>
+' }}{% endif %}{{'<|im_start|>' + message['role'] + '
+' + message['content'] + '<|im_end|>' + '
+'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
+' }}{% endif %}

checkpoint-1000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0e6afd6f6659f29d948d88153cd2bcc0efaea4885a62014c0d066cb8c366d52
+size 14950667

checkpoint-1000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8363b231e1dbed9a1bec5f309af311ec37800cc626a920ead274e4b95875d26d
+size 14645

checkpoint-1000/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14ae2a2128444abab378aa06c09a61a84665f758fcc19fc46f5789b0bc1b5665
+size 1383

checkpoint-1000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:458c01d724b924da0f67feecdb71910c7d5255196b1858f519282fa895263558
+size 1465

checkpoint-1000/tokenizer_config.json

@@ -0,0 +1,17 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|im_start|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "is_local": false,
+  "model_max_length": 8192,
+  "pad_token": "<|im_end|>",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "<|endoftext|>",
+  "vocab_size": 49152
+}

checkpoint-1000/trainer_state.json

@@ -0,0 +1,1034 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.8591065292096219,
+  "eval_steps": 500,
+  "global_step": 1000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.1566081464290618,
+      "epoch": 0.00859106529209622,
+      "grad_norm": 0.15214066207408905,
+      "learning_rate": 0.00029884020618556697,
+      "loss": 1.226055908203125,
+      "mean_token_accuracy": 0.7206311166286469,
+      "num_tokens": 26541.0,
+      "step": 10
+    },
+    {
+      "entropy": 1.2002840995788575,
+      "epoch": 0.01718213058419244,
+      "grad_norm": 0.1833968460559845,
+      "learning_rate": 0.00029755154639175255,
+      "loss": 1.2007878303527832,
+      "mean_token_accuracy": 0.7304173231124877,
+      "num_tokens": 52280.0,
+      "step": 20
+    },
+    {
+      "entropy": 1.083837980031967,
+      "epoch": 0.02577319587628866,
+      "grad_norm": 0.15183115005493164,
+      "learning_rate": 0.0002962628865979381,
+      "loss": 1.1582960128784179,
+      "mean_token_accuracy": 0.7341346919536591,
+      "num_tokens": 79603.0,
+      "step": 30
+    },
+    {
+      "entropy": 0.9830838143825531,
+      "epoch": 0.03436426116838488,
+      "grad_norm": 0.1640024185180664,
+      "learning_rate": 0.00029497422680412364,
+      "loss": 0.989743423461914,
+      "mean_token_accuracy": 0.7652481079101563,
+      "num_tokens": 106301.0,
+      "step": 40
+    },
+    {
+      "entropy": 0.9237875938415527,
+      "epoch": 0.0429553264604811,
+      "grad_norm": 0.1496010422706604,
+      "learning_rate": 0.0002936855670103092,
+      "loss": 0.9489053726196289,
+      "mean_token_accuracy": 0.7682253539562225,
+      "num_tokens": 131791.0,
+      "step": 50
+    },
+    {
+      "entropy": 0.945674192905426,
+      "epoch": 0.05154639175257732,
+      "grad_norm": 0.16326577961444855,
+      "learning_rate": 0.0002923969072164948,
+      "loss": 0.9549709320068359,
+      "mean_token_accuracy": 0.7691607773303986,
+      "num_tokens": 156376.0,
+      "step": 60
+    },
+    {
+      "entropy": 0.9946551442146301,
+      "epoch": 0.06013745704467354,
+      "grad_norm": 0.14451684057712555,
+      "learning_rate": 0.00029110824742268037,
+      "loss": 0.9979846954345704,
+      "mean_token_accuracy": 0.7634965121746063,
+      "num_tokens": 183753.0,
+      "step": 70
+    },
+    {
+      "entropy": 0.9941512286663056,
+      "epoch": 0.06872852233676977,
+      "grad_norm": 0.15088772773742676,
+      "learning_rate": 0.00028981958762886595,
+      "loss": 0.9917948722839356,
+      "mean_token_accuracy": 0.7609466493129731,
+      "num_tokens": 209154.0,
+      "step": 80
+    },
+    {
+      "entropy": 0.899951022863388,
+      "epoch": 0.07731958762886598,
+      "grad_norm": 0.1635286509990692,
+      "learning_rate": 0.0002885309278350515,
+      "loss": 0.9084607124328613,
+      "mean_token_accuracy": 0.773688817024231,
+      "num_tokens": 232285.0,
+      "step": 90
+    },
+    {
+      "entropy": 0.9840596318244934,
+      "epoch": 0.0859106529209622,
+      "grad_norm": 0.1687513142824173,
+      "learning_rate": 0.0002872422680412371,
+      "loss": 1.005988883972168,
+      "mean_token_accuracy": 0.7565966248512268,
+      "num_tokens": 258165.0,
+      "step": 100
+    },
+    {
+      "entropy": 0.9256654202938079,
+      "epoch": 0.09450171821305842,
+      "grad_norm": 0.15977086126804352,
+      "learning_rate": 0.0002859536082474227,
+      "loss": 0.8968193054199218,
+      "mean_token_accuracy": 0.7728870630264282,
+      "num_tokens": 283613.0,
+      "step": 110
+    },
+    {
+      "entropy": 0.9430408120155335,
+      "epoch": 0.10309278350515463,
+      "grad_norm": 0.14840301871299744,
+      "learning_rate": 0.0002846649484536082,
+      "loss": 0.9234983444213867,
+      "mean_token_accuracy": 0.7652122378349304,
+      "num_tokens": 308257.0,
+      "step": 120
+    },
+    {
+      "entropy": 0.9537142395973206,
+      "epoch": 0.11168384879725086,
+      "grad_norm": 0.1754530817270279,
+      "learning_rate": 0.0002833762886597938,
+      "loss": 0.949033260345459,
+      "mean_token_accuracy": 0.7616437077522278,
+      "num_tokens": 333700.0,
+      "step": 130
+    },
+    {
+      "entropy": 0.9385378897190094,
+      "epoch": 0.12027491408934708,
+      "grad_norm": 0.16817958652973175,
+      "learning_rate": 0.00028208762886597935,
+      "loss": 0.9222366333007812,
+      "mean_token_accuracy": 0.7721138775348664,
+      "num_tokens": 359746.0,
+      "step": 140
+    },
+    {
+      "entropy": 0.8476657152175904,
+      "epoch": 0.12886597938144329,
+      "grad_norm": 0.15769453346729279,
+      "learning_rate": 0.0002807989690721649,
+      "loss": 0.8225071907043457,
+      "mean_token_accuracy": 0.7880932271480561,
+      "num_tokens": 383468.0,
+      "step": 150
+    },
+    {
+      "entropy": 0.8862757325172425,
+      "epoch": 0.13745704467353953,
+      "grad_norm": 0.17620904743671417,
+      "learning_rate": 0.0002795103092783505,
+      "loss": 0.8871653556823731,
+      "mean_token_accuracy": 0.7726393282413483,
+      "num_tokens": 409660.0,
+      "step": 160
+    },
+    {
+      "entropy": 0.9064472615718842,
+      "epoch": 0.14604810996563575,
+      "grad_norm": 0.15342549979686737,
+      "learning_rate": 0.0002782216494845361,
+      "loss": 0.9092922210693359,
+      "mean_token_accuracy": 0.7713103413581848,
+      "num_tokens": 437071.0,
+      "step": 170
+    },
+    {
+      "entropy": 0.9095455348491669,
+      "epoch": 0.15463917525773196,
+      "grad_norm": 0.17525391280651093,
+      "learning_rate": 0.00027693298969072165,
+      "loss": 0.8956180572509765,
+      "mean_token_accuracy": 0.7772489190101624,
+      "num_tokens": 460852.0,
+      "step": 180
+    },
+    {
+      "entropy": 0.8514094173908233,
+      "epoch": 0.16323024054982818,
+      "grad_norm": 0.15829823911190033,
+      "learning_rate": 0.0002756443298969072,
+      "loss": 0.8513413429260254,
+      "mean_token_accuracy": 0.779003256559372,
+      "num_tokens": 486114.0,
+      "step": 190
+    },
+    {
+      "entropy": 0.8995061337947845,
+      "epoch": 0.1718213058419244,
+      "grad_norm": 0.15622206032276154,
+      "learning_rate": 0.00027435567010309275,
+      "loss": 0.898340892791748,
+      "mean_token_accuracy": 0.7708433449268342,
+      "num_tokens": 512397.0,
+      "step": 200
+    },
+    {
+      "entropy": 0.8732269763946533,
+      "epoch": 0.18041237113402062,
+      "grad_norm": 0.17759552597999573,
+      "learning_rate": 0.0002730670103092783,
+      "loss": 0.8695596694946289,
+      "mean_token_accuracy": 0.7793173313140869,
+      "num_tokens": 536554.0,
+      "step": 210
+    },
+    {
+      "entropy": 0.9366106212139129,
+      "epoch": 0.18900343642611683,
+      "grad_norm": 0.1767597794532776,
+      "learning_rate": 0.0002717783505154639,
+      "loss": 0.9249121665954589,
+      "mean_token_accuracy": 0.7724991559982299,
+      "num_tokens": 561637.0,
+      "step": 220
+    },
+    {
+      "entropy": 0.8857321858406066,
+      "epoch": 0.19759450171821305,
+      "grad_norm": 0.20788735151290894,
+      "learning_rate": 0.0002704896907216495,
+      "loss": 0.8790401458740235,
+      "mean_token_accuracy": 0.7778670430183411,
+      "num_tokens": 586614.0,
+      "step": 230
+    },
+    {
+      "entropy": 0.9273143649101258,
+      "epoch": 0.20618556701030927,
+      "grad_norm": 0.16100187599658966,
+      "learning_rate": 0.00026920103092783505,
+      "loss": 0.9232232093811035,
+      "mean_token_accuracy": 0.7677364766597747,
+      "num_tokens": 612428.0,
+      "step": 240
+    },
+    {
+      "entropy": 0.8786617696285248,
+      "epoch": 0.21477663230240548,
+      "grad_norm": 0.1608886867761612,
+      "learning_rate": 0.00026791237113402063,
+      "loss": 0.8883329391479492,
+      "mean_token_accuracy": 0.7775621056556702,
+      "num_tokens": 638564.0,
+      "step": 250
+    },
+    {
+      "entropy": 0.9227964103221893,
+      "epoch": 0.22336769759450173,
+      "grad_norm": 0.1510680615901947,
+      "learning_rate": 0.00026662371134020615,
+      "loss": 0.9296038627624512,
+      "mean_token_accuracy": 0.7714014649391174,
+      "num_tokens": 665263.0,
+      "step": 260
+    },
+    {
+      "entropy": 0.9126189887523651,
+      "epoch": 0.23195876288659795,
+      "grad_norm": 0.16434459388256073,
+      "learning_rate": 0.00026533505154639173,
+      "loss": 0.9191689491271973,
+      "mean_token_accuracy": 0.7705935597419739,
+      "num_tokens": 691956.0,
+      "step": 270
+    },
+    {
+      "entropy": 0.8797955513000488,
+      "epoch": 0.24054982817869416,
+      "grad_norm": 0.1714707911014557,
+      "learning_rate": 0.0002640463917525773,
+      "loss": 0.8856119155883789,
+      "mean_token_accuracy": 0.7766761302947998,
+      "num_tokens": 716820.0,
+      "step": 280
+    },
+    {
+      "entropy": 0.877093505859375,
+      "epoch": 0.24914089347079038,
+      "grad_norm": 0.15326625108718872,
+      "learning_rate": 0.0002627577319587629,
+      "loss": 0.8566678047180176,
+      "mean_token_accuracy": 0.7812119901180268,
+      "num_tokens": 742221.0,
+      "step": 290
+    },
+    {
+      "entropy": 0.87042076587677,
+      "epoch": 0.25773195876288657,
+      "grad_norm": 0.1710847020149231,
+      "learning_rate": 0.00026146907216494846,
+      "loss": 0.8829298973083496,
+      "mean_token_accuracy": 0.7770332932472229,
+      "num_tokens": 765839.0,
+      "step": 300
+    },
+    {
+      "entropy": 0.8417458057403564,
+      "epoch": 0.2663230240549828,
+      "grad_norm": 0.1496821790933609,
+      "learning_rate": 0.00026018041237113403,
+      "loss": 0.8433545112609864,
+      "mean_token_accuracy": 0.7857938230037689,
+      "num_tokens": 791131.0,
+      "step": 310
+    },
+    {
+      "entropy": 0.9158999443054199,
+      "epoch": 0.27491408934707906,
+      "grad_norm": 0.17929035425186157,
+      "learning_rate": 0.00025889175257731955,
+      "loss": 0.9123490333557129,
+      "mean_token_accuracy": 0.775556480884552,
+      "num_tokens": 816984.0,
+      "step": 320
+    },
+    {
+      "entropy": 0.8314016401767731,
+      "epoch": 0.28350515463917525,
+      "grad_norm": 0.16051091253757477,
+      "learning_rate": 0.00025760309278350513,
+      "loss": 0.8279628753662109,
+      "mean_token_accuracy": 0.7862712442874908,
+      "num_tokens": 842220.0,
+      "step": 330
+    },
+    {
+      "entropy": 0.8677021145820618,
+      "epoch": 0.2920962199312715,
+      "grad_norm": 0.19157563149929047,
+      "learning_rate": 0.0002563144329896907,
+      "loss": 0.8552864074707032,
+      "mean_token_accuracy": 0.7835110187530517,
+      "num_tokens": 867868.0,
+      "step": 340
+    },
+    {
+      "entropy": 0.8919401407241822,
+      "epoch": 0.3006872852233677,
+      "grad_norm": 0.14258626103401184,
+      "learning_rate": 0.0002550257731958763,
+      "loss": 0.9019416809082031,
+      "mean_token_accuracy": 0.7760707855224609,
+      "num_tokens": 894676.0,
+      "step": 350
+    },
+    {
+      "entropy": 0.8415017545223236,
+      "epoch": 0.30927835051546393,
+      "grad_norm": 0.14830808341503143,
+      "learning_rate": 0.00025373711340206186,
+      "loss": 0.8072587013244629,
+      "mean_token_accuracy": 0.7912742376327515,
+      "num_tokens": 921632.0,
+      "step": 360
+    },
+    {
+      "entropy": 0.8371863842010498,
+      "epoch": 0.3178694158075601,
+      "grad_norm": 0.15538977086544037,
+      "learning_rate": 0.0002524484536082474,
+      "loss": 0.8308550834655761,
+      "mean_token_accuracy": 0.7849089920520782,
+      "num_tokens": 945955.0,
+      "step": 370
+    },
+    {
+      "entropy": 0.8338176369667053,
+      "epoch": 0.32646048109965636,
+      "grad_norm": 0.1711844801902771,
+      "learning_rate": 0.00025115979381443295,
+      "loss": 0.8479556083679199,
+      "mean_token_accuracy": 0.7850408136844635,
+      "num_tokens": 971508.0,
+      "step": 380
+    },
+    {
+      "entropy": 0.8876297354698182,
+      "epoch": 0.33505154639175255,
+      "grad_norm": 0.14902909100055695,
+      "learning_rate": 0.00024987113402061853,
+      "loss": 0.8821809768676758,
+      "mean_token_accuracy": 0.7764640510082245,
+      "num_tokens": 995124.0,
+      "step": 390
+    },
+    {
+      "entropy": 0.8212514638900756,
+      "epoch": 0.3436426116838488,
+      "grad_norm": 0.17950233817100525,
+      "learning_rate": 0.0002485824742268041,
+      "loss": 0.8255527496337891,
+      "mean_token_accuracy": 0.7870513379573822,
+      "num_tokens": 1019735.0,
+      "step": 400
+    },
+    {
+      "entropy": 0.9294387817382812,
+      "epoch": 0.35223367697594504,
+      "grad_norm": 0.15237703919410706,
+      "learning_rate": 0.0002472938144329897,
+      "loss": 0.9235431671142578,
+      "mean_token_accuracy": 0.7703655660152435,
+      "num_tokens": 1046295.0,
+      "step": 410
+    },
+    {
+      "entropy": 0.8429355442523956,
+      "epoch": 0.36082474226804123,
+      "grad_norm": 0.1589658558368683,
+      "learning_rate": 0.0002460051546391752,
+      "loss": 0.8329957008361817,
+      "mean_token_accuracy": 0.7897172749042511,
+      "num_tokens": 1072020.0,
+      "step": 420
+    },
+    {
+      "entropy": 0.8407810032367706,
+      "epoch": 0.3694158075601375,
+      "grad_norm": 0.17521116137504578,
+      "learning_rate": 0.0002447164948453608,
+      "loss": 0.8535200119018554,
+      "mean_token_accuracy": 0.7809976935386658,
+      "num_tokens": 1098420.0,
+      "step": 430
+    },
+    {
+      "entropy": 0.8515082120895385,
+      "epoch": 0.37800687285223367,
+      "grad_norm": 0.15218117833137512,
+      "learning_rate": 0.00024342783505154638,
+      "loss": 0.8459153175354004,
+      "mean_token_accuracy": 0.7833445191383361,
+      "num_tokens": 1124235.0,
+      "step": 440
+    },
+    {
+      "entropy": 0.8506015419960022,
+      "epoch": 0.3865979381443299,
+      "grad_norm": 0.1809423863887787,
+      "learning_rate": 0.00024213917525773193,
+      "loss": 0.8384982109069824,
+      "mean_token_accuracy": 0.7841372966766358,
+      "num_tokens": 1149636.0,
+      "step": 450
+    },
+    {
+      "entropy": 0.9110018074512481,
+      "epoch": 0.3951890034364261,
+      "grad_norm": 0.15085963904857635,
+      "learning_rate": 0.0002408505154639175,
+      "loss": 0.9191832542419434,
+      "mean_token_accuracy": 0.7757422685623169,
+      "num_tokens": 1173711.0,
+      "step": 460
+    },
+    {
+      "entropy": 0.9000201225280762,
+      "epoch": 0.40378006872852235,
+      "grad_norm": 0.16748838126659393,
+      "learning_rate": 0.00023956185567010308,
+      "loss": 0.9107230186462403,
+      "mean_token_accuracy": 0.7768173336982727,
+      "num_tokens": 1199340.0,
+      "step": 470
+    },
+    {
+      "entropy": 0.8751251935958863,
+      "epoch": 0.41237113402061853,
+      "grad_norm": 0.1593722254037857,
+      "learning_rate": 0.00023827319587628863,
+      "loss": 0.8804635047912598,
+      "mean_token_accuracy": 0.7814023733139038,
+      "num_tokens": 1226032.0,
+      "step": 480
+    },
+    {
+      "entropy": 0.8843371510505676,
+      "epoch": 0.4209621993127148,
+      "grad_norm": 0.12278851121664047,
+      "learning_rate": 0.0002369845360824742,
+      "loss": 0.8643080711364746,
+      "mean_token_accuracy": 0.7809630572795868,
+      "num_tokens": 1252769.0,
+      "step": 490
+    },
+    {
+      "entropy": 0.8834336459636688,
+      "epoch": 0.42955326460481097,
+      "grad_norm": 0.1367531716823578,
+      "learning_rate": 0.00023569587628865976,
+      "loss": 0.8869522094726563,
+      "mean_token_accuracy": 0.7775909185409546,
+      "num_tokens": 1278979.0,
+      "step": 500
+    },
+    {
+      "entropy": 0.9062523484230042,
+      "epoch": 0.4381443298969072,
+      "grad_norm": 0.14829133450984955,
+      "learning_rate": 0.00023440721649484533,
+      "loss": 0.9153310775756835,
+      "mean_token_accuracy": 0.7755910754203796,
+      "num_tokens": 1305141.0,
+      "step": 510
+    },
+    {
+      "entropy": 0.8907318472862243,
+      "epoch": 0.44673539518900346,
+      "grad_norm": 0.19640029966831207,
+      "learning_rate": 0.0002331185567010309,
+      "loss": 0.8736312866210938,
+      "mean_token_accuracy": 0.7823721885681152,
+      "num_tokens": 1333535.0,
+      "step": 520
+    },
+    {
+      "entropy": 0.9001505076885223,
+      "epoch": 0.45532646048109965,
+      "grad_norm": 0.1615416407585144,
+      "learning_rate": 0.00023182989690721648,
+      "loss": 0.9240070343017578,
+      "mean_token_accuracy": 0.7720246851444245,
+      "num_tokens": 1359100.0,
+      "step": 530
+    },
+    {
+      "entropy": 0.8964834988117218,
+      "epoch": 0.4639175257731959,
+      "grad_norm": 0.16148629784584045,
+      "learning_rate": 0.00023054123711340206,
+      "loss": 0.9040334701538086,
+      "mean_token_accuracy": 0.7697900414466858,
+      "num_tokens": 1386642.0,
+      "step": 540
+    },
+    {
+      "entropy": 0.9152944028377533,
+      "epoch": 0.4725085910652921,
+      "grad_norm": 0.16060467064380646,
+      "learning_rate": 0.00022925257731958758,
+      "loss": 0.9094497680664062,
+      "mean_token_accuracy": 0.771862530708313,
+      "num_tokens": 1412712.0,
+      "step": 550
+    },
+    {
+      "entropy": 0.8443439364433288,
+      "epoch": 0.48109965635738833,
+      "grad_norm": 0.16139253973960876,
+      "learning_rate": 0.00022796391752577316,
+      "loss": 0.8445466041564942,
+      "mean_token_accuracy": 0.7862408220767975,
+      "num_tokens": 1439208.0,
+      "step": 560
+    },
+    {
+      "entropy": 0.8669894218444825,
+      "epoch": 0.4896907216494845,
+      "grad_norm": 0.13149064779281616,
+      "learning_rate": 0.00022667525773195873,
+      "loss": 0.8791823387145996,
+      "mean_token_accuracy": 0.7771342694759369,
+      "num_tokens": 1466602.0,
+      "step": 570
+    },
+    {
+      "entropy": 0.8713687837123871,
+      "epoch": 0.49828178694158076,
+      "grad_norm": 0.15979787707328796,
+      "learning_rate": 0.0002253865979381443,
+      "loss": 0.8594341278076172,
+      "mean_token_accuracy": 0.7810667157173157,
+      "num_tokens": 1493048.0,
+      "step": 580
+    },
+    {
+      "entropy": 0.9155345737934113,
+      "epoch": 0.506872852233677,
+      "grad_norm": 0.18420852720737457,
+      "learning_rate": 0.00022409793814432989,
+      "loss": 0.915378475189209,
+      "mean_token_accuracy": 0.7724271833896637,
+      "num_tokens": 1517512.0,
+      "step": 590
+    },
+    {
+      "entropy": 0.891716331243515,
+      "epoch": 0.5154639175257731,
+      "grad_norm": 0.1665642410516739,
+      "learning_rate": 0.00022280927835051546,
+      "loss": 0.8950610160827637,
+      "mean_token_accuracy": 0.777475905418396,
+      "num_tokens": 1542765.0,
+      "step": 600
+    },
+    {
+      "entropy": 0.9547060489654541,
+      "epoch": 0.5240549828178694,
+      "grad_norm": 0.19496814906597137,
+      "learning_rate": 0.000221520618556701,
+      "loss": 0.9572202682495117,
+      "mean_token_accuracy": 0.7599396347999573,
+      "num_tokens": 1570704.0,
+      "step": 610
+    },
+    {
+      "entropy": 0.894726449251175,
+      "epoch": 0.5326460481099656,
+      "grad_norm": 0.162724569439888,
+      "learning_rate": 0.00022023195876288656,
+      "loss": 0.8876501083374023,
+      "mean_token_accuracy": 0.7815112411975861,
+      "num_tokens": 1594675.0,
+      "step": 620
+    },
+    {
+      "entropy": 0.8966928541660308,
+      "epoch": 0.5412371134020618,
+      "grad_norm": 0.1820981204509735,
+      "learning_rate": 0.00021894329896907214,
+      "loss": 0.8992101669311523,
+      "mean_token_accuracy": 0.7774552226066589,
+      "num_tokens": 1621381.0,
+      "step": 630
+    },
+    {
+      "entropy": 0.8320938229560852,
+      "epoch": 0.5498281786941581,
+      "grad_norm": 0.18641315400600433,
+      "learning_rate": 0.0002176546391752577,
+      "loss": 0.8333114624023438,
+      "mean_token_accuracy": 0.7836415946483613,
+      "num_tokens": 1646589.0,
+      "step": 640
+    },
+    {
+      "entropy": 0.9350853085517883,
+      "epoch": 0.5584192439862543,
+      "grad_norm": 0.15154911577701569,
+      "learning_rate": 0.0002163659793814433,
+      "loss": 0.9300890922546386,
+      "mean_token_accuracy": 0.7711542308330536,
+      "num_tokens": 1672281.0,
+      "step": 650
+    },
+    {
+      "entropy": 0.8372068881988526,
+      "epoch": 0.5670103092783505,
+      "grad_norm": 0.157988041639328,
+      "learning_rate": 0.00021507731958762884,
+      "loss": 0.8490092277526855,
+      "mean_token_accuracy": 0.7828391373157502,
+      "num_tokens": 1697203.0,
+      "step": 660
+    },
+    {
+      "entropy": 0.8175156831741333,
+      "epoch": 0.5756013745704467,
+      "grad_norm": 0.16615159809589386,
+      "learning_rate": 0.0002137886597938144,
+      "loss": 0.7962069988250733,
+      "mean_token_accuracy": 0.793628740310669,
+      "num_tokens": 1722977.0,
+      "step": 670
+    },
+    {
+      "entropy": 0.9293979525566101,
+      "epoch": 0.584192439862543,
+      "grad_norm": 0.13969369232654572,
+      "learning_rate": 0.0002125,
+      "loss": 0.9364766120910645,
+      "mean_token_accuracy": 0.7693193078041076,
+      "num_tokens": 1748869.0,
+      "step": 680
+    },
+    {
+      "entropy": 0.8958774268627167,
+      "epoch": 0.5927835051546392,
+      "grad_norm": 0.14425267279148102,
+      "learning_rate": 0.00021121134020618556,
+      "loss": 0.8842126846313476,
+      "mean_token_accuracy": 0.7759437382221221,
+      "num_tokens": 1774715.0,
+      "step": 690
+    },
+    {
+      "entropy": 0.929528433084488,
+      "epoch": 0.6013745704467354,
+      "grad_norm": 0.15875831246376038,
+      "learning_rate": 0.0002099226804123711,
+      "loss": 0.9353519439697265,
+      "mean_token_accuracy": 0.769885140657425,
+      "num_tokens": 1800430.0,
+      "step": 700
+    },
+    {
+      "entropy": 0.8651838064193725,
+      "epoch": 0.6099656357388317,
+      "grad_norm": 0.19464287161827087,
+      "learning_rate": 0.00020863402061855666,
+      "loss": 0.8678315162658692,
+      "mean_token_accuracy": 0.7783544957637787,
+      "num_tokens": 1824631.0,
+      "step": 710
+    },
+    {
+      "entropy": 0.874721884727478,
+      "epoch": 0.6185567010309279,
+      "grad_norm": 0.17015859484672546,
+      "learning_rate": 0.00020734536082474224,
+      "loss": 0.847961139678955,
+      "mean_token_accuracy": 0.7813271641731262,
+      "num_tokens": 1849593.0,
+      "step": 720
+    },
+    {
+      "entropy": 0.8706691145896912,
+      "epoch": 0.627147766323024,
+      "grad_norm": 0.15960460901260376,
+      "learning_rate": 0.00020605670103092781,
+      "loss": 0.8743976593017578,
+      "mean_token_accuracy": 0.7772314369678497,
+      "num_tokens": 1874593.0,
+      "step": 730
+    },
+    {
+      "entropy": 0.824019831418991,
+      "epoch": 0.6357388316151202,
+      "grad_norm": 0.16192808747291565,
+      "learning_rate": 0.0002047680412371134,
+      "loss": 0.8214377403259278,
+      "mean_token_accuracy": 0.7877136409282685,
+      "num_tokens": 1899592.0,
+      "step": 740
+    },
+    {
+      "entropy": 0.853943508863449,
+      "epoch": 0.6443298969072165,
+      "grad_norm": 0.14331476390361786,
+      "learning_rate": 0.00020347938144329897,
+      "loss": 0.8542372703552246,
+      "mean_token_accuracy": 0.7825242638587951,
+      "num_tokens": 1925105.0,
+      "step": 750
+    },
+    {
+      "entropy": 0.9206405401229858,
+      "epoch": 0.6529209621993127,
+      "grad_norm": 0.16269327700138092,
+      "learning_rate": 0.00020219072164948454,
+      "loss": 0.9131902694702149,
+      "mean_token_accuracy": 0.7722071945667267,
+      "num_tokens": 1952126.0,
+      "step": 760
+    },
+    {
+      "entropy": 0.8438917219638824,
+      "epoch": 0.6615120274914089,
+      "grad_norm": 0.18783624470233917,
+      "learning_rate": 0.00020090206185567006,
+      "loss": 0.8612903594970703,
+      "mean_token_accuracy": 0.7805359125137329,
+      "num_tokens": 1976727.0,
+      "step": 770
+    },
+    {
+      "entropy": 0.8782703518867493,
+      "epoch": 0.6701030927835051,
+      "grad_norm": 0.1749051958322525,
+      "learning_rate": 0.00019961340206185564,
+      "loss": 0.8877654075622559,
+      "mean_token_accuracy": 0.775201940536499,
+      "num_tokens": 2004204.0,
+      "step": 780
+    },
+    {
+      "entropy": 0.9036802053451538,
+      "epoch": 0.6786941580756014,
+      "grad_norm": 0.17728140950202942,
+      "learning_rate": 0.00019832474226804121,
+      "loss": 0.8946940422058105,
+      "mean_token_accuracy": 0.7781059086322785,
+      "num_tokens": 2031926.0,
+      "step": 790
+    },
+    {
+      "entropy": 0.9057375133037567,
+      "epoch": 0.6872852233676976,
+      "grad_norm": 0.1804327517747879,
+      "learning_rate": 0.0001970360824742268,
+      "loss": 0.9067662239074707,
+      "mean_token_accuracy": 0.7786577641963959,
+      "num_tokens": 2059628.0,
+      "step": 800
+    },
+    {
+      "entropy": 0.8421910405158997,
+      "epoch": 0.6958762886597938,
+      "grad_norm": 0.1570490598678589,
+      "learning_rate": 0.00019574742268041237,
+      "loss": 0.8346380233764649,
+      "mean_token_accuracy": 0.7839872419834137,
+      "num_tokens": 2083943.0,
+      "step": 810
+    },
+    {
+      "entropy": 0.8743102788925171,
+      "epoch": 0.7044673539518901,
+      "grad_norm": 0.1737687736749649,
+      "learning_rate": 0.00019445876288659794,
+      "loss": 0.8622690200805664,
+      "mean_token_accuracy": 0.7791435241699218,
+      "num_tokens": 2109444.0,
+      "step": 820
+    },
+    {
+      "entropy": 0.8926449298858643,
+      "epoch": 0.7130584192439863,
+      "grad_norm": 0.1972922682762146,
+      "learning_rate": 0.0001931701030927835,
+      "loss": 0.895075511932373,
+      "mean_token_accuracy": 0.7716107368469238,
+      "num_tokens": 2134209.0,
+      "step": 830
+    },
+    {
+      "entropy": 0.8712981760501861,
+      "epoch": 0.7216494845360825,
+      "grad_norm": 0.16580969095230103,
+      "learning_rate": 0.00019188144329896904,
+      "loss": 0.8615394592285156,
+      "mean_token_accuracy": 0.780440878868103,
+      "num_tokens": 2160322.0,
+      "step": 840
+    },
+    {
+      "entropy": 0.7872600853443146,
+      "epoch": 0.7302405498281787,
+      "grad_norm": 0.17089906334877014,
+      "learning_rate": 0.00019059278350515462,
+      "loss": 0.7738356590270996,
+      "mean_token_accuracy": 0.7984008610248565,
+      "num_tokens": 2186673.0,
+      "step": 850
+    },
+    {
+      "entropy": 0.8024394273757934,
+      "epoch": 0.738831615120275,
+      "grad_norm": 0.15015770494937897,
+      "learning_rate": 0.0001893041237113402,
+      "loss": 0.801088809967041,
+      "mean_token_accuracy": 0.7919513285160065,
+      "num_tokens": 2211288.0,
+      "step": 860
+    },
+    {
+      "entropy": 0.8289069414138794,
+      "epoch": 0.7474226804123711,
+      "grad_norm": 0.15804092586040497,
+      "learning_rate": 0.00018801546391752577,
+      "loss": 0.8228999137878418,
+      "mean_token_accuracy": 0.7900383174419403,
+      "num_tokens": 2235494.0,
+      "step": 870
+    },
+    {
+      "entropy": 0.8337613046169281,
+      "epoch": 0.7560137457044673,
+      "grad_norm": 0.16564202308654785,
+      "learning_rate": 0.00018672680412371132,
+      "loss": 0.8357515335083008,
+      "mean_token_accuracy": 0.7796721041202546,
+      "num_tokens": 2261853.0,
+      "step": 880
+    },
+    {
+      "entropy": 0.8514188408851624,
+      "epoch": 0.7646048109965635,
+      "grad_norm": 0.15763898193836212,
+      "learning_rate": 0.0001854381443298969,
+      "loss": 0.8596377372741699,
+      "mean_token_accuracy": 0.781908392906189,
+      "num_tokens": 2288235.0,
+      "step": 890
+    },
+    {
+      "entropy": 0.8182059168815613,
+      "epoch": 0.7731958762886598,
+      "grad_norm": 0.14843082427978516,
+      "learning_rate": 0.00018414948453608247,
+      "loss": 0.8056727409362793,
+      "mean_token_accuracy": 0.7887085616588593,
+      "num_tokens": 2312636.0,
+      "step": 900
+    },
+    {
+      "entropy": 0.8019815027713776,
+      "epoch": 0.781786941580756,
+      "grad_norm": 0.1444498896598816,
+      "learning_rate": 0.00018286082474226804,
+      "loss": 0.7791801452636719,
+      "mean_token_accuracy": 0.7921311616897583,
+      "num_tokens": 2337590.0,
+      "step": 910
+    },
+    {
+      "entropy": 0.8811851978302002,
+      "epoch": 0.7903780068728522,
+      "grad_norm": 0.20295798778533936,
+      "learning_rate": 0.0001815721649484536,
+      "loss": 0.9093887329101562,
+      "mean_token_accuracy": 0.7765615165233613,
+      "num_tokens": 2364733.0,
+      "step": 920
+    },
+    {
+      "entropy": 0.8845545649528503,
+      "epoch": 0.7989690721649485,
+      "grad_norm": 0.16211840510368347,
+      "learning_rate": 0.00018028350515463914,
+      "loss": 0.87130765914917,
+      "mean_token_accuracy": 0.773591297864914,
+      "num_tokens": 2389138.0,
+      "step": 930
+    },
+    {
+      "entropy": 0.8366316437721253,
+      "epoch": 0.8075601374570447,
+      "grad_norm": 0.1796175092458725,
+      "learning_rate": 0.00017899484536082472,
+      "loss": 0.8393503189086914,
+      "mean_token_accuracy": 0.7834821224212647,
+      "num_tokens": 2416288.0,
+      "step": 940
+    },
+    {
+      "entropy": 0.8687749207019806,
+      "epoch": 0.8161512027491409,
+      "grad_norm": 0.17602433264255524,
+      "learning_rate": 0.0001777061855670103,
+      "loss": 0.8831927299499511,
+      "mean_token_accuracy": 0.7806714773178101,
+      "num_tokens": 2442282.0,
+      "step": 950
+    },
+    {
+      "entropy": 0.8602447748184204,
+      "epoch": 0.8247422680412371,
+      "grad_norm": 0.14779216051101685,
+      "learning_rate": 0.00017641752577319587,
+      "loss": 0.8557798385620117,
+      "mean_token_accuracy": 0.7800233542919159,
+      "num_tokens": 2469865.0,
+      "step": 960
+    },
+    {
+      "entropy": 0.8987692832946778,
+      "epoch": 0.8333333333333334,
+      "grad_norm": 0.16065306961536407,
+      "learning_rate": 0.00017512886597938145,
+      "loss": 0.8930519104003907,
+      "mean_token_accuracy": 0.7771966874599456,
+      "num_tokens": 2497716.0,
+      "step": 970
+    },
+    {
+      "entropy": 0.847240948677063,
+      "epoch": 0.8419243986254296,
+      "grad_norm": 0.16063280403614044,
+      "learning_rate": 0.00017384020618556702,
+      "loss": 0.8212727546691895,
+      "mean_token_accuracy": 0.7900629937648773,
+      "num_tokens": 2523292.0,
+      "step": 980
+    },
+    {
+      "entropy": 0.8685175359249115,
+      "epoch": 0.8505154639175257,
+      "grad_norm": 0.19293011724948883,
+      "learning_rate": 0.00017255154639175254,
+      "loss": 0.8839738845825196,
+      "mean_token_accuracy": 0.7780547857284545,
+      "num_tokens": 2547529.0,
+      "step": 990
+    },
+    {
+      "entropy": 0.8589172720909118,
+      "epoch": 0.8591065292096219,
+      "grad_norm": 0.13081640005111694,
+      "learning_rate": 0.00017126288659793812,
+      "loss": 0.8760468482971191,
+      "mean_token_accuracy": 0.7810062050819397,
+      "num_tokens": 2574943.0,
+      "step": 1000
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 2328,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 4219358487773184.0,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-1000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2af45870b4c4d611381846df6bdd4a32cb292a7b5897b79b7e5f196be57c596
+size 5585

checkpoint-1500/README.md

@@ -0,0 +1,209 @@
+---
+base_model: HuggingFaceTB/SmolLM2-135M-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:HuggingFaceTB/SmolLM2-135M-Instruct
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

checkpoint-1500/adapter_config.json

@@ -0,0 +1,43 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "HuggingFaceTB/SmolLM2-135M-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "k_proj",
+    "q_proj",
+    "v_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-1500/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a8a8ac95f14935d4b0f3b9b714a958644a969c8852eec5d6a0cb9a0d964922b
+size 7404368

checkpoint-1500/chat_template.jinja

@@ -0,0 +1,6 @@
+{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system
+You are a helpful AI assistant named SmolLM, trained by Hugging Face<|im_end|>
+' }}{% endif %}{{'<|im_start|>' + message['role'] + '
+' + message['content'] + '<|im_end|>' + '
+'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
+' }}{% endif %}

checkpoint-1500/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6a377fa5af5cf2d7038f86402b71ce7234829f74d7bbb08c33f589517454578
+size 14950667

checkpoint-1500/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8ea63bc095d2dce18c525cee8c55d93344b51e8fec1a86f8f2f1df60fbe2b74
+size 14645

checkpoint-1500/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca372268f4fa9335030c0cb7aedb6cdba75f457da50e7a4034abb1a2d0843689
+size 1383

checkpoint-1500/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cdaed8eaa91e201684212ae8f5031fb818b28a8dd09b8187a56ebc01552b1a6d
+size 1465

checkpoint-1500/tokenizer_config.json

@@ -0,0 +1,17 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|im_start|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "is_local": false,
+  "model_max_length": 8192,
+  "pad_token": "<|im_end|>",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "<|endoftext|>",
+  "vocab_size": 49152
+}

checkpoint-1500/trainer_state.json

@@ -0,0 +1,1534 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.2886597938144329,
+  "eval_steps": 500,
+  "global_step": 1500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.1566081464290618,
+      "epoch": 0.00859106529209622,
+      "grad_norm": 0.15214066207408905,
+      "learning_rate": 0.00029884020618556697,
+      "loss": 1.226055908203125,
+      "mean_token_accuracy": 0.7206311166286469,
+      "num_tokens": 26541.0,
+      "step": 10
+    },
+    {
+      "entropy": 1.2002840995788575,
+      "epoch": 0.01718213058419244,
+      "grad_norm": 0.1833968460559845,
+      "learning_rate": 0.00029755154639175255,
+      "loss": 1.2007878303527832,
+      "mean_token_accuracy": 0.7304173231124877,
+      "num_tokens": 52280.0,
+      "step": 20
+    },
+    {
+      "entropy": 1.083837980031967,
+      "epoch": 0.02577319587628866,
+      "grad_norm": 0.15183115005493164,
+      "learning_rate": 0.0002962628865979381,
+      "loss": 1.1582960128784179,
+      "mean_token_accuracy": 0.7341346919536591,
+      "num_tokens": 79603.0,
+      "step": 30
+    },
+    {
+      "entropy": 0.9830838143825531,
+      "epoch": 0.03436426116838488,
+      "grad_norm": 0.1640024185180664,
+      "learning_rate": 0.00029497422680412364,
+      "loss": 0.989743423461914,
+      "mean_token_accuracy": 0.7652481079101563,
+      "num_tokens": 106301.0,
+      "step": 40
+    },
+    {
+      "entropy": 0.9237875938415527,
+      "epoch": 0.0429553264604811,
+      "grad_norm": 0.1496010422706604,
+      "learning_rate": 0.0002936855670103092,
+      "loss": 0.9489053726196289,
+      "mean_token_accuracy": 0.7682253539562225,
+      "num_tokens": 131791.0,
+      "step": 50
+    },
+    {
+      "entropy": 0.945674192905426,
+      "epoch": 0.05154639175257732,
+      "grad_norm": 0.16326577961444855,
+      "learning_rate": 0.0002923969072164948,
+      "loss": 0.9549709320068359,
+      "mean_token_accuracy": 0.7691607773303986,
+      "num_tokens": 156376.0,
+      "step": 60
+    },
+    {
+      "entropy": 0.9946551442146301,
+      "epoch": 0.06013745704467354,
+      "grad_norm": 0.14451684057712555,
+      "learning_rate": 0.00029110824742268037,
+      "loss": 0.9979846954345704,
+      "mean_token_accuracy": 0.7634965121746063,
+      "num_tokens": 183753.0,
+      "step": 70
+    },
+    {
+      "entropy": 0.9941512286663056,
+      "epoch": 0.06872852233676977,
+      "grad_norm": 0.15088772773742676,
+      "learning_rate": 0.00028981958762886595,
+      "loss": 0.9917948722839356,
+      "mean_token_accuracy": 0.7609466493129731,
+      "num_tokens": 209154.0,
+      "step": 80
+    },
+    {
+      "entropy": 0.899951022863388,
+      "epoch": 0.07731958762886598,
+      "grad_norm": 0.1635286509990692,
+      "learning_rate": 0.0002885309278350515,
+      "loss": 0.9084607124328613,
+      "mean_token_accuracy": 0.773688817024231,
+      "num_tokens": 232285.0,
+      "step": 90
+    },
+    {
+      "entropy": 0.9840596318244934,
+      "epoch": 0.0859106529209622,
+      "grad_norm": 0.1687513142824173,
+      "learning_rate": 0.0002872422680412371,
+      "loss": 1.005988883972168,
+      "mean_token_accuracy": 0.7565966248512268,
+      "num_tokens": 258165.0,
+      "step": 100
+    },
+    {
+      "entropy": 0.9256654202938079,
+      "epoch": 0.09450171821305842,
+      "grad_norm": 0.15977086126804352,
+      "learning_rate": 0.0002859536082474227,
+      "loss": 0.8968193054199218,
+      "mean_token_accuracy": 0.7728870630264282,
+      "num_tokens": 283613.0,
+      "step": 110
+    },
+    {
+      "entropy": 0.9430408120155335,
+      "epoch": 0.10309278350515463,
+      "grad_norm": 0.14840301871299744,
+      "learning_rate": 0.0002846649484536082,
+      "loss": 0.9234983444213867,
+      "mean_token_accuracy": 0.7652122378349304,
+      "num_tokens": 308257.0,
+      "step": 120
+    },
+    {
+      "entropy": 0.9537142395973206,
+      "epoch": 0.11168384879725086,
+      "grad_norm": 0.1754530817270279,
+      "learning_rate": 0.0002833762886597938,
+      "loss": 0.949033260345459,
+      "mean_token_accuracy": 0.7616437077522278,
+      "num_tokens": 333700.0,
+      "step": 130
+    },
+    {
+      "entropy": 0.9385378897190094,
+      "epoch": 0.12027491408934708,
+      "grad_norm": 0.16817958652973175,
+      "learning_rate": 0.00028208762886597935,
+      "loss": 0.9222366333007812,
+      "mean_token_accuracy": 0.7721138775348664,
+      "num_tokens": 359746.0,
+      "step": 140
+    },
+    {
+      "entropy": 0.8476657152175904,
+      "epoch": 0.12886597938144329,
+      "grad_norm": 0.15769453346729279,
+      "learning_rate": 0.0002807989690721649,
+      "loss": 0.8225071907043457,
+      "mean_token_accuracy": 0.7880932271480561,
+      "num_tokens": 383468.0,
+      "step": 150
+    },
+    {
+      "entropy": 0.8862757325172425,
+      "epoch": 0.13745704467353953,
+      "grad_norm": 0.17620904743671417,
+      "learning_rate": 0.0002795103092783505,
+      "loss": 0.8871653556823731,
+      "mean_token_accuracy": 0.7726393282413483,
+      "num_tokens": 409660.0,
+      "step": 160
+    },
+    {
+      "entropy": 0.9064472615718842,
+      "epoch": 0.14604810996563575,
+      "grad_norm": 0.15342549979686737,
+      "learning_rate": 0.0002782216494845361,
+      "loss": 0.9092922210693359,
+      "mean_token_accuracy": 0.7713103413581848,
+      "num_tokens": 437071.0,
+      "step": 170
+    },
+    {
+      "entropy": 0.9095455348491669,
+      "epoch": 0.15463917525773196,
+      "grad_norm": 0.17525391280651093,
+      "learning_rate": 0.00027693298969072165,
+      "loss": 0.8956180572509765,
+      "mean_token_accuracy": 0.7772489190101624,
+      "num_tokens": 460852.0,
+      "step": 180
+    },
+    {
+      "entropy": 0.8514094173908233,
+      "epoch": 0.16323024054982818,
+      "grad_norm": 0.15829823911190033,
+      "learning_rate": 0.0002756443298969072,
+      "loss": 0.8513413429260254,
+      "mean_token_accuracy": 0.779003256559372,
+      "num_tokens": 486114.0,
+      "step": 190
+    },
+    {
+      "entropy": 0.8995061337947845,
+      "epoch": 0.1718213058419244,
+      "grad_norm": 0.15622206032276154,
+      "learning_rate": 0.00027435567010309275,
+      "loss": 0.898340892791748,
+      "mean_token_accuracy": 0.7708433449268342,
+      "num_tokens": 512397.0,
+      "step": 200
+    },
+    {
+      "entropy": 0.8732269763946533,
+      "epoch": 0.18041237113402062,
+      "grad_norm": 0.17759552597999573,
+      "learning_rate": 0.0002730670103092783,
+      "loss": 0.8695596694946289,
+      "mean_token_accuracy": 0.7793173313140869,
+      "num_tokens": 536554.0,
+      "step": 210
+    },
+    {
+      "entropy": 0.9366106212139129,
+      "epoch": 0.18900343642611683,
+      "grad_norm": 0.1767597794532776,
+      "learning_rate": 0.0002717783505154639,
+      "loss": 0.9249121665954589,
+      "mean_token_accuracy": 0.7724991559982299,
+      "num_tokens": 561637.0,
+      "step": 220
+    },
+    {
+      "entropy": 0.8857321858406066,
+      "epoch": 0.19759450171821305,
+      "grad_norm": 0.20788735151290894,
+      "learning_rate": 0.0002704896907216495,
+      "loss": 0.8790401458740235,
+      "mean_token_accuracy": 0.7778670430183411,
+      "num_tokens": 586614.0,
+      "step": 230
+    },
+    {
+      "entropy": 0.9273143649101258,
+      "epoch": 0.20618556701030927,
+      "grad_norm": 0.16100187599658966,
+      "learning_rate": 0.00026920103092783505,
+      "loss": 0.9232232093811035,
+      "mean_token_accuracy": 0.7677364766597747,
+      "num_tokens": 612428.0,
+      "step": 240
+    },
+    {
+      "entropy": 0.8786617696285248,
+      "epoch": 0.21477663230240548,
+      "grad_norm": 0.1608886867761612,
+      "learning_rate": 0.00026791237113402063,
+      "loss": 0.8883329391479492,
+      "mean_token_accuracy": 0.7775621056556702,
+      "num_tokens": 638564.0,
+      "step": 250
+    },
+    {
+      "entropy": 0.9227964103221893,
+      "epoch": 0.22336769759450173,
+      "grad_norm": 0.1510680615901947,
+      "learning_rate": 0.00026662371134020615,
+      "loss": 0.9296038627624512,
+      "mean_token_accuracy": 0.7714014649391174,
+      "num_tokens": 665263.0,
+      "step": 260
+    },
+    {
+      "entropy": 0.9126189887523651,
+      "epoch": 0.23195876288659795,
+      "grad_norm": 0.16434459388256073,
+      "learning_rate": 0.00026533505154639173,
+      "loss": 0.9191689491271973,
+      "mean_token_accuracy": 0.7705935597419739,
+      "num_tokens": 691956.0,
+      "step": 270
+    },
+    {
+      "entropy": 0.8797955513000488,
+      "epoch": 0.24054982817869416,
+      "grad_norm": 0.1714707911014557,
+      "learning_rate": 0.0002640463917525773,
+      "loss": 0.8856119155883789,
+      "mean_token_accuracy": 0.7766761302947998,
+      "num_tokens": 716820.0,
+      "step": 280
+    },
+    {
+      "entropy": 0.877093505859375,
+      "epoch": 0.24914089347079038,
+      "grad_norm": 0.15326625108718872,
+      "learning_rate": 0.0002627577319587629,
+      "loss": 0.8566678047180176,
+      "mean_token_accuracy": 0.7812119901180268,
+      "num_tokens": 742221.0,
+      "step": 290
+    },
+    {
+      "entropy": 0.87042076587677,
+      "epoch": 0.25773195876288657,
+      "grad_norm": 0.1710847020149231,
+      "learning_rate": 0.00026146907216494846,
+      "loss": 0.8829298973083496,
+      "mean_token_accuracy": 0.7770332932472229,
+      "num_tokens": 765839.0,
+      "step": 300
+    },
+    {
+      "entropy": 0.8417458057403564,
+      "epoch": 0.2663230240549828,
+      "grad_norm": 0.1496821790933609,
+      "learning_rate": 0.00026018041237113403,
+      "loss": 0.8433545112609864,
+      "mean_token_accuracy": 0.7857938230037689,
+      "num_tokens": 791131.0,
+      "step": 310
+    },
+    {
+      "entropy": 0.9158999443054199,
+      "epoch": 0.27491408934707906,
+      "grad_norm": 0.17929035425186157,
+      "learning_rate": 0.00025889175257731955,
+      "loss": 0.9123490333557129,
+      "mean_token_accuracy": 0.775556480884552,
+      "num_tokens": 816984.0,
+      "step": 320
+    },
+    {
+      "entropy": 0.8314016401767731,
+      "epoch": 0.28350515463917525,
+      "grad_norm": 0.16051091253757477,
+      "learning_rate": 0.00025760309278350513,
+      "loss": 0.8279628753662109,
+      "mean_token_accuracy": 0.7862712442874908,
+      "num_tokens": 842220.0,
+      "step": 330
+    },
+    {
+      "entropy": 0.8677021145820618,
+      "epoch": 0.2920962199312715,
+      "grad_norm": 0.19157563149929047,
+      "learning_rate": 0.0002563144329896907,
+      "loss": 0.8552864074707032,
+      "mean_token_accuracy": 0.7835110187530517,
+      "num_tokens": 867868.0,
+      "step": 340
+    },
+    {
+      "entropy": 0.8919401407241822,
+      "epoch": 0.3006872852233677,
+      "grad_norm": 0.14258626103401184,
+      "learning_rate": 0.0002550257731958763,
+      "loss": 0.9019416809082031,
+      "mean_token_accuracy": 0.7760707855224609,
+      "num_tokens": 894676.0,
+      "step": 350
+    },
+    {
+      "entropy": 0.8415017545223236,
+      "epoch": 0.30927835051546393,
+      "grad_norm": 0.14830808341503143,
+      "learning_rate": 0.00025373711340206186,
+      "loss": 0.8072587013244629,
+      "mean_token_accuracy": 0.7912742376327515,
+      "num_tokens": 921632.0,
+      "step": 360
+    },
+    {
+      "entropy": 0.8371863842010498,
+      "epoch": 0.3178694158075601,
+      "grad_norm": 0.15538977086544037,
+      "learning_rate": 0.0002524484536082474,
+      "loss": 0.8308550834655761,
+      "mean_token_accuracy": 0.7849089920520782,
+      "num_tokens": 945955.0,
+      "step": 370
+    },
+    {
+      "entropy": 0.8338176369667053,
+      "epoch": 0.32646048109965636,
+      "grad_norm": 0.1711844801902771,
+      "learning_rate": 0.00025115979381443295,
+      "loss": 0.8479556083679199,
+      "mean_token_accuracy": 0.7850408136844635,
+      "num_tokens": 971508.0,
+      "step": 380
+    },
+    {
+      "entropy": 0.8876297354698182,
+      "epoch": 0.33505154639175255,
+      "grad_norm": 0.14902909100055695,
+      "learning_rate": 0.00024987113402061853,
+      "loss": 0.8821809768676758,
+      "mean_token_accuracy": 0.7764640510082245,
+      "num_tokens": 995124.0,
+      "step": 390
+    },
+    {
+      "entropy": 0.8212514638900756,
+      "epoch": 0.3436426116838488,
+      "grad_norm": 0.17950233817100525,
+      "learning_rate": 0.0002485824742268041,
+      "loss": 0.8255527496337891,
+      "mean_token_accuracy": 0.7870513379573822,
+      "num_tokens": 1019735.0,
+      "step": 400
+    },
+    {
+      "entropy": 0.9294387817382812,
+      "epoch": 0.35223367697594504,
+      "grad_norm": 0.15237703919410706,
+      "learning_rate": 0.0002472938144329897,
+      "loss": 0.9235431671142578,
+      "mean_token_accuracy": 0.7703655660152435,
+      "num_tokens": 1046295.0,
+      "step": 410
+    },
+    {
+      "entropy": 0.8429355442523956,
+      "epoch": 0.36082474226804123,
+      "grad_norm": 0.1589658558368683,
+      "learning_rate": 0.0002460051546391752,
+      "loss": 0.8329957008361817,
+      "mean_token_accuracy": 0.7897172749042511,
+      "num_tokens": 1072020.0,
+      "step": 420
+    },
+    {
+      "entropy": 0.8407810032367706,
+      "epoch": 0.3694158075601375,
+      "grad_norm": 0.17521116137504578,
+      "learning_rate": 0.0002447164948453608,
+      "loss": 0.8535200119018554,
+      "mean_token_accuracy": 0.7809976935386658,
+      "num_tokens": 1098420.0,
+      "step": 430
+    },
+    {
+      "entropy": 0.8515082120895385,
+      "epoch": 0.37800687285223367,
+      "grad_norm": 0.15218117833137512,
+      "learning_rate": 0.00024342783505154638,
+      "loss": 0.8459153175354004,
+      "mean_token_accuracy": 0.7833445191383361,
+      "num_tokens": 1124235.0,
+      "step": 440
+    },
+    {
+      "entropy": 0.8506015419960022,
+      "epoch": 0.3865979381443299,
+      "grad_norm": 0.1809423863887787,
+      "learning_rate": 0.00024213917525773193,
+      "loss": 0.8384982109069824,
+      "mean_token_accuracy": 0.7841372966766358,
+      "num_tokens": 1149636.0,
+      "step": 450
+    },
+    {
+      "entropy": 0.9110018074512481,
+      "epoch": 0.3951890034364261,
+      "grad_norm": 0.15085963904857635,
+      "learning_rate": 0.0002408505154639175,
+      "loss": 0.9191832542419434,
+      "mean_token_accuracy": 0.7757422685623169,
+      "num_tokens": 1173711.0,
+      "step": 460
+    },
+    {
+      "entropy": 0.9000201225280762,
+      "epoch": 0.40378006872852235,
+      "grad_norm": 0.16748838126659393,
+      "learning_rate": 0.00023956185567010308,
+      "loss": 0.9107230186462403,
+      "mean_token_accuracy": 0.7768173336982727,
+      "num_tokens": 1199340.0,
+      "step": 470
+    },
+    {
+      "entropy": 0.8751251935958863,
+      "epoch": 0.41237113402061853,
+      "grad_norm": 0.1593722254037857,
+      "learning_rate": 0.00023827319587628863,
+      "loss": 0.8804635047912598,
+      "mean_token_accuracy": 0.7814023733139038,
+      "num_tokens": 1226032.0,
+      "step": 480
+    },
+    {
+      "entropy": 0.8843371510505676,
+      "epoch": 0.4209621993127148,
+      "grad_norm": 0.12278851121664047,
+      "learning_rate": 0.0002369845360824742,
+      "loss": 0.8643080711364746,
+      "mean_token_accuracy": 0.7809630572795868,
+      "num_tokens": 1252769.0,
+      "step": 490
+    },
+    {
+      "entropy": 0.8834336459636688,
+      "epoch": 0.42955326460481097,
+      "grad_norm": 0.1367531716823578,
+      "learning_rate": 0.00023569587628865976,
+      "loss": 0.8869522094726563,
+      "mean_token_accuracy": 0.7775909185409546,
+      "num_tokens": 1278979.0,
+      "step": 500
+    },
+    {
+      "entropy": 0.9062523484230042,
+      "epoch": 0.4381443298969072,
+      "grad_norm": 0.14829133450984955,
+      "learning_rate": 0.00023440721649484533,
+      "loss": 0.9153310775756835,
+      "mean_token_accuracy": 0.7755910754203796,
+      "num_tokens": 1305141.0,
+      "step": 510
+    },
+    {
+      "entropy": 0.8907318472862243,
+      "epoch": 0.44673539518900346,
+      "grad_norm": 0.19640029966831207,
+      "learning_rate": 0.0002331185567010309,
+      "loss": 0.8736312866210938,
+      "mean_token_accuracy": 0.7823721885681152,
+      "num_tokens": 1333535.0,
+      "step": 520
+    },
+    {
+      "entropy": 0.9001505076885223,
+      "epoch": 0.45532646048109965,
+      "grad_norm": 0.1615416407585144,
+      "learning_rate": 0.00023182989690721648,
+      "loss": 0.9240070343017578,
+      "mean_token_accuracy": 0.7720246851444245,
+      "num_tokens": 1359100.0,
+      "step": 530
+    },
+    {
+      "entropy": 0.8964834988117218,
+      "epoch": 0.4639175257731959,
+      "grad_norm": 0.16148629784584045,
+      "learning_rate": 0.00023054123711340206,
+      "loss": 0.9040334701538086,
+      "mean_token_accuracy": 0.7697900414466858,
+      "num_tokens": 1386642.0,
+      "step": 540
+    },
+    {
+      "entropy": 0.9152944028377533,
+      "epoch": 0.4725085910652921,
+      "grad_norm": 0.16060467064380646,
+      "learning_rate": 0.00022925257731958758,
+      "loss": 0.9094497680664062,
+      "mean_token_accuracy": 0.771862530708313,
+      "num_tokens": 1412712.0,
+      "step": 550
+    },
+    {
+      "entropy": 0.8443439364433288,
+      "epoch": 0.48109965635738833,
+      "grad_norm": 0.16139253973960876,
+      "learning_rate": 0.00022796391752577316,
+      "loss": 0.8445466041564942,
+      "mean_token_accuracy": 0.7862408220767975,
+      "num_tokens": 1439208.0,
+      "step": 560
+    },
+    {
+      "entropy": 0.8669894218444825,
+      "epoch": 0.4896907216494845,
+      "grad_norm": 0.13149064779281616,
+      "learning_rate": 0.00022667525773195873,
+      "loss": 0.8791823387145996,
+      "mean_token_accuracy": 0.7771342694759369,
+      "num_tokens": 1466602.0,
+      "step": 570
+    },
+    {
+      "entropy": 0.8713687837123871,
+      "epoch": 0.49828178694158076,
+      "grad_norm": 0.15979787707328796,
+      "learning_rate": 0.0002253865979381443,
+      "loss": 0.8594341278076172,
+      "mean_token_accuracy": 0.7810667157173157,
+      "num_tokens": 1493048.0,
+      "step": 580
+    },
+    {
+      "entropy": 0.9155345737934113,
+      "epoch": 0.506872852233677,
+      "grad_norm": 0.18420852720737457,
+      "learning_rate": 0.00022409793814432989,
+      "loss": 0.915378475189209,
+      "mean_token_accuracy": 0.7724271833896637,
+      "num_tokens": 1517512.0,
+      "step": 590
+    },
+    {
+      "entropy": 0.891716331243515,
+      "epoch": 0.5154639175257731,
+      "grad_norm": 0.1665642410516739,
+      "learning_rate": 0.00022280927835051546,
+      "loss": 0.8950610160827637,
+      "mean_token_accuracy": 0.777475905418396,
+      "num_tokens": 1542765.0,
+      "step": 600
+    },
+    {
+      "entropy": 0.9547060489654541,
+      "epoch": 0.5240549828178694,
+      "grad_norm": 0.19496814906597137,
+      "learning_rate": 0.000221520618556701,
+      "loss": 0.9572202682495117,
+      "mean_token_accuracy": 0.7599396347999573,
+      "num_tokens": 1570704.0,
+      "step": 610
+    },
+    {
+      "entropy": 0.894726449251175,
+      "epoch": 0.5326460481099656,
+      "grad_norm": 0.162724569439888,
+      "learning_rate": 0.00022023195876288656,
+      "loss": 0.8876501083374023,
+      "mean_token_accuracy": 0.7815112411975861,
+      "num_tokens": 1594675.0,
+      "step": 620
+    },
+    {
+      "entropy": 0.8966928541660308,
+      "epoch": 0.5412371134020618,
+      "grad_norm": 0.1820981204509735,
+      "learning_rate": 0.00021894329896907214,
+      "loss": 0.8992101669311523,
+      "mean_token_accuracy": 0.7774552226066589,
+      "num_tokens": 1621381.0,
+      "step": 630
+    },
+    {
+      "entropy": 0.8320938229560852,
+      "epoch": 0.5498281786941581,
+      "grad_norm": 0.18641315400600433,
+      "learning_rate": 0.0002176546391752577,
+      "loss": 0.8333114624023438,
+      "mean_token_accuracy": 0.7836415946483613,
+      "num_tokens": 1646589.0,
+      "step": 640
+    },
+    {
+      "entropy": 0.9350853085517883,
+      "epoch": 0.5584192439862543,
+      "grad_norm": 0.15154911577701569,
+      "learning_rate": 0.0002163659793814433,
+      "loss": 0.9300890922546386,
+      "mean_token_accuracy": 0.7711542308330536,
+      "num_tokens": 1672281.0,
+      "step": 650
+    },
+    {
+      "entropy": 0.8372068881988526,
+      "epoch": 0.5670103092783505,
+      "grad_norm": 0.157988041639328,
+      "learning_rate": 0.00021507731958762884,
+      "loss": 0.8490092277526855,
+      "mean_token_accuracy": 0.7828391373157502,
+      "num_tokens": 1697203.0,
+      "step": 660
+    },
+    {
+      "entropy": 0.8175156831741333,
+      "epoch": 0.5756013745704467,
+      "grad_norm": 0.16615159809589386,
+      "learning_rate": 0.0002137886597938144,
+      "loss": 0.7962069988250733,
+      "mean_token_accuracy": 0.793628740310669,
+      "num_tokens": 1722977.0,
+      "step": 670
+    },
+    {
+      "entropy": 0.9293979525566101,
+      "epoch": 0.584192439862543,
+      "grad_norm": 0.13969369232654572,
+      "learning_rate": 0.0002125,
+      "loss": 0.9364766120910645,
+      "mean_token_accuracy": 0.7693193078041076,
+      "num_tokens": 1748869.0,
+      "step": 680
+    },
+    {
+      "entropy": 0.8958774268627167,
+      "epoch": 0.5927835051546392,
+      "grad_norm": 0.14425267279148102,
+      "learning_rate": 0.00021121134020618556,
+      "loss": 0.8842126846313476,
+      "mean_token_accuracy": 0.7759437382221221,
+      "num_tokens": 1774715.0,
+      "step": 690
+    },
+    {
+      "entropy": 0.929528433084488,
+      "epoch": 0.6013745704467354,
+      "grad_norm": 0.15875831246376038,
+      "learning_rate": 0.0002099226804123711,
+      "loss": 0.9353519439697265,
+      "mean_token_accuracy": 0.769885140657425,
+      "num_tokens": 1800430.0,
+      "step": 700
+    },
+    {
+      "entropy": 0.8651838064193725,
+      "epoch": 0.6099656357388317,
+      "grad_norm": 0.19464287161827087,
+      "learning_rate": 0.00020863402061855666,
+      "loss": 0.8678315162658692,
+      "mean_token_accuracy": 0.7783544957637787,
+      "num_tokens": 1824631.0,
+      "step": 710
+    },
+    {
+      "entropy": 0.874721884727478,
+      "epoch": 0.6185567010309279,
+      "grad_norm": 0.17015859484672546,
+      "learning_rate": 0.00020734536082474224,
+      "loss": 0.847961139678955,
+      "mean_token_accuracy": 0.7813271641731262,
+      "num_tokens": 1849593.0,
+      "step": 720
+    },
+    {
+      "entropy": 0.8706691145896912,
+      "epoch": 0.627147766323024,
+      "grad_norm": 0.15960460901260376,
+      "learning_rate": 0.00020605670103092781,
+      "loss": 0.8743976593017578,
+      "mean_token_accuracy": 0.7772314369678497,
+      "num_tokens": 1874593.0,
+      "step": 730
+    },
+    {
+      "entropy": 0.824019831418991,
+      "epoch": 0.6357388316151202,
+      "grad_norm": 0.16192808747291565,
+      "learning_rate": 0.0002047680412371134,
+      "loss": 0.8214377403259278,
+      "mean_token_accuracy": 0.7877136409282685,
+      "num_tokens": 1899592.0,
+      "step": 740
+    },
+    {
+      "entropy": 0.853943508863449,
+      "epoch": 0.6443298969072165,
+      "grad_norm": 0.14331476390361786,
+      "learning_rate": 0.00020347938144329897,
+      "loss": 0.8542372703552246,
+      "mean_token_accuracy": 0.7825242638587951,
+      "num_tokens": 1925105.0,
+      "step": 750
+    },
+    {
+      "entropy": 0.9206405401229858,
+      "epoch": 0.6529209621993127,
+      "grad_norm": 0.16269327700138092,
+      "learning_rate": 0.00020219072164948454,
+      "loss": 0.9131902694702149,
+      "mean_token_accuracy": 0.7722071945667267,
+      "num_tokens": 1952126.0,
+      "step": 760
+    },
+    {
+      "entropy": 0.8438917219638824,
+      "epoch": 0.6615120274914089,
+      "grad_norm": 0.18783624470233917,
+      "learning_rate": 0.00020090206185567006,
+      "loss": 0.8612903594970703,
+      "mean_token_accuracy": 0.7805359125137329,
+      "num_tokens": 1976727.0,
+      "step": 770
+    },
+    {
+      "entropy": 0.8782703518867493,
+      "epoch": 0.6701030927835051,
+      "grad_norm": 0.1749051958322525,
+      "learning_rate": 0.00019961340206185564,
+      "loss": 0.8877654075622559,
+      "mean_token_accuracy": 0.775201940536499,
+      "num_tokens": 2004204.0,
+      "step": 780
+    },
+    {
+      "entropy": 0.9036802053451538,
+      "epoch": 0.6786941580756014,
+      "grad_norm": 0.17728140950202942,
+      "learning_rate": 0.00019832474226804121,
+      "loss": 0.8946940422058105,
+      "mean_token_accuracy": 0.7781059086322785,
+      "num_tokens": 2031926.0,
+      "step": 790
+    },
+    {
+      "entropy": 0.9057375133037567,
+      "epoch": 0.6872852233676976,
+      "grad_norm": 0.1804327517747879,
+      "learning_rate": 0.0001970360824742268,
+      "loss": 0.9067662239074707,
+      "mean_token_accuracy": 0.7786577641963959,
+      "num_tokens": 2059628.0,
+      "step": 800
+    },
+    {
+      "entropy": 0.8421910405158997,
+      "epoch": 0.6958762886597938,
+      "grad_norm": 0.1570490598678589,
+      "learning_rate": 0.00019574742268041237,
+      "loss": 0.8346380233764649,
+      "mean_token_accuracy": 0.7839872419834137,
+      "num_tokens": 2083943.0,
+      "step": 810
+    },
+    {
+      "entropy": 0.8743102788925171,
+      "epoch": 0.7044673539518901,
+      "grad_norm": 0.1737687736749649,
+      "learning_rate": 0.00019445876288659794,
+      "loss": 0.8622690200805664,
+      "mean_token_accuracy": 0.7791435241699218,
+      "num_tokens": 2109444.0,
+      "step": 820
+    },
+    {
+      "entropy": 0.8926449298858643,
+      "epoch": 0.7130584192439863,
+      "grad_norm": 0.1972922682762146,
+      "learning_rate": 0.0001931701030927835,
+      "loss": 0.895075511932373,
+      "mean_token_accuracy": 0.7716107368469238,
+      "num_tokens": 2134209.0,
+      "step": 830
+    },
+    {
+      "entropy": 0.8712981760501861,
+      "epoch": 0.7216494845360825,
+      "grad_norm": 0.16580969095230103,
+      "learning_rate": 0.00019188144329896904,
+      "loss": 0.8615394592285156,
+      "mean_token_accuracy": 0.780440878868103,
+      "num_tokens": 2160322.0,
+      "step": 840
+    },
+    {
+      "entropy": 0.7872600853443146,
+      "epoch": 0.7302405498281787,
+      "grad_norm": 0.17089906334877014,
+      "learning_rate": 0.00019059278350515462,
+      "loss": 0.7738356590270996,
+      "mean_token_accuracy": 0.7984008610248565,
+      "num_tokens": 2186673.0,
+      "step": 850
+    },
+    {
+      "entropy": 0.8024394273757934,
+      "epoch": 0.738831615120275,
+      "grad_norm": 0.15015770494937897,
+      "learning_rate": 0.0001893041237113402,
+      "loss": 0.801088809967041,
+      "mean_token_accuracy": 0.7919513285160065,
+      "num_tokens": 2211288.0,
+      "step": 860
+    },
+    {
+      "entropy": 0.8289069414138794,
+      "epoch": 0.7474226804123711,
+      "grad_norm": 0.15804092586040497,
+      "learning_rate": 0.00018801546391752577,
+      "loss": 0.8228999137878418,
+      "mean_token_accuracy": 0.7900383174419403,
+      "num_tokens": 2235494.0,
+      "step": 870
+    },
+    {
+      "entropy": 0.8337613046169281,
+      "epoch": 0.7560137457044673,
+      "grad_norm": 0.16564202308654785,
+      "learning_rate": 0.00018672680412371132,
+      "loss": 0.8357515335083008,
+      "mean_token_accuracy": 0.7796721041202546,
+      "num_tokens": 2261853.0,
+      "step": 880
+    },
+    {
+      "entropy": 0.8514188408851624,
+      "epoch": 0.7646048109965635,
+      "grad_norm": 0.15763898193836212,
+      "learning_rate": 0.0001854381443298969,
+      "loss": 0.8596377372741699,
+      "mean_token_accuracy": 0.781908392906189,
+      "num_tokens": 2288235.0,
+      "step": 890
+    },
+    {
+      "entropy": 0.8182059168815613,
+      "epoch": 0.7731958762886598,
+      "grad_norm": 0.14843082427978516,
+      "learning_rate": 0.00018414948453608247,
+      "loss": 0.8056727409362793,
+      "mean_token_accuracy": 0.7887085616588593,
+      "num_tokens": 2312636.0,
+      "step": 900
+    },
+    {
+      "entropy": 0.8019815027713776,
+      "epoch": 0.781786941580756,
+      "grad_norm": 0.1444498896598816,
+      "learning_rate": 0.00018286082474226804,
+      "loss": 0.7791801452636719,
+      "mean_token_accuracy": 0.7921311616897583,
+      "num_tokens": 2337590.0,
+      "step": 910
+    },
+    {
+      "entropy": 0.8811851978302002,
+      "epoch": 0.7903780068728522,
+      "grad_norm": 0.20295798778533936,
+      "learning_rate": 0.0001815721649484536,
+      "loss": 0.9093887329101562,
+      "mean_token_accuracy": 0.7765615165233613,
+      "num_tokens": 2364733.0,
+      "step": 920
+    },
+    {
+      "entropy": 0.8845545649528503,
+      "epoch": 0.7989690721649485,
+      "grad_norm": 0.16211840510368347,
+      "learning_rate": 0.00018028350515463914,
+      "loss": 0.87130765914917,
+      "mean_token_accuracy": 0.773591297864914,
+      "num_tokens": 2389138.0,
+      "step": 930
+    },
+    {
+      "entropy": 0.8366316437721253,
+      "epoch": 0.8075601374570447,
+      "grad_norm": 0.1796175092458725,
+      "learning_rate": 0.00017899484536082472,
+      "loss": 0.8393503189086914,
+      "mean_token_accuracy": 0.7834821224212647,
+      "num_tokens": 2416288.0,
+      "step": 940
+    },
+    {
+      "entropy": 0.8687749207019806,
+      "epoch": 0.8161512027491409,
+      "grad_norm": 0.17602433264255524,
+      "learning_rate": 0.0001777061855670103,
+      "loss": 0.8831927299499511,
+      "mean_token_accuracy": 0.7806714773178101,
+      "num_tokens": 2442282.0,
+      "step": 950
+    },
+    {
+      "entropy": 0.8602447748184204,
+      "epoch": 0.8247422680412371,
+      "grad_norm": 0.14779216051101685,
+      "learning_rate": 0.00017641752577319587,
+      "loss": 0.8557798385620117,
+      "mean_token_accuracy": 0.7800233542919159,
+      "num_tokens": 2469865.0,
+      "step": 960
+    },
+    {
+      "entropy": 0.8987692832946778,
+      "epoch": 0.8333333333333334,
+      "grad_norm": 0.16065306961536407,
+      "learning_rate": 0.00017512886597938145,
+      "loss": 0.8930519104003907,
+      "mean_token_accuracy": 0.7771966874599456,
+      "num_tokens": 2497716.0,
+      "step": 970
+    },
+    {
+      "entropy": 0.847240948677063,
+      "epoch": 0.8419243986254296,
+      "grad_norm": 0.16063280403614044,
+      "learning_rate": 0.00017384020618556702,
+      "loss": 0.8212727546691895,
+      "mean_token_accuracy": 0.7900629937648773,
+      "num_tokens": 2523292.0,
+      "step": 980
+    },
+    {
+      "entropy": 0.8685175359249115,
+      "epoch": 0.8505154639175257,
+      "grad_norm": 0.19293011724948883,
+      "learning_rate": 0.00017255154639175254,
+      "loss": 0.8839738845825196,
+      "mean_token_accuracy": 0.7780547857284545,
+      "num_tokens": 2547529.0,
+      "step": 990
+    },
+    {
+      "entropy": 0.8589172720909118,
+      "epoch": 0.8591065292096219,
+      "grad_norm": 0.13081640005111694,
+      "learning_rate": 0.00017126288659793812,
+      "loss": 0.8760468482971191,
+      "mean_token_accuracy": 0.7810062050819397,
+      "num_tokens": 2574943.0,
+      "step": 1000
+    },
+    {
+      "entropy": 0.8676146745681763,
+      "epoch": 0.8676975945017182,
+      "grad_norm": 0.18423762917518616,
+      "learning_rate": 0.0001699742268041237,
+      "loss": 0.8576597213745117,
+      "mean_token_accuracy": 0.7845786452293396,
+      "num_tokens": 2599770.0,
+      "step": 1010
+    },
+    {
+      "entropy": 0.8578267991542816,
+      "epoch": 0.8762886597938144,
+      "grad_norm": 0.15971733629703522,
+      "learning_rate": 0.00016868556701030927,
+      "loss": 0.8597639083862305,
+      "mean_token_accuracy": 0.7836935818195343,
+      "num_tokens": 2629050.0,
+      "step": 1020
+    },
+    {
+      "entropy": 0.78883256316185,
+      "epoch": 0.8848797250859106,
+      "grad_norm": 0.19334255158901215,
+      "learning_rate": 0.00016739690721649485,
+      "loss": 0.775438642501831,
+      "mean_token_accuracy": 0.7994126856327057,
+      "num_tokens": 2652418.0,
+      "step": 1030
+    },
+    {
+      "entropy": 0.827907818555832,
+      "epoch": 0.8934707903780069,
+      "grad_norm": 0.17899778485298157,
+      "learning_rate": 0.0001661082474226804,
+      "loss": 0.8155632019042969,
+      "mean_token_accuracy": 0.7867587625980377,
+      "num_tokens": 2677020.0,
+      "step": 1040
+    },
+    {
+      "entropy": 0.8488841831684113,
+      "epoch": 0.9020618556701031,
+      "grad_norm": 0.17594760656356812,
+      "learning_rate": 0.00016481958762886597,
+      "loss": 0.8443450927734375,
+      "mean_token_accuracy": 0.7823632657527924,
+      "num_tokens": 2700589.0,
+      "step": 1050
+    },
+    {
+      "entropy": 0.9077254295349121,
+      "epoch": 0.9106529209621993,
+      "grad_norm": 0.17107197642326355,
+      "learning_rate": 0.00016353092783505152,
+      "loss": 0.9043283462524414,
+      "mean_token_accuracy": 0.7685754477977753,
+      "num_tokens": 2727925.0,
+      "step": 1060
+    },
+    {
+      "entropy": 0.8681511282920837,
+      "epoch": 0.9192439862542955,
+      "grad_norm": 0.16261200606822968,
+      "learning_rate": 0.0001622422680412371,
+      "loss": 0.8636059761047363,
+      "mean_token_accuracy": 0.7801066517829895,
+      "num_tokens": 2754434.0,
+      "step": 1070
+    },
+    {
+      "entropy": 0.8201885402202607,
+      "epoch": 0.9278350515463918,
+      "grad_norm": 0.15494069457054138,
+      "learning_rate": 0.00016095360824742267,
+      "loss": 0.8267613410949707,
+      "mean_token_accuracy": 0.7909399390220642,
+      "num_tokens": 2778856.0,
+      "step": 1080
+    },
+    {
+      "entropy": 0.8281429171562195,
+      "epoch": 0.936426116838488,
+      "grad_norm": 0.15491366386413574,
+      "learning_rate": 0.00015966494845360822,
+      "loss": 0.8283931732177734,
+      "mean_token_accuracy": 0.7858192563056946,
+      "num_tokens": 2802467.0,
+      "step": 1090
+    },
+    {
+      "entropy": 0.8585192024707794,
+      "epoch": 0.9450171821305842,
+      "grad_norm": 0.16744765639305115,
+      "learning_rate": 0.0001583762886597938,
+      "loss": 0.8450031280517578,
+      "mean_token_accuracy": 0.7824375092983246,
+      "num_tokens": 2828404.0,
+      "step": 1100
+    },
+    {
+      "entropy": 0.9703075528144837,
+      "epoch": 0.9536082474226805,
+      "grad_norm": 0.16334781050682068,
+      "learning_rate": 0.00015708762886597937,
+      "loss": 0.9652547836303711,
+      "mean_token_accuracy": 0.7639409124851226,
+      "num_tokens": 2854621.0,
+      "step": 1110
+    },
+    {
+      "entropy": 0.8723820388317108,
+      "epoch": 0.9621993127147767,
+      "grad_norm": 0.15771399438381195,
+      "learning_rate": 0.00015579896907216495,
+      "loss": 0.8663213729858399,
+      "mean_token_accuracy": 0.7808757722377777,
+      "num_tokens": 2880448.0,
+      "step": 1120
+    },
+    {
+      "entropy": 0.8375514090061188,
+      "epoch": 0.9707903780068728,
+      "grad_norm": 0.16101613640785217,
+      "learning_rate": 0.0001545103092783505,
+      "loss": 0.8438360214233398,
+      "mean_token_accuracy": 0.7864978194236756,
+      "num_tokens": 2907346.0,
+      "step": 1130
+    },
+    {
+      "entropy": 0.836071914434433,
+      "epoch": 0.979381443298969,
+      "grad_norm": 0.19959381222724915,
+      "learning_rate": 0.00015322164948453605,
+      "loss": 0.8347750663757324,
+      "mean_token_accuracy": 0.7878862261772156,
+      "num_tokens": 2932769.0,
+      "step": 1140
+    },
+    {
+      "entropy": 0.8218921303749085,
+      "epoch": 0.9879725085910653,
+      "grad_norm": 0.17999783158302307,
+      "learning_rate": 0.00015193298969072162,
+      "loss": 0.81912841796875,
+      "mean_token_accuracy": 0.7904701411724091,
+      "num_tokens": 2956439.0,
+      "step": 1150
+    },
+    {
+      "entropy": 0.8836544632911683,
+      "epoch": 0.9965635738831615,
+      "grad_norm": 0.1764528602361679,
+      "learning_rate": 0.0001506443298969072,
+      "loss": 0.8860954284667969,
+      "mean_token_accuracy": 0.7801528871059418,
+      "num_tokens": 2983309.0,
+      "step": 1160
+    },
+    {
+      "entropy": 0.8893479645252228,
+      "epoch": 1.0051546391752577,
+      "grad_norm": 0.153923898935318,
+      "learning_rate": 0.00014935567010309277,
+      "loss": 0.8747451782226563,
+      "mean_token_accuracy": 0.7765235006809235,
+      "num_tokens": 3010175.0,
+      "step": 1170
+    },
+    {
+      "entropy": 0.8393925905227662,
+      "epoch": 1.013745704467354,
+      "grad_norm": 0.16801179945468903,
+      "learning_rate": 0.00014806701030927835,
+      "loss": 0.8316682815551758,
+      "mean_token_accuracy": 0.7851088464260101,
+      "num_tokens": 3036705.0,
+      "step": 1180
+    },
+    {
+      "entropy": 0.8521030902862549,
+      "epoch": 1.02233676975945,
+      "grad_norm": 0.16726894676685333,
+      "learning_rate": 0.0001467783505154639,
+      "loss": 0.8441553115844727,
+      "mean_token_accuracy": 0.7883111536502838,
+      "num_tokens": 3062184.0,
+      "step": 1190
+    },
+    {
+      "entropy": 0.8722131133079529,
+      "epoch": 1.0309278350515463,
+      "grad_norm": 0.1777733713388443,
+      "learning_rate": 0.00014548969072164948,
+      "loss": 0.8671235084533692,
+      "mean_token_accuracy": 0.7776070535182953,
+      "num_tokens": 3089854.0,
+      "step": 1200
+    },
+    {
+      "entropy": 0.8229423344135285,
+      "epoch": 1.0395189003436427,
+      "grad_norm": 0.13976716995239258,
+      "learning_rate": 0.00014420103092783505,
+      "loss": 0.8249235153198242,
+      "mean_token_accuracy": 0.7889478981494904,
+      "num_tokens": 3116938.0,
+      "step": 1210
+    },
+    {
+      "entropy": 0.8179661929607391,
+      "epoch": 1.0481099656357389,
+      "grad_norm": 0.16781210899353027,
+      "learning_rate": 0.0001429123711340206,
+      "loss": 0.8102549552917481,
+      "mean_token_accuracy": 0.7927488446235657,
+      "num_tokens": 3143825.0,
+      "step": 1220
+    },
+    {
+      "entropy": 0.7997653067111969,
+      "epoch": 1.056701030927835,
+      "grad_norm": 0.1647312045097351,
+      "learning_rate": 0.00014162371134020618,
+      "loss": 0.7900454998016357,
+      "mean_token_accuracy": 0.7910003781318664,
+      "num_tokens": 3168362.0,
+      "step": 1230
+    },
+    {
+      "entropy": 0.8539471685886383,
+      "epoch": 1.0652920962199313,
+      "grad_norm": 0.16146644949913025,
+      "learning_rate": 0.00014033505154639175,
+      "loss": 0.8489527702331543,
+      "mean_token_accuracy": 0.7828418910503387,
+      "num_tokens": 3192789.0,
+      "step": 1240
+    },
+    {
+      "entropy": 0.7987524449825287,
+      "epoch": 1.0738831615120275,
+      "grad_norm": 0.16997887194156647,
+      "learning_rate": 0.0001390463917525773,
+      "loss": 0.7815001964569092,
+      "mean_token_accuracy": 0.7958592414855957,
+      "num_tokens": 3218189.0,
+      "step": 1250
+    },
+    {
+      "entropy": 0.8433911919593811,
+      "epoch": 1.0824742268041236,
+      "grad_norm": 0.20293352007865906,
+      "learning_rate": 0.00013775773195876288,
+      "loss": 0.8536633491516114,
+      "mean_token_accuracy": 0.7847901046276092,
+      "num_tokens": 3242357.0,
+      "step": 1260
+    },
+    {
+      "entropy": 0.7989895343780518,
+      "epoch": 1.0910652920962198,
+      "grad_norm": 0.17577318847179413,
+      "learning_rate": 0.00013646907216494845,
+      "loss": 0.795603084564209,
+      "mean_token_accuracy": 0.7937948822975158,
+      "num_tokens": 3266634.0,
+      "step": 1270
+    },
+    {
+      "entropy": 0.849328339099884,
+      "epoch": 1.0996563573883162,
+      "grad_norm": 0.16344796121120453,
+      "learning_rate": 0.000135180412371134,
+      "loss": 0.8569498062133789,
+      "mean_token_accuracy": 0.784570574760437,
+      "num_tokens": 3292457.0,
+      "step": 1280
+    },
+    {
+      "entropy": 0.9115039288997651,
+      "epoch": 1.1082474226804124,
+      "grad_norm": 0.16120180487632751,
+      "learning_rate": 0.00013389175257731958,
+      "loss": 0.9172770500183105,
+      "mean_token_accuracy": 0.7738337874412536,
+      "num_tokens": 3319975.0,
+      "step": 1290
+    },
+    {
+      "entropy": 0.9116923689842225,
+      "epoch": 1.1168384879725086,
+      "grad_norm": 0.1507127285003662,
+      "learning_rate": 0.00013260309278350513,
+      "loss": 0.9115254402160644,
+      "mean_token_accuracy": 0.7727989673614502,
+      "num_tokens": 3349370.0,
+      "step": 1300
+    },
+    {
+      "entropy": 0.8705029428005219,
+      "epoch": 1.1254295532646048,
+      "grad_norm": 0.16798697412014008,
+      "learning_rate": 0.0001313144329896907,
+      "loss": 0.8673007011413574,
+      "mean_token_accuracy": 0.7829907596111297,
+      "num_tokens": 3376086.0,
+      "step": 1310
+    },
+    {
+      "entropy": 0.854372763633728,
+      "epoch": 1.134020618556701,
+      "grad_norm": 0.17355380952358246,
+      "learning_rate": 0.00013002577319587628,
+      "loss": 0.8472127914428711,
+      "mean_token_accuracy": 0.783273845911026,
+      "num_tokens": 3402983.0,
+      "step": 1320
+    },
+    {
+      "entropy": 0.8452303171157837,
+      "epoch": 1.1426116838487972,
+      "grad_norm": 0.17868086695671082,
+      "learning_rate": 0.00012873711340206183,
+      "loss": 0.846036434173584,
+      "mean_token_accuracy": 0.781141072511673,
+      "num_tokens": 3430354.0,
+      "step": 1330
+    },
+    {
+      "entropy": 0.8574681997299194,
+      "epoch": 1.1512027491408934,
+      "grad_norm": 0.17731839418411255,
+      "learning_rate": 0.0001274484536082474,
+      "loss": 0.8377547264099121,
+      "mean_token_accuracy": 0.7869145214557648,
+      "num_tokens": 3456008.0,
+      "step": 1340
+    },
+    {
+      "entropy": 0.8843120455741882,
+      "epoch": 1.1597938144329896,
+      "grad_norm": 0.17710870504379272,
+      "learning_rate": 0.00012615979381443298,
+      "loss": 0.894843578338623,
+      "mean_token_accuracy": 0.7717236757278443,
+      "num_tokens": 3481183.0,
+      "step": 1350
+    },
+    {
+      "entropy": 0.82523393034935,
+      "epoch": 1.168384879725086,
+      "grad_norm": 0.1979239135980606,
+      "learning_rate": 0.00012487113402061855,
+      "loss": 0.8107866287231446,
+      "mean_token_accuracy": 0.7911741256713867,
+      "num_tokens": 3508453.0,
+      "step": 1360
+    },
+    {
+      "entropy": 0.7964858949184418,
+      "epoch": 1.1769759450171822,
+      "grad_norm": 0.14254362881183624,
+      "learning_rate": 0.0001235824742268041,
+      "loss": 0.7863273620605469,
+      "mean_token_accuracy": 0.7948981881141662,
+      "num_tokens": 3534922.0,
+      "step": 1370
+    },
+    {
+      "entropy": 0.8244574010372162,
+      "epoch": 1.1855670103092784,
+      "grad_norm": 0.17570418119430542,
+      "learning_rate": 0.00012229381443298968,
+      "loss": 0.8179922103881836,
+      "mean_token_accuracy": 0.7896173536777497,
+      "num_tokens": 3559367.0,
+      "step": 1380
+    },
+    {
+      "entropy": 0.8134994566440582,
+      "epoch": 1.1941580756013745,
+      "grad_norm": 0.1930961012840271,
+      "learning_rate": 0.00012100515463917526,
+      "loss": 0.8178740501403808,
+      "mean_token_accuracy": 0.7915168821811676,
+      "num_tokens": 3585435.0,
+      "step": 1390
+    },
+    {
+      "entropy": 0.8384514689445496,
+      "epoch": 1.2027491408934707,
+      "grad_norm": 0.17272983491420746,
+      "learning_rate": 0.00011971649484536082,
+      "loss": 0.8454058647155762,
+      "mean_token_accuracy": 0.7868218302726746,
+      "num_tokens": 3613232.0,
+      "step": 1400
+    },
+    {
+      "entropy": 0.9280701756477356,
+      "epoch": 1.211340206185567,
+      "grad_norm": 0.1684003472328186,
+      "learning_rate": 0.00011842783505154638,
+      "loss": 0.9219778060913086,
+      "mean_token_accuracy": 0.7722154259681702,
+      "num_tokens": 3641457.0,
+      "step": 1410
+    },
+    {
+      "entropy": 0.832447475194931,
+      "epoch": 1.2199312714776633,
+      "grad_norm": 0.1773725152015686,
+      "learning_rate": 0.00011713917525773194,
+      "loss": 0.8198453903198242,
+      "mean_token_accuracy": 0.7914207100868225,
+      "num_tokens": 3666482.0,
+      "step": 1420
+    },
+    {
+      "entropy": 0.8738070428371429,
+      "epoch": 1.2285223367697595,
+      "grad_norm": 0.18506592512130737,
+      "learning_rate": 0.00011585051546391752,
+      "loss": 0.8723980903625488,
+      "mean_token_accuracy": 0.778689730167389,
+      "num_tokens": 3692349.0,
+      "step": 1430
+    },
+    {
+      "entropy": 0.8338236033916473,
+      "epoch": 1.2371134020618557,
+      "grad_norm": 0.17101170122623444,
+      "learning_rate": 0.00011456185567010308,
+      "loss": 0.837257194519043,
+      "mean_token_accuracy": 0.7857484579086303,
+      "num_tokens": 3717280.0,
+      "step": 1440
+    },
+    {
+      "entropy": 0.8157192528247833,
+      "epoch": 1.245704467353952,
+      "grad_norm": 0.15433582663536072,
+      "learning_rate": 0.00011327319587628864,
+      "loss": 0.8053474426269531,
+      "mean_token_accuracy": 0.7934693098068237,
+      "num_tokens": 3742595.0,
+      "step": 1450
+    },
+    {
+      "entropy": 0.8770197212696076,
+      "epoch": 1.254295532646048,
+      "grad_norm": 0.17446650564670563,
+      "learning_rate": 0.00011198453608247422,
+      "loss": 0.880764389038086,
+      "mean_token_accuracy": 0.7784990787506103,
+      "num_tokens": 3767913.0,
+      "step": 1460
+    },
+    {
+      "entropy": 0.8850863575935364,
+      "epoch": 1.2628865979381443,
+      "grad_norm": 0.186509370803833,
+      "learning_rate": 0.0001106958762886598,
+      "loss": 0.870394229888916,
+      "mean_token_accuracy": 0.7804449439048767,
+      "num_tokens": 3792206.0,
+      "step": 1470
+    },
+    {
+      "entropy": 0.8432760655879974,
+      "epoch": 1.2714776632302405,
+      "grad_norm": 0.17969489097595215,
+      "learning_rate": 0.00010940721649484534,
+      "loss": 0.8241963386535645,
+      "mean_token_accuracy": 0.7888677537441253,
+      "num_tokens": 3818423.0,
+      "step": 1480
+    },
+    {
+      "entropy": 0.8146856188774109,
+      "epoch": 1.2800687285223367,
+      "grad_norm": 0.15870600938796997,
+      "learning_rate": 0.00010811855670103092,
+      "loss": 0.8046764373779297,
+      "mean_token_accuracy": 0.7905527114868164,
+      "num_tokens": 3844217.0,
+      "step": 1490
+    },
+    {
+      "entropy": 0.8613445937633515,
+      "epoch": 1.2886597938144329,
+      "grad_norm": 0.16997215151786804,
+      "learning_rate": 0.00010682989690721648,
+      "loss": 0.8584735870361329,
+      "mean_token_accuracy": 0.7848546147346497,
+      "num_tokens": 3873420.0,
+      "step": 1500
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 2328,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 6335948395777536.0,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-1500/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2af45870b4c4d611381846df6bdd4a32cb292a7b5897b79b7e5f196be57c596
+size 5585

checkpoint-2000/README.md

@@ -0,0 +1,209 @@
+---
+base_model: HuggingFaceTB/SmolLM2-135M-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:HuggingFaceTB/SmolLM2-135M-Instruct
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

checkpoint-2000/adapter_config.json

@@ -0,0 +1,43 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "HuggingFaceTB/SmolLM2-135M-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "k_proj",
+    "q_proj",
+    "v_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-2000/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5eefccd5f12a1c3dd9a421aefffc3b03d29ca7dde1c282e81d29b1d63dbd5d3
+size 7404368

checkpoint-2000/chat_template.jinja

@@ -0,0 +1,6 @@
+{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system
+You are a helpful AI assistant named SmolLM, trained by Hugging Face<|im_end|>
+' }}{% endif %}{{'<|im_start|>' + message['role'] + '
+' + message['content'] + '<|im_end|>' + '
+'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
+' }}{% endif %}

checkpoint-2000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7f2a40c452403b39aa69d0053c8711ea9445d69827debf5e6ff85a59d0ead36
+size 14950667

checkpoint-2000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c69f08e7e7238f8f16c24ef86292d12caa0a4ae096db194a9f2f78781144bdb
+size 14645

checkpoint-2000/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4aa03f6e0cd07cf67ce1fbe3101d545f5771ef9148b9debf02b11cf6948da5c
+size 1383

checkpoint-2000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cfbe1f4fd21e5230551934a2e47447c4edbeafac827bb210ae27474717167e0b
+size 1465

checkpoint-2000/tokenizer_config.json

@@ -0,0 +1,17 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|im_start|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "is_local": false,
+  "model_max_length": 8192,
+  "pad_token": "<|im_end|>",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "<|endoftext|>",
+  "vocab_size": 49152
+}

checkpoint-2000/trainer_state.json

@@ -0,0 +1,2034 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.718213058419244,
+  "eval_steps": 500,
+  "global_step": 2000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.1566081464290618,
+      "epoch": 0.00859106529209622,
+      "grad_norm": 0.15214066207408905,
+      "learning_rate": 0.00029884020618556697,
+      "loss": 1.226055908203125,
+      "mean_token_accuracy": 0.7206311166286469,
+      "num_tokens": 26541.0,
+      "step": 10
+    },
+    {
+      "entropy": 1.2002840995788575,
+      "epoch": 0.01718213058419244,
+      "grad_norm": 0.1833968460559845,
+      "learning_rate": 0.00029755154639175255,
+      "loss": 1.2007878303527832,
+      "mean_token_accuracy": 0.7304173231124877,
+      "num_tokens": 52280.0,
+      "step": 20
+    },
+    {
+      "entropy": 1.083837980031967,
+      "epoch": 0.02577319587628866,
+      "grad_norm": 0.15183115005493164,
+      "learning_rate": 0.0002962628865979381,
+      "loss": 1.1582960128784179,
+      "mean_token_accuracy": 0.7341346919536591,
+      "num_tokens": 79603.0,
+      "step": 30
+    },
+    {
+      "entropy": 0.9830838143825531,
+      "epoch": 0.03436426116838488,
+      "grad_norm": 0.1640024185180664,
+      "learning_rate": 0.00029497422680412364,
+      "loss": 0.989743423461914,
+      "mean_token_accuracy": 0.7652481079101563,
+      "num_tokens": 106301.0,
+      "step": 40
+    },
+    {
+      "entropy": 0.9237875938415527,
+      "epoch": 0.0429553264604811,
+      "grad_norm": 0.1496010422706604,
+      "learning_rate": 0.0002936855670103092,
+      "loss": 0.9489053726196289,
+      "mean_token_accuracy": 0.7682253539562225,
+      "num_tokens": 131791.0,
+      "step": 50
+    },
+    {
+      "entropy": 0.945674192905426,
+      "epoch": 0.05154639175257732,
+      "grad_norm": 0.16326577961444855,
+      "learning_rate": 0.0002923969072164948,
+      "loss": 0.9549709320068359,
+      "mean_token_accuracy": 0.7691607773303986,
+      "num_tokens": 156376.0,
+      "step": 60
+    },
+    {
+      "entropy": 0.9946551442146301,
+      "epoch": 0.06013745704467354,
+      "grad_norm": 0.14451684057712555,
+      "learning_rate": 0.00029110824742268037,
+      "loss": 0.9979846954345704,
+      "mean_token_accuracy": 0.7634965121746063,
+      "num_tokens": 183753.0,
+      "step": 70
+    },
+    {
+      "entropy": 0.9941512286663056,
+      "epoch": 0.06872852233676977,
+      "grad_norm": 0.15088772773742676,
+      "learning_rate": 0.00028981958762886595,
+      "loss": 0.9917948722839356,
+      "mean_token_accuracy": 0.7609466493129731,
+      "num_tokens": 209154.0,
+      "step": 80
+    },
+    {
+      "entropy": 0.899951022863388,
+      "epoch": 0.07731958762886598,
+      "grad_norm": 0.1635286509990692,
+      "learning_rate": 0.0002885309278350515,
+      "loss": 0.9084607124328613,
+      "mean_token_accuracy": 0.773688817024231,
+      "num_tokens": 232285.0,
+      "step": 90
+    },
+    {
+      "entropy": 0.9840596318244934,
+      "epoch": 0.0859106529209622,
+      "grad_norm": 0.1687513142824173,
+      "learning_rate": 0.0002872422680412371,
+      "loss": 1.005988883972168,
+      "mean_token_accuracy": 0.7565966248512268,
+      "num_tokens": 258165.0,
+      "step": 100
+    },
+    {
+      "entropy": 0.9256654202938079,
+      "epoch": 0.09450171821305842,
+      "grad_norm": 0.15977086126804352,
+      "learning_rate": 0.0002859536082474227,
+      "loss": 0.8968193054199218,
+      "mean_token_accuracy": 0.7728870630264282,
+      "num_tokens": 283613.0,
+      "step": 110
+    },
+    {
+      "entropy": 0.9430408120155335,
+      "epoch": 0.10309278350515463,
+      "grad_norm": 0.14840301871299744,
+      "learning_rate": 0.0002846649484536082,
+      "loss": 0.9234983444213867,
+      "mean_token_accuracy": 0.7652122378349304,
+      "num_tokens": 308257.0,
+      "step": 120
+    },
+    {
+      "entropy": 0.9537142395973206,
+      "epoch": 0.11168384879725086,
+      "grad_norm": 0.1754530817270279,
+      "learning_rate": 0.0002833762886597938,
+      "loss": 0.949033260345459,
+      "mean_token_accuracy": 0.7616437077522278,
+      "num_tokens": 333700.0,
+      "step": 130
+    },
+    {
+      "entropy": 0.9385378897190094,
+      "epoch": 0.12027491408934708,
+      "grad_norm": 0.16817958652973175,
+      "learning_rate": 0.00028208762886597935,
+      "loss": 0.9222366333007812,
+      "mean_token_accuracy": 0.7721138775348664,
+      "num_tokens": 359746.0,
+      "step": 140
+    },
+    {
+      "entropy": 0.8476657152175904,
+      "epoch": 0.12886597938144329,
+      "grad_norm": 0.15769453346729279,
+      "learning_rate": 0.0002807989690721649,
+      "loss": 0.8225071907043457,
+      "mean_token_accuracy": 0.7880932271480561,
+      "num_tokens": 383468.0,
+      "step": 150
+    },
+    {
+      "entropy": 0.8862757325172425,
+      "epoch": 0.13745704467353953,
+      "grad_norm": 0.17620904743671417,
+      "learning_rate": 0.0002795103092783505,
+      "loss": 0.8871653556823731,
+      "mean_token_accuracy": 0.7726393282413483,
+      "num_tokens": 409660.0,
+      "step": 160
+    },
+    {
+      "entropy": 0.9064472615718842,
+      "epoch": 0.14604810996563575,
+      "grad_norm": 0.15342549979686737,
+      "learning_rate": 0.0002782216494845361,
+      "loss": 0.9092922210693359,
+      "mean_token_accuracy": 0.7713103413581848,
+      "num_tokens": 437071.0,
+      "step": 170
+    },
+    {
+      "entropy": 0.9095455348491669,
+      "epoch": 0.15463917525773196,
+      "grad_norm": 0.17525391280651093,
+      "learning_rate": 0.00027693298969072165,
+      "loss": 0.8956180572509765,
+      "mean_token_accuracy": 0.7772489190101624,
+      "num_tokens": 460852.0,
+      "step": 180
+    },
+    {
+      "entropy": 0.8514094173908233,
+      "epoch": 0.16323024054982818,
+      "grad_norm": 0.15829823911190033,
+      "learning_rate": 0.0002756443298969072,
+      "loss": 0.8513413429260254,
+      "mean_token_accuracy": 0.779003256559372,
+      "num_tokens": 486114.0,
+      "step": 190
+    },
+    {
+      "entropy": 0.8995061337947845,
+      "epoch": 0.1718213058419244,
+      "grad_norm": 0.15622206032276154,
+      "learning_rate": 0.00027435567010309275,
+      "loss": 0.898340892791748,
+      "mean_token_accuracy": 0.7708433449268342,
+      "num_tokens": 512397.0,
+      "step": 200
+    },
+    {
+      "entropy": 0.8732269763946533,
+      "epoch": 0.18041237113402062,
+      "grad_norm": 0.17759552597999573,
+      "learning_rate": 0.0002730670103092783,
+      "loss": 0.8695596694946289,
+      "mean_token_accuracy": 0.7793173313140869,
+      "num_tokens": 536554.0,
+      "step": 210
+    },
+    {
+      "entropy": 0.9366106212139129,
+      "epoch": 0.18900343642611683,
+      "grad_norm": 0.1767597794532776,
+      "learning_rate": 0.0002717783505154639,
+      "loss": 0.9249121665954589,
+      "mean_token_accuracy": 0.7724991559982299,
+      "num_tokens": 561637.0,
+      "step": 220
+    },
+    {
+      "entropy": 0.8857321858406066,
+      "epoch": 0.19759450171821305,
+      "grad_norm": 0.20788735151290894,
+      "learning_rate": 0.0002704896907216495,
+      "loss": 0.8790401458740235,
+      "mean_token_accuracy": 0.7778670430183411,
+      "num_tokens": 586614.0,
+      "step": 230
+    },
+    {
+      "entropy": 0.9273143649101258,
+      "epoch": 0.20618556701030927,
+      "grad_norm": 0.16100187599658966,
+      "learning_rate": 0.00026920103092783505,
+      "loss": 0.9232232093811035,
+      "mean_token_accuracy": 0.7677364766597747,
+      "num_tokens": 612428.0,
+      "step": 240
+    },
+    {
+      "entropy": 0.8786617696285248,
+      "epoch": 0.21477663230240548,
+      "grad_norm": 0.1608886867761612,
+      "learning_rate": 0.00026791237113402063,
+      "loss": 0.8883329391479492,
+      "mean_token_accuracy": 0.7775621056556702,
+      "num_tokens": 638564.0,
+      "step": 250
+    },
+    {
+      "entropy": 0.9227964103221893,
+      "epoch": 0.22336769759450173,
+      "grad_norm": 0.1510680615901947,
+      "learning_rate": 0.00026662371134020615,
+      "loss": 0.9296038627624512,
+      "mean_token_accuracy": 0.7714014649391174,
+      "num_tokens": 665263.0,
+      "step": 260
+    },
+    {
+      "entropy": 0.9126189887523651,
+      "epoch": 0.23195876288659795,
+      "grad_norm": 0.16434459388256073,
+      "learning_rate": 0.00026533505154639173,
+      "loss": 0.9191689491271973,
+      "mean_token_accuracy": 0.7705935597419739,
+      "num_tokens": 691956.0,
+      "step": 270
+    },
+    {
+      "entropy": 0.8797955513000488,
+      "epoch": 0.24054982817869416,
+      "grad_norm": 0.1714707911014557,
+      "learning_rate": 0.0002640463917525773,
+      "loss": 0.8856119155883789,
+      "mean_token_accuracy": 0.7766761302947998,
+      "num_tokens": 716820.0,
+      "step": 280
+    },
+    {
+      "entropy": 0.877093505859375,
+      "epoch": 0.24914089347079038,
+      "grad_norm": 0.15326625108718872,
+      "learning_rate": 0.0002627577319587629,
+      "loss": 0.8566678047180176,
+      "mean_token_accuracy": 0.7812119901180268,
+      "num_tokens": 742221.0,
+      "step": 290
+    },
+    {
+      "entropy": 0.87042076587677,
+      "epoch": 0.25773195876288657,
+      "grad_norm": 0.1710847020149231,
+      "learning_rate": 0.00026146907216494846,
+      "loss": 0.8829298973083496,
+      "mean_token_accuracy": 0.7770332932472229,
+      "num_tokens": 765839.0,
+      "step": 300
+    },
+    {
+      "entropy": 0.8417458057403564,
+      "epoch": 0.2663230240549828,
+      "grad_norm": 0.1496821790933609,
+      "learning_rate": 0.00026018041237113403,
+      "loss": 0.8433545112609864,
+      "mean_token_accuracy": 0.7857938230037689,
+      "num_tokens": 791131.0,
+      "step": 310
+    },
+    {
+      "entropy": 0.9158999443054199,
+      "epoch": 0.27491408934707906,
+      "grad_norm": 0.17929035425186157,
+      "learning_rate": 0.00025889175257731955,
+      "loss": 0.9123490333557129,
+      "mean_token_accuracy": 0.775556480884552,
+      "num_tokens": 816984.0,
+      "step": 320
+    },
+    {
+      "entropy": 0.8314016401767731,
+      "epoch": 0.28350515463917525,
+      "grad_norm": 0.16051091253757477,
+      "learning_rate": 0.00025760309278350513,
+      "loss": 0.8279628753662109,
+      "mean_token_accuracy": 0.7862712442874908,
+      "num_tokens": 842220.0,
+      "step": 330
+    },
+    {
+      "entropy": 0.8677021145820618,
+      "epoch": 0.2920962199312715,
+      "grad_norm": 0.19157563149929047,
+      "learning_rate": 0.0002563144329896907,
+      "loss": 0.8552864074707032,
+      "mean_token_accuracy": 0.7835110187530517,
+      "num_tokens": 867868.0,
+      "step": 340
+    },
+    {
+      "entropy": 0.8919401407241822,
+      "epoch": 0.3006872852233677,
+      "grad_norm": 0.14258626103401184,
+      "learning_rate": 0.0002550257731958763,
+      "loss": 0.9019416809082031,
+      "mean_token_accuracy": 0.7760707855224609,
+      "num_tokens": 894676.0,
+      "step": 350
+    },
+    {
+      "entropy": 0.8415017545223236,
+      "epoch": 0.30927835051546393,
+      "grad_norm": 0.14830808341503143,
+      "learning_rate": 0.00025373711340206186,
+      "loss": 0.8072587013244629,
+      "mean_token_accuracy": 0.7912742376327515,
+      "num_tokens": 921632.0,
+      "step": 360
+    },
+    {
+      "entropy": 0.8371863842010498,
+      "epoch": 0.3178694158075601,
+      "grad_norm": 0.15538977086544037,
+      "learning_rate": 0.0002524484536082474,
+      "loss": 0.8308550834655761,
+      "mean_token_accuracy": 0.7849089920520782,
+      "num_tokens": 945955.0,
+      "step": 370
+    },
+    {
+      "entropy": 0.8338176369667053,
+      "epoch": 0.32646048109965636,
+      "grad_norm": 0.1711844801902771,
+      "learning_rate": 0.00025115979381443295,
+      "loss": 0.8479556083679199,
+      "mean_token_accuracy": 0.7850408136844635,
+      "num_tokens": 971508.0,
+      "step": 380
+    },
+    {
+      "entropy": 0.8876297354698182,
+      "epoch": 0.33505154639175255,
+      "grad_norm": 0.14902909100055695,
+      "learning_rate": 0.00024987113402061853,
+      "loss": 0.8821809768676758,
+      "mean_token_accuracy": 0.7764640510082245,
+      "num_tokens": 995124.0,
+      "step": 390
+    },
+    {
+      "entropy": 0.8212514638900756,
+      "epoch": 0.3436426116838488,
+      "grad_norm": 0.17950233817100525,
+      "learning_rate": 0.0002485824742268041,
+      "loss": 0.8255527496337891,
+      "mean_token_accuracy": 0.7870513379573822,
+      "num_tokens": 1019735.0,
+      "step": 400
+    },
+    {
+      "entropy": 0.9294387817382812,
+      "epoch": 0.35223367697594504,
+      "grad_norm": 0.15237703919410706,
+      "learning_rate": 0.0002472938144329897,
+      "loss": 0.9235431671142578,
+      "mean_token_accuracy": 0.7703655660152435,
+      "num_tokens": 1046295.0,
+      "step": 410
+    },
+    {
+      "entropy": 0.8429355442523956,
+      "epoch": 0.36082474226804123,
+      "grad_norm": 0.1589658558368683,
+      "learning_rate": 0.0002460051546391752,
+      "loss": 0.8329957008361817,
+      "mean_token_accuracy": 0.7897172749042511,
+      "num_tokens": 1072020.0,
+      "step": 420
+    },
+    {
+      "entropy": 0.8407810032367706,
+      "epoch": 0.3694158075601375,
+      "grad_norm": 0.17521116137504578,
+      "learning_rate": 0.0002447164948453608,
+      "loss": 0.8535200119018554,
+      "mean_token_accuracy": 0.7809976935386658,
+      "num_tokens": 1098420.0,
+      "step": 430
+    },
+    {
+      "entropy": 0.8515082120895385,
+      "epoch": 0.37800687285223367,
+      "grad_norm": 0.15218117833137512,
+      "learning_rate": 0.00024342783505154638,
+      "loss": 0.8459153175354004,
+      "mean_token_accuracy": 0.7833445191383361,
+      "num_tokens": 1124235.0,
+      "step": 440
+    },
+    {
+      "entropy": 0.8506015419960022,
+      "epoch": 0.3865979381443299,
+      "grad_norm": 0.1809423863887787,
+      "learning_rate": 0.00024213917525773193,
+      "loss": 0.8384982109069824,
+      "mean_token_accuracy": 0.7841372966766358,
+      "num_tokens": 1149636.0,
+      "step": 450
+    },
+    {
+      "entropy": 0.9110018074512481,
+      "epoch": 0.3951890034364261,
+      "grad_norm": 0.15085963904857635,
+      "learning_rate": 0.0002408505154639175,
+      "loss": 0.9191832542419434,
+      "mean_token_accuracy": 0.7757422685623169,
+      "num_tokens": 1173711.0,
+      "step": 460
+    },
+    {
+      "entropy": 0.9000201225280762,
+      "epoch": 0.40378006872852235,
+      "grad_norm": 0.16748838126659393,
+      "learning_rate": 0.00023956185567010308,
+      "loss": 0.9107230186462403,
+      "mean_token_accuracy": 0.7768173336982727,
+      "num_tokens": 1199340.0,
+      "step": 470
+    },
+    {
+      "entropy": 0.8751251935958863,
+      "epoch": 0.41237113402061853,
+      "grad_norm": 0.1593722254037857,
+      "learning_rate": 0.00023827319587628863,
+      "loss": 0.8804635047912598,
+      "mean_token_accuracy": 0.7814023733139038,
+      "num_tokens": 1226032.0,
+      "step": 480
+    },
+    {
+      "entropy": 0.8843371510505676,
+      "epoch": 0.4209621993127148,
+      "grad_norm": 0.12278851121664047,
+      "learning_rate": 0.0002369845360824742,
+      "loss": 0.8643080711364746,
+      "mean_token_accuracy": 0.7809630572795868,
+      "num_tokens": 1252769.0,
+      "step": 490
+    },
+    {
+      "entropy": 0.8834336459636688,
+      "epoch": 0.42955326460481097,
+      "grad_norm": 0.1367531716823578,
+      "learning_rate": 0.00023569587628865976,
+      "loss": 0.8869522094726563,
+      "mean_token_accuracy": 0.7775909185409546,
+      "num_tokens": 1278979.0,
+      "step": 500
+    },
+    {
+      "entropy": 0.9062523484230042,
+      "epoch": 0.4381443298969072,
+      "grad_norm": 0.14829133450984955,
+      "learning_rate": 0.00023440721649484533,
+      "loss": 0.9153310775756835,
+      "mean_token_accuracy": 0.7755910754203796,
+      "num_tokens": 1305141.0,
+      "step": 510
+    },
+    {
+      "entropy": 0.8907318472862243,
+      "epoch": 0.44673539518900346,
+      "grad_norm": 0.19640029966831207,
+      "learning_rate": 0.0002331185567010309,
+      "loss": 0.8736312866210938,
+      "mean_token_accuracy": 0.7823721885681152,
+      "num_tokens": 1333535.0,
+      "step": 520
+    },
+    {
+      "entropy": 0.9001505076885223,
+      "epoch": 0.45532646048109965,
+      "grad_norm": 0.1615416407585144,
+      "learning_rate": 0.00023182989690721648,
+      "loss": 0.9240070343017578,
+      "mean_token_accuracy": 0.7720246851444245,
+      "num_tokens": 1359100.0,
+      "step": 530
+    },
+    {
+      "entropy": 0.8964834988117218,
+      "epoch": 0.4639175257731959,
+      "grad_norm": 0.16148629784584045,
+      "learning_rate": 0.00023054123711340206,
+      "loss": 0.9040334701538086,
+      "mean_token_accuracy": 0.7697900414466858,
+      "num_tokens": 1386642.0,
+      "step": 540
+    },
+    {
+      "entropy": 0.9152944028377533,
+      "epoch": 0.4725085910652921,
+      "grad_norm": 0.16060467064380646,
+      "learning_rate": 0.00022925257731958758,
+      "loss": 0.9094497680664062,
+      "mean_token_accuracy": 0.771862530708313,
+      "num_tokens": 1412712.0,
+      "step": 550
+    },
+    {
+      "entropy": 0.8443439364433288,
+      "epoch": 0.48109965635738833,
+      "grad_norm": 0.16139253973960876,
+      "learning_rate": 0.00022796391752577316,
+      "loss": 0.8445466041564942,
+      "mean_token_accuracy": 0.7862408220767975,
+      "num_tokens": 1439208.0,
+      "step": 560
+    },
+    {
+      "entropy": 0.8669894218444825,
+      "epoch": 0.4896907216494845,
+      "grad_norm": 0.13149064779281616,
+      "learning_rate": 0.00022667525773195873,
+      "loss": 0.8791823387145996,
+      "mean_token_accuracy": 0.7771342694759369,
+      "num_tokens": 1466602.0,
+      "step": 570
+    },
+    {
+      "entropy": 0.8713687837123871,
+      "epoch": 0.49828178694158076,
+      "grad_norm": 0.15979787707328796,
+      "learning_rate": 0.0002253865979381443,
+      "loss": 0.8594341278076172,
+      "mean_token_accuracy": 0.7810667157173157,
+      "num_tokens": 1493048.0,
+      "step": 580
+    },
+    {
+      "entropy": 0.9155345737934113,
+      "epoch": 0.506872852233677,
+      "grad_norm": 0.18420852720737457,
+      "learning_rate": 0.00022409793814432989,
+      "loss": 0.915378475189209,
+      "mean_token_accuracy": 0.7724271833896637,
+      "num_tokens": 1517512.0,
+      "step": 590
+    },
+    {
+      "entropy": 0.891716331243515,
+      "epoch": 0.5154639175257731,
+      "grad_norm": 0.1665642410516739,
+      "learning_rate": 0.00022280927835051546,
+      "loss": 0.8950610160827637,
+      "mean_token_accuracy": 0.777475905418396,
+      "num_tokens": 1542765.0,
+      "step": 600
+    },
+    {
+      "entropy": 0.9547060489654541,
+      "epoch": 0.5240549828178694,
+      "grad_norm": 0.19496814906597137,
+      "learning_rate": 0.000221520618556701,
+      "loss": 0.9572202682495117,
+      "mean_token_accuracy": 0.7599396347999573,
+      "num_tokens": 1570704.0,
+      "step": 610
+    },
+    {
+      "entropy": 0.894726449251175,
+      "epoch": 0.5326460481099656,
+      "grad_norm": 0.162724569439888,
+      "learning_rate": 0.00022023195876288656,
+      "loss": 0.8876501083374023,
+      "mean_token_accuracy": 0.7815112411975861,
+      "num_tokens": 1594675.0,
+      "step": 620
+    },
+    {
+      "entropy": 0.8966928541660308,
+      "epoch": 0.5412371134020618,
+      "grad_norm": 0.1820981204509735,
+      "learning_rate": 0.00021894329896907214,
+      "loss": 0.8992101669311523,
+      "mean_token_accuracy": 0.7774552226066589,
+      "num_tokens": 1621381.0,
+      "step": 630
+    },
+    {
+      "entropy": 0.8320938229560852,
+      "epoch": 0.5498281786941581,
+      "grad_norm": 0.18641315400600433,
+      "learning_rate": 0.0002176546391752577,
+      "loss": 0.8333114624023438,
+      "mean_token_accuracy": 0.7836415946483613,
+      "num_tokens": 1646589.0,
+      "step": 640
+    },
+    {
+      "entropy": 0.9350853085517883,
+      "epoch": 0.5584192439862543,
+      "grad_norm": 0.15154911577701569,
+      "learning_rate": 0.0002163659793814433,
+      "loss": 0.9300890922546386,
+      "mean_token_accuracy": 0.7711542308330536,
+      "num_tokens": 1672281.0,
+      "step": 650
+    },
+    {
+      "entropy": 0.8372068881988526,
+      "epoch": 0.5670103092783505,
+      "grad_norm": 0.157988041639328,
+      "learning_rate": 0.00021507731958762884,
+      "loss": 0.8490092277526855,
+      "mean_token_accuracy": 0.7828391373157502,
+      "num_tokens": 1697203.0,
+      "step": 660
+    },
+    {
+      "entropy": 0.8175156831741333,
+      "epoch": 0.5756013745704467,
+      "grad_norm": 0.16615159809589386,
+      "learning_rate": 0.0002137886597938144,
+      "loss": 0.7962069988250733,
+      "mean_token_accuracy": 0.793628740310669,
+      "num_tokens": 1722977.0,
+      "step": 670
+    },
+    {
+      "entropy": 0.9293979525566101,
+      "epoch": 0.584192439862543,
+      "grad_norm": 0.13969369232654572,
+      "learning_rate": 0.0002125,
+      "loss": 0.9364766120910645,
+      "mean_token_accuracy": 0.7693193078041076,
+      "num_tokens": 1748869.0,
+      "step": 680
+    },
+    {
+      "entropy": 0.8958774268627167,
+      "epoch": 0.5927835051546392,
+      "grad_norm": 0.14425267279148102,
+      "learning_rate": 0.00021121134020618556,
+      "loss": 0.8842126846313476,
+      "mean_token_accuracy": 0.7759437382221221,
+      "num_tokens": 1774715.0,
+      "step": 690
+    },
+    {
+      "entropy": 0.929528433084488,
+      "epoch": 0.6013745704467354,
+      "grad_norm": 0.15875831246376038,
+      "learning_rate": 0.0002099226804123711,
+      "loss": 0.9353519439697265,
+      "mean_token_accuracy": 0.769885140657425,
+      "num_tokens": 1800430.0,
+      "step": 700
+    },
+    {
+      "entropy": 0.8651838064193725,
+      "epoch": 0.6099656357388317,
+      "grad_norm": 0.19464287161827087,
+      "learning_rate": 0.00020863402061855666,
+      "loss": 0.8678315162658692,
+      "mean_token_accuracy": 0.7783544957637787,
+      "num_tokens": 1824631.0,
+      "step": 710
+    },
+    {
+      "entropy": 0.874721884727478,
+      "epoch": 0.6185567010309279,
+      "grad_norm": 0.17015859484672546,
+      "learning_rate": 0.00020734536082474224,
+      "loss": 0.847961139678955,
+      "mean_token_accuracy": 0.7813271641731262,
+      "num_tokens": 1849593.0,
+      "step": 720
+    },
+    {
+      "entropy": 0.8706691145896912,
+      "epoch": 0.627147766323024,
+      "grad_norm": 0.15960460901260376,
+      "learning_rate": 0.00020605670103092781,
+      "loss": 0.8743976593017578,
+      "mean_token_accuracy": 0.7772314369678497,
+      "num_tokens": 1874593.0,
+      "step": 730
+    },
+    {
+      "entropy": 0.824019831418991,
+      "epoch": 0.6357388316151202,
+      "grad_norm": 0.16192808747291565,
+      "learning_rate": 0.0002047680412371134,
+      "loss": 0.8214377403259278,
+      "mean_token_accuracy": 0.7877136409282685,
+      "num_tokens": 1899592.0,
+      "step": 740
+    },
+    {
+      "entropy": 0.853943508863449,
+      "epoch": 0.6443298969072165,
+      "grad_norm": 0.14331476390361786,
+      "learning_rate": 0.00020347938144329897,
+      "loss": 0.8542372703552246,
+      "mean_token_accuracy": 0.7825242638587951,
+      "num_tokens": 1925105.0,
+      "step": 750
+    },
+    {
+      "entropy": 0.9206405401229858,
+      "epoch": 0.6529209621993127,
+      "grad_norm": 0.16269327700138092,
+      "learning_rate": 0.00020219072164948454,
+      "loss": 0.9131902694702149,
+      "mean_token_accuracy": 0.7722071945667267,
+      "num_tokens": 1952126.0,
+      "step": 760
+    },
+    {
+      "entropy": 0.8438917219638824,
+      "epoch": 0.6615120274914089,
+      "grad_norm": 0.18783624470233917,
+      "learning_rate": 0.00020090206185567006,
+      "loss": 0.8612903594970703,
+      "mean_token_accuracy": 0.7805359125137329,
+      "num_tokens": 1976727.0,
+      "step": 770
+    },
+    {
+      "entropy": 0.8782703518867493,
+      "epoch": 0.6701030927835051,
+      "grad_norm": 0.1749051958322525,
+      "learning_rate": 0.00019961340206185564,
+      "loss": 0.8877654075622559,
+      "mean_token_accuracy": 0.775201940536499,
+      "num_tokens": 2004204.0,
+      "step": 780
+    },
+    {
+      "entropy": 0.9036802053451538,
+      "epoch": 0.6786941580756014,
+      "grad_norm": 0.17728140950202942,
+      "learning_rate": 0.00019832474226804121,
+      "loss": 0.8946940422058105,
+      "mean_token_accuracy": 0.7781059086322785,
+      "num_tokens": 2031926.0,
+      "step": 790
+    },
+    {
+      "entropy": 0.9057375133037567,
+      "epoch": 0.6872852233676976,
+      "grad_norm": 0.1804327517747879,
+      "learning_rate": 0.0001970360824742268,
+      "loss": 0.9067662239074707,
+      "mean_token_accuracy": 0.7786577641963959,
+      "num_tokens": 2059628.0,
+      "step": 800
+    },
+    {
+      "entropy": 0.8421910405158997,
+      "epoch": 0.6958762886597938,
+      "grad_norm": 0.1570490598678589,
+      "learning_rate": 0.00019574742268041237,
+      "loss": 0.8346380233764649,
+      "mean_token_accuracy": 0.7839872419834137,
+      "num_tokens": 2083943.0,
+      "step": 810
+    },
+    {
+      "entropy": 0.8743102788925171,
+      "epoch": 0.7044673539518901,
+      "grad_norm": 0.1737687736749649,
+      "learning_rate": 0.00019445876288659794,
+      "loss": 0.8622690200805664,
+      "mean_token_accuracy": 0.7791435241699218,
+      "num_tokens": 2109444.0,
+      "step": 820
+    },
+    {
+      "entropy": 0.8926449298858643,
+      "epoch": 0.7130584192439863,
+      "grad_norm": 0.1972922682762146,
+      "learning_rate": 0.0001931701030927835,
+      "loss": 0.895075511932373,
+      "mean_token_accuracy": 0.7716107368469238,
+      "num_tokens": 2134209.0,
+      "step": 830
+    },
+    {
+      "entropy": 0.8712981760501861,
+      "epoch": 0.7216494845360825,
+      "grad_norm": 0.16580969095230103,
+      "learning_rate": 0.00019188144329896904,
+      "loss": 0.8615394592285156,
+      "mean_token_accuracy": 0.780440878868103,
+      "num_tokens": 2160322.0,
+      "step": 840
+    },
+    {
+      "entropy": 0.7872600853443146,
+      "epoch": 0.7302405498281787,
+      "grad_norm": 0.17089906334877014,
+      "learning_rate": 0.00019059278350515462,
+      "loss": 0.7738356590270996,
+      "mean_token_accuracy": 0.7984008610248565,
+      "num_tokens": 2186673.0,
+      "step": 850
+    },
+    {
+      "entropy": 0.8024394273757934,
+      "epoch": 0.738831615120275,
+      "grad_norm": 0.15015770494937897,
+      "learning_rate": 0.0001893041237113402,
+      "loss": 0.801088809967041,
+      "mean_token_accuracy": 0.7919513285160065,
+      "num_tokens": 2211288.0,
+      "step": 860
+    },
+    {
+      "entropy": 0.8289069414138794,
+      "epoch": 0.7474226804123711,
+      "grad_norm": 0.15804092586040497,
+      "learning_rate": 0.00018801546391752577,
+      "loss": 0.8228999137878418,
+      "mean_token_accuracy": 0.7900383174419403,
+      "num_tokens": 2235494.0,
+      "step": 870
+    },
+    {
+      "entropy": 0.8337613046169281,
+      "epoch": 0.7560137457044673,
+      "grad_norm": 0.16564202308654785,
+      "learning_rate": 0.00018672680412371132,
+      "loss": 0.8357515335083008,
+      "mean_token_accuracy": 0.7796721041202546,
+      "num_tokens": 2261853.0,
+      "step": 880
+    },
+    {
+      "entropy": 0.8514188408851624,
+      "epoch": 0.7646048109965635,
+      "grad_norm": 0.15763898193836212,
+      "learning_rate": 0.0001854381443298969,
+      "loss": 0.8596377372741699,
+      "mean_token_accuracy": 0.781908392906189,
+      "num_tokens": 2288235.0,
+      "step": 890
+    },
+    {
+      "entropy": 0.8182059168815613,
+      "epoch": 0.7731958762886598,
+      "grad_norm": 0.14843082427978516,
+      "learning_rate": 0.00018414948453608247,
+      "loss": 0.8056727409362793,
+      "mean_token_accuracy": 0.7887085616588593,
+      "num_tokens": 2312636.0,
+      "step": 900
+    },
+    {
+      "entropy": 0.8019815027713776,
+      "epoch": 0.781786941580756,
+      "grad_norm": 0.1444498896598816,
+      "learning_rate": 0.00018286082474226804,
+      "loss": 0.7791801452636719,
+      "mean_token_accuracy": 0.7921311616897583,
+      "num_tokens": 2337590.0,
+      "step": 910
+    },
+    {
+      "entropy": 0.8811851978302002,
+      "epoch": 0.7903780068728522,
+      "grad_norm": 0.20295798778533936,
+      "learning_rate": 0.0001815721649484536,
+      "loss": 0.9093887329101562,
+      "mean_token_accuracy": 0.7765615165233613,
+      "num_tokens": 2364733.0,
+      "step": 920
+    },
+    {
+      "entropy": 0.8845545649528503,
+      "epoch": 0.7989690721649485,
+      "grad_norm": 0.16211840510368347,
+      "learning_rate": 0.00018028350515463914,
+      "loss": 0.87130765914917,
+      "mean_token_accuracy": 0.773591297864914,
+      "num_tokens": 2389138.0,
+      "step": 930
+    },
+    {
+      "entropy": 0.8366316437721253,
+      "epoch": 0.8075601374570447,
+      "grad_norm": 0.1796175092458725,
+      "learning_rate": 0.00017899484536082472,
+      "loss": 0.8393503189086914,
+      "mean_token_accuracy": 0.7834821224212647,
+      "num_tokens": 2416288.0,
+      "step": 940
+    },
+    {
+      "entropy": 0.8687749207019806,
+      "epoch": 0.8161512027491409,
+      "grad_norm": 0.17602433264255524,
+      "learning_rate": 0.0001777061855670103,
+      "loss": 0.8831927299499511,
+      "mean_token_accuracy": 0.7806714773178101,
+      "num_tokens": 2442282.0,
+      "step": 950
+    },
+    {
+      "entropy": 0.8602447748184204,
+      "epoch": 0.8247422680412371,
+      "grad_norm": 0.14779216051101685,
+      "learning_rate": 0.00017641752577319587,
+      "loss": 0.8557798385620117,
+      "mean_token_accuracy": 0.7800233542919159,
+      "num_tokens": 2469865.0,
+      "step": 960
+    },
+    {
+      "entropy": 0.8987692832946778,
+      "epoch": 0.8333333333333334,
+      "grad_norm": 0.16065306961536407,
+      "learning_rate": 0.00017512886597938145,
+      "loss": 0.8930519104003907,
+      "mean_token_accuracy": 0.7771966874599456,
+      "num_tokens": 2497716.0,
+      "step": 970
+    },
+    {
+      "entropy": 0.847240948677063,
+      "epoch": 0.8419243986254296,
+      "grad_norm": 0.16063280403614044,
+      "learning_rate": 0.00017384020618556702,
+      "loss": 0.8212727546691895,
+      "mean_token_accuracy": 0.7900629937648773,
+      "num_tokens": 2523292.0,
+      "step": 980
+    },
+    {
+      "entropy": 0.8685175359249115,
+      "epoch": 0.8505154639175257,
+      "grad_norm": 0.19293011724948883,
+      "learning_rate": 0.00017255154639175254,
+      "loss": 0.8839738845825196,
+      "mean_token_accuracy": 0.7780547857284545,
+      "num_tokens": 2547529.0,
+      "step": 990
+    },
+    {
+      "entropy": 0.8589172720909118,
+      "epoch": 0.8591065292096219,
+      "grad_norm": 0.13081640005111694,
+      "learning_rate": 0.00017126288659793812,
+      "loss": 0.8760468482971191,
+      "mean_token_accuracy": 0.7810062050819397,
+      "num_tokens": 2574943.0,
+      "step": 1000
+    },
+    {
+      "entropy": 0.8676146745681763,
+      "epoch": 0.8676975945017182,
+      "grad_norm": 0.18423762917518616,
+      "learning_rate": 0.0001699742268041237,
+      "loss": 0.8576597213745117,
+      "mean_token_accuracy": 0.7845786452293396,
+      "num_tokens": 2599770.0,
+      "step": 1010
+    },
+    {
+      "entropy": 0.8578267991542816,
+      "epoch": 0.8762886597938144,
+      "grad_norm": 0.15971733629703522,
+      "learning_rate": 0.00016868556701030927,
+      "loss": 0.8597639083862305,
+      "mean_token_accuracy": 0.7836935818195343,
+      "num_tokens": 2629050.0,
+      "step": 1020
+    },
+    {
+      "entropy": 0.78883256316185,
+      "epoch": 0.8848797250859106,
+      "grad_norm": 0.19334255158901215,
+      "learning_rate": 0.00016739690721649485,
+      "loss": 0.775438642501831,
+      "mean_token_accuracy": 0.7994126856327057,
+      "num_tokens": 2652418.0,
+      "step": 1030
+    },
+    {
+      "entropy": 0.827907818555832,
+      "epoch": 0.8934707903780069,
+      "grad_norm": 0.17899778485298157,
+      "learning_rate": 0.0001661082474226804,
+      "loss": 0.8155632019042969,
+      "mean_token_accuracy": 0.7867587625980377,
+      "num_tokens": 2677020.0,
+      "step": 1040
+    },
+    {
+      "entropy": 0.8488841831684113,
+      "epoch": 0.9020618556701031,
+      "grad_norm": 0.17594760656356812,
+      "learning_rate": 0.00016481958762886597,
+      "loss": 0.8443450927734375,
+      "mean_token_accuracy": 0.7823632657527924,
+      "num_tokens": 2700589.0,
+      "step": 1050
+    },
+    {
+      "entropy": 0.9077254295349121,
+      "epoch": 0.9106529209621993,
+      "grad_norm": 0.17107197642326355,
+      "learning_rate": 0.00016353092783505152,
+      "loss": 0.9043283462524414,
+      "mean_token_accuracy": 0.7685754477977753,
+      "num_tokens": 2727925.0,
+      "step": 1060
+    },
+    {
+      "entropy": 0.8681511282920837,
+      "epoch": 0.9192439862542955,
+      "grad_norm": 0.16261200606822968,
+      "learning_rate": 0.0001622422680412371,
+      "loss": 0.8636059761047363,
+      "mean_token_accuracy": 0.7801066517829895,
+      "num_tokens": 2754434.0,
+      "step": 1070
+    },
+    {
+      "entropy": 0.8201885402202607,
+      "epoch": 0.9278350515463918,
+      "grad_norm": 0.15494069457054138,
+      "learning_rate": 0.00016095360824742267,
+      "loss": 0.8267613410949707,
+      "mean_token_accuracy": 0.7909399390220642,
+      "num_tokens": 2778856.0,
+      "step": 1080
+    },
+    {
+      "entropy": 0.8281429171562195,
+      "epoch": 0.936426116838488,
+      "grad_norm": 0.15491366386413574,
+      "learning_rate": 0.00015966494845360822,
+      "loss": 0.8283931732177734,
+      "mean_token_accuracy": 0.7858192563056946,
+      "num_tokens": 2802467.0,
+      "step": 1090
+    },
+    {
+      "entropy": 0.8585192024707794,
+      "epoch": 0.9450171821305842,
+      "grad_norm": 0.16744765639305115,
+      "learning_rate": 0.0001583762886597938,
+      "loss": 0.8450031280517578,
+      "mean_token_accuracy": 0.7824375092983246,
+      "num_tokens": 2828404.0,
+      "step": 1100
+    },
+    {
+      "entropy": 0.9703075528144837,
+      "epoch": 0.9536082474226805,
+      "grad_norm": 0.16334781050682068,
+      "learning_rate": 0.00015708762886597937,
+      "loss": 0.9652547836303711,
+      "mean_token_accuracy": 0.7639409124851226,
+      "num_tokens": 2854621.0,
+      "step": 1110
+    },
+    {
+      "entropy": 0.8723820388317108,
+      "epoch": 0.9621993127147767,
+      "grad_norm": 0.15771399438381195,
+      "learning_rate": 0.00015579896907216495,
+      "loss": 0.8663213729858399,
+      "mean_token_accuracy": 0.7808757722377777,
+      "num_tokens": 2880448.0,
+      "step": 1120
+    },
+    {
+      "entropy": 0.8375514090061188,
+      "epoch": 0.9707903780068728,
+      "grad_norm": 0.16101613640785217,
+      "learning_rate": 0.0001545103092783505,
+      "loss": 0.8438360214233398,
+      "mean_token_accuracy": 0.7864978194236756,
+      "num_tokens": 2907346.0,
+      "step": 1130
+    },
+    {
+      "entropy": 0.836071914434433,
+      "epoch": 0.979381443298969,
+      "grad_norm": 0.19959381222724915,
+      "learning_rate": 0.00015322164948453605,
+      "loss": 0.8347750663757324,
+      "mean_token_accuracy": 0.7878862261772156,
+      "num_tokens": 2932769.0,
+      "step": 1140
+    },
+    {
+      "entropy": 0.8218921303749085,
+      "epoch": 0.9879725085910653,
+      "grad_norm": 0.17999783158302307,
+      "learning_rate": 0.00015193298969072162,
+      "loss": 0.81912841796875,
+      "mean_token_accuracy": 0.7904701411724091,
+      "num_tokens": 2956439.0,
+      "step": 1150
+    },
+    {
+      "entropy": 0.8836544632911683,
+      "epoch": 0.9965635738831615,
+      "grad_norm": 0.1764528602361679,
+      "learning_rate": 0.0001506443298969072,
+      "loss": 0.8860954284667969,
+      "mean_token_accuracy": 0.7801528871059418,
+      "num_tokens": 2983309.0,
+      "step": 1160
+    },
+    {
+      "entropy": 0.8893479645252228,
+      "epoch": 1.0051546391752577,
+      "grad_norm": 0.153923898935318,
+      "learning_rate": 0.00014935567010309277,
+      "loss": 0.8747451782226563,
+      "mean_token_accuracy": 0.7765235006809235,
+      "num_tokens": 3010175.0,
+      "step": 1170
+    },
+    {
+      "entropy": 0.8393925905227662,
+      "epoch": 1.013745704467354,
+      "grad_norm": 0.16801179945468903,
+      "learning_rate": 0.00014806701030927835,
+      "loss": 0.8316682815551758,
+      "mean_token_accuracy": 0.7851088464260101,
+      "num_tokens": 3036705.0,
+      "step": 1180
+    },
+    {
+      "entropy": 0.8521030902862549,
+      "epoch": 1.02233676975945,
+      "grad_norm": 0.16726894676685333,
+      "learning_rate": 0.0001467783505154639,
+      "loss": 0.8441553115844727,
+      "mean_token_accuracy": 0.7883111536502838,
+      "num_tokens": 3062184.0,
+      "step": 1190
+    },
+    {
+      "entropy": 0.8722131133079529,
+      "epoch": 1.0309278350515463,
+      "grad_norm": 0.1777733713388443,
+      "learning_rate": 0.00014548969072164948,
+      "loss": 0.8671235084533692,
+      "mean_token_accuracy": 0.7776070535182953,
+      "num_tokens": 3089854.0,
+      "step": 1200
+    },
+    {
+      "entropy": 0.8229423344135285,
+      "epoch": 1.0395189003436427,
+      "grad_norm": 0.13976716995239258,
+      "learning_rate": 0.00014420103092783505,
+      "loss": 0.8249235153198242,
+      "mean_token_accuracy": 0.7889478981494904,
+      "num_tokens": 3116938.0,
+      "step": 1210
+    },
+    {
+      "entropy": 0.8179661929607391,
+      "epoch": 1.0481099656357389,
+      "grad_norm": 0.16781210899353027,
+      "learning_rate": 0.0001429123711340206,
+      "loss": 0.8102549552917481,
+      "mean_token_accuracy": 0.7927488446235657,
+      "num_tokens": 3143825.0,
+      "step": 1220
+    },
+    {
+      "entropy": 0.7997653067111969,
+      "epoch": 1.056701030927835,
+      "grad_norm": 0.1647312045097351,
+      "learning_rate": 0.00014162371134020618,
+      "loss": 0.7900454998016357,
+      "mean_token_accuracy": 0.7910003781318664,
+      "num_tokens": 3168362.0,
+      "step": 1230
+    },
+    {
+      "entropy": 0.8539471685886383,
+      "epoch": 1.0652920962199313,
+      "grad_norm": 0.16146644949913025,
+      "learning_rate": 0.00014033505154639175,
+      "loss": 0.8489527702331543,
+      "mean_token_accuracy": 0.7828418910503387,
+      "num_tokens": 3192789.0,
+      "step": 1240
+    },
+    {
+      "entropy": 0.7987524449825287,
+      "epoch": 1.0738831615120275,
+      "grad_norm": 0.16997887194156647,
+      "learning_rate": 0.0001390463917525773,
+      "loss": 0.7815001964569092,
+      "mean_token_accuracy": 0.7958592414855957,
+      "num_tokens": 3218189.0,
+      "step": 1250
+    },
+    {
+      "entropy": 0.8433911919593811,
+      "epoch": 1.0824742268041236,
+      "grad_norm": 0.20293352007865906,
+      "learning_rate": 0.00013775773195876288,
+      "loss": 0.8536633491516114,
+      "mean_token_accuracy": 0.7847901046276092,
+      "num_tokens": 3242357.0,
+      "step": 1260
+    },
+    {
+      "entropy": 0.7989895343780518,
+      "epoch": 1.0910652920962198,
+      "grad_norm": 0.17577318847179413,
+      "learning_rate": 0.00013646907216494845,
+      "loss": 0.795603084564209,
+      "mean_token_accuracy": 0.7937948822975158,
+      "num_tokens": 3266634.0,
+      "step": 1270
+    },
+    {
+      "entropy": 0.849328339099884,
+      "epoch": 1.0996563573883162,
+      "grad_norm": 0.16344796121120453,
+      "learning_rate": 0.000135180412371134,
+      "loss": 0.8569498062133789,
+      "mean_token_accuracy": 0.784570574760437,
+      "num_tokens": 3292457.0,
+      "step": 1280
+    },
+    {
+      "entropy": 0.9115039288997651,
+      "epoch": 1.1082474226804124,
+      "grad_norm": 0.16120180487632751,
+      "learning_rate": 0.00013389175257731958,
+      "loss": 0.9172770500183105,
+      "mean_token_accuracy": 0.7738337874412536,
+      "num_tokens": 3319975.0,
+      "step": 1290
+    },
+    {
+      "entropy": 0.9116923689842225,
+      "epoch": 1.1168384879725086,
+      "grad_norm": 0.1507127285003662,
+      "learning_rate": 0.00013260309278350513,
+      "loss": 0.9115254402160644,
+      "mean_token_accuracy": 0.7727989673614502,
+      "num_tokens": 3349370.0,
+      "step": 1300
+    },
+    {
+      "entropy": 0.8705029428005219,
+      "epoch": 1.1254295532646048,
+      "grad_norm": 0.16798697412014008,
+      "learning_rate": 0.0001313144329896907,
+      "loss": 0.8673007011413574,
+      "mean_token_accuracy": 0.7829907596111297,
+      "num_tokens": 3376086.0,
+      "step": 1310
+    },
+    {
+      "entropy": 0.854372763633728,
+      "epoch": 1.134020618556701,
+      "grad_norm": 0.17355380952358246,
+      "learning_rate": 0.00013002577319587628,
+      "loss": 0.8472127914428711,
+      "mean_token_accuracy": 0.783273845911026,
+      "num_tokens": 3402983.0,
+      "step": 1320
+    },
+    {
+      "entropy": 0.8452303171157837,
+      "epoch": 1.1426116838487972,
+      "grad_norm": 0.17868086695671082,
+      "learning_rate": 0.00012873711340206183,
+      "loss": 0.846036434173584,
+      "mean_token_accuracy": 0.781141072511673,
+      "num_tokens": 3430354.0,
+      "step": 1330
+    },
+    {
+      "entropy": 0.8574681997299194,
+      "epoch": 1.1512027491408934,
+      "grad_norm": 0.17731839418411255,
+      "learning_rate": 0.0001274484536082474,
+      "loss": 0.8377547264099121,
+      "mean_token_accuracy": 0.7869145214557648,
+      "num_tokens": 3456008.0,
+      "step": 1340
+    },
+    {
+      "entropy": 0.8843120455741882,
+      "epoch": 1.1597938144329896,
+      "grad_norm": 0.17710870504379272,
+      "learning_rate": 0.00012615979381443298,
+      "loss": 0.894843578338623,
+      "mean_token_accuracy": 0.7717236757278443,
+      "num_tokens": 3481183.0,
+      "step": 1350
+    },
+    {
+      "entropy": 0.82523393034935,
+      "epoch": 1.168384879725086,
+      "grad_norm": 0.1979239135980606,
+      "learning_rate": 0.00012487113402061855,
+      "loss": 0.8107866287231446,
+      "mean_token_accuracy": 0.7911741256713867,
+      "num_tokens": 3508453.0,
+      "step": 1360
+    },
+    {
+      "entropy": 0.7964858949184418,
+      "epoch": 1.1769759450171822,
+      "grad_norm": 0.14254362881183624,
+      "learning_rate": 0.0001235824742268041,
+      "loss": 0.7863273620605469,
+      "mean_token_accuracy": 0.7948981881141662,
+      "num_tokens": 3534922.0,
+      "step": 1370
+    },
+    {
+      "entropy": 0.8244574010372162,
+      "epoch": 1.1855670103092784,
+      "grad_norm": 0.17570418119430542,
+      "learning_rate": 0.00012229381443298968,
+      "loss": 0.8179922103881836,
+      "mean_token_accuracy": 0.7896173536777497,
+      "num_tokens": 3559367.0,
+      "step": 1380
+    },
+    {
+      "entropy": 0.8134994566440582,
+      "epoch": 1.1941580756013745,
+      "grad_norm": 0.1930961012840271,
+      "learning_rate": 0.00012100515463917526,
+      "loss": 0.8178740501403808,
+      "mean_token_accuracy": 0.7915168821811676,
+      "num_tokens": 3585435.0,
+      "step": 1390
+    },
+    {
+      "entropy": 0.8384514689445496,
+      "epoch": 1.2027491408934707,
+      "grad_norm": 0.17272983491420746,
+      "learning_rate": 0.00011971649484536082,
+      "loss": 0.8454058647155762,
+      "mean_token_accuracy": 0.7868218302726746,
+      "num_tokens": 3613232.0,
+      "step": 1400
+    },
+    {
+      "entropy": 0.9280701756477356,
+      "epoch": 1.211340206185567,
+      "grad_norm": 0.1684003472328186,
+      "learning_rate": 0.00011842783505154638,
+      "loss": 0.9219778060913086,
+      "mean_token_accuracy": 0.7722154259681702,
+      "num_tokens": 3641457.0,
+      "step": 1410
+    },
+    {
+      "entropy": 0.832447475194931,
+      "epoch": 1.2199312714776633,
+      "grad_norm": 0.1773725152015686,
+      "learning_rate": 0.00011713917525773194,
+      "loss": 0.8198453903198242,
+      "mean_token_accuracy": 0.7914207100868225,
+      "num_tokens": 3666482.0,
+      "step": 1420
+    },
+    {
+      "entropy": 0.8738070428371429,
+      "epoch": 1.2285223367697595,
+      "grad_norm": 0.18506592512130737,
+      "learning_rate": 0.00011585051546391752,
+      "loss": 0.8723980903625488,
+      "mean_token_accuracy": 0.778689730167389,
+      "num_tokens": 3692349.0,
+      "step": 1430
+    },
+    {
+      "entropy": 0.8338236033916473,
+      "epoch": 1.2371134020618557,
+      "grad_norm": 0.17101170122623444,
+      "learning_rate": 0.00011456185567010308,
+      "loss": 0.837257194519043,
+      "mean_token_accuracy": 0.7857484579086303,
+      "num_tokens": 3717280.0,
+      "step": 1440
+    },
+    {
+      "entropy": 0.8157192528247833,
+      "epoch": 1.245704467353952,
+      "grad_norm": 0.15433582663536072,
+      "learning_rate": 0.00011327319587628864,
+      "loss": 0.8053474426269531,
+      "mean_token_accuracy": 0.7934693098068237,
+      "num_tokens": 3742595.0,
+      "step": 1450
+    },
+    {
+      "entropy": 0.8770197212696076,
+      "epoch": 1.254295532646048,
+      "grad_norm": 0.17446650564670563,
+      "learning_rate": 0.00011198453608247422,
+      "loss": 0.880764389038086,
+      "mean_token_accuracy": 0.7784990787506103,
+      "num_tokens": 3767913.0,
+      "step": 1460
+    },
+    {
+      "entropy": 0.8850863575935364,
+      "epoch": 1.2628865979381443,
+      "grad_norm": 0.186509370803833,
+      "learning_rate": 0.0001106958762886598,
+      "loss": 0.870394229888916,
+      "mean_token_accuracy": 0.7804449439048767,
+      "num_tokens": 3792206.0,
+      "step": 1470
+    },
+    {
+      "entropy": 0.8432760655879974,
+      "epoch": 1.2714776632302405,
+      "grad_norm": 0.17969489097595215,
+      "learning_rate": 0.00010940721649484534,
+      "loss": 0.8241963386535645,
+      "mean_token_accuracy": 0.7888677537441253,
+      "num_tokens": 3818423.0,
+      "step": 1480
+    },
+    {
+      "entropy": 0.8146856188774109,
+      "epoch": 1.2800687285223367,
+      "grad_norm": 0.15870600938796997,
+      "learning_rate": 0.00010811855670103092,
+      "loss": 0.8046764373779297,
+      "mean_token_accuracy": 0.7905527114868164,
+      "num_tokens": 3844217.0,
+      "step": 1490
+    },
+    {
+      "entropy": 0.8613445937633515,
+      "epoch": 1.2886597938144329,
+      "grad_norm": 0.16997215151786804,
+      "learning_rate": 0.00010682989690721648,
+      "loss": 0.8584735870361329,
+      "mean_token_accuracy": 0.7848546147346497,
+      "num_tokens": 3873420.0,
+      "step": 1500
+    },
+    {
+      "entropy": 0.8456706523895263,
+      "epoch": 1.2972508591065293,
+      "grad_norm": 0.17711228132247925,
+      "learning_rate": 0.00010554123711340206,
+      "loss": 0.8444679260253907,
+      "mean_token_accuracy": 0.7832227885723114,
+      "num_tokens": 3899136.0,
+      "step": 1510
+    },
+    {
+      "entropy": 0.8241195738315582,
+      "epoch": 1.3058419243986255,
+      "grad_norm": 0.1689673215150833,
+      "learning_rate": 0.00010425257731958762,
+      "loss": 0.8167056083679199,
+      "mean_token_accuracy": 0.7913166880607605,
+      "num_tokens": 3924700.0,
+      "step": 1520
+    },
+    {
+      "entropy": 0.7866684377193451,
+      "epoch": 1.3144329896907216,
+      "grad_norm": 0.2105344831943512,
+      "learning_rate": 0.00010296391752577318,
+      "loss": 0.7983303546905518,
+      "mean_token_accuracy": 0.7952485740184784,
+      "num_tokens": 3950430.0,
+      "step": 1530
+    },
+    {
+      "entropy": 0.8364840149879456,
+      "epoch": 1.3230240549828178,
+      "grad_norm": 0.16485024988651276,
+      "learning_rate": 0.00010167525773195876,
+      "loss": 0.8393770217895508,
+      "mean_token_accuracy": 0.7850684821605682,
+      "num_tokens": 3974488.0,
+      "step": 1540
+    },
+    {
+      "entropy": 0.8457872688770294,
+      "epoch": 1.331615120274914,
+      "grad_norm": 0.1678028106689453,
+      "learning_rate": 0.00010038659793814431,
+      "loss": 0.8469937324523926,
+      "mean_token_accuracy": 0.7852873146533966,
+      "num_tokens": 3999599.0,
+      "step": 1550
+    },
+    {
+      "entropy": 0.8001497447490692,
+      "epoch": 1.3402061855670104,
+      "grad_norm": 0.18752388656139374,
+      "learning_rate": 9.909793814432988e-05,
+      "loss": 0.802821159362793,
+      "mean_token_accuracy": 0.7936214208602905,
+      "num_tokens": 4024485.0,
+      "step": 1560
+    },
+    {
+      "entropy": 0.7805682122707367,
+      "epoch": 1.3487972508591066,
+      "grad_norm": 0.18578168749809265,
+      "learning_rate": 9.780927835051546e-05,
+      "loss": 0.7720298767089844,
+      "mean_token_accuracy": 0.7950874090194702,
+      "num_tokens": 4046854.0,
+      "step": 1570
+    },
+    {
+      "entropy": 0.8566708385944366,
+      "epoch": 1.3573883161512028,
+      "grad_norm": 0.20412200689315796,
+      "learning_rate": 9.652061855670104e-05,
+      "loss": 0.8537209510803223,
+      "mean_token_accuracy": 0.7804512619972229,
+      "num_tokens": 4072069.0,
+      "step": 1580
+    },
+    {
+      "entropy": 0.8159787476062774,
+      "epoch": 1.365979381443299,
+      "grad_norm": 0.20074458420276642,
+      "learning_rate": 9.523195876288658e-05,
+      "loss": 0.8033108711242676,
+      "mean_token_accuracy": 0.7907186150550842,
+      "num_tokens": 4098259.0,
+      "step": 1590
+    },
+    {
+      "entropy": 0.885843676328659,
+      "epoch": 1.3745704467353952,
+      "grad_norm": 0.1880599558353424,
+      "learning_rate": 9.394329896907216e-05,
+      "loss": 0.8871627807617187,
+      "mean_token_accuracy": 0.7741815447807312,
+      "num_tokens": 4124396.0,
+      "step": 1600
+    },
+    {
+      "entropy": 0.7828668534755707,
+      "epoch": 1.3831615120274914,
+      "grad_norm": 0.16421489417552948,
+      "learning_rate": 9.265463917525772e-05,
+      "loss": 0.7735602855682373,
+      "mean_token_accuracy": 0.7981120824813843,
+      "num_tokens": 4151812.0,
+      "step": 1610
+    },
+    {
+      "entropy": 0.869307404756546,
+      "epoch": 1.3917525773195876,
+      "grad_norm": 0.15142889320850372,
+      "learning_rate": 9.136597938144328e-05,
+      "loss": 0.8756357192993164,
+      "mean_token_accuracy": 0.7807520747184753,
+      "num_tokens": 4177965.0,
+      "step": 1620
+    },
+    {
+      "entropy": 0.888411694765091,
+      "epoch": 1.4003436426116838,
+      "grad_norm": 0.19432614743709564,
+      "learning_rate": 9.007731958762886e-05,
+      "loss": 0.8830459594726563,
+      "mean_token_accuracy": 0.7777862966060638,
+      "num_tokens": 4202779.0,
+      "step": 1630
+    },
+    {
+      "entropy": 0.7828814208507537,
+      "epoch": 1.40893470790378,
+      "grad_norm": 0.18378111720085144,
+      "learning_rate": 8.878865979381442e-05,
+      "loss": 0.7943001747131347,
+      "mean_token_accuracy": 0.7928287029266358,
+      "num_tokens": 4227115.0,
+      "step": 1640
+    },
+    {
+      "entropy": 0.8880654036998749,
+      "epoch": 1.4175257731958764,
+      "grad_norm": 0.18914829194545746,
+      "learning_rate": 8.75e-05,
+      "loss": 0.8926235198974609,
+      "mean_token_accuracy": 0.7749776244163513,
+      "num_tokens": 4253321.0,
+      "step": 1650
+    },
+    {
+      "entropy": 0.8914766550064087,
+      "epoch": 1.4261168384879725,
+      "grad_norm": 0.17971955239772797,
+      "learning_rate": 8.621134020618555e-05,
+      "loss": 0.8880870819091797,
+      "mean_token_accuracy": 0.7773807585239411,
+      "num_tokens": 4278329.0,
+      "step": 1660
+    },
+    {
+      "entropy": 0.8003105163574219,
+      "epoch": 1.4347079037800687,
+      "grad_norm": 0.17955686151981354,
+      "learning_rate": 8.492268041237112e-05,
+      "loss": 0.7776397705078125,
+      "mean_token_accuracy": 0.7956097960472107,
+      "num_tokens": 4304103.0,
+      "step": 1670
+    },
+    {
+      "entropy": 0.9243861019611359,
+      "epoch": 1.443298969072165,
+      "grad_norm": 0.1783088743686676,
+      "learning_rate": 8.36340206185567e-05,
+      "loss": 0.9214775085449218,
+      "mean_token_accuracy": 0.7736267566680908,
+      "num_tokens": 4331574.0,
+      "step": 1680
+    },
+    {
+      "entropy": 0.822593355178833,
+      "epoch": 1.4518900343642611,
+      "grad_norm": 0.15796494483947754,
+      "learning_rate": 8.234536082474226e-05,
+      "loss": 0.8136516571044922,
+      "mean_token_accuracy": 0.7927704989910126,
+      "num_tokens": 4356400.0,
+      "step": 1690
+    },
+    {
+      "entropy": 0.7921082556247712,
+      "epoch": 1.4604810996563573,
+      "grad_norm": 0.15428173542022705,
+      "learning_rate": 8.105670103092782e-05,
+      "loss": 0.799358606338501,
+      "mean_token_accuracy": 0.7963292002677917,
+      "num_tokens": 4380067.0,
+      "step": 1700
+    },
+    {
+      "entropy": 0.8694933772087097,
+      "epoch": 1.4690721649484537,
+      "grad_norm": 0.15096421539783478,
+      "learning_rate": 7.97680412371134e-05,
+      "loss": 0.8806812286376953,
+      "mean_token_accuracy": 0.7833354890346527,
+      "num_tokens": 4406454.0,
+      "step": 1710
+    },
+    {
+      "entropy": 0.9037382245063782,
+      "epoch": 1.47766323024055,
+      "grad_norm": 0.14613334834575653,
+      "learning_rate": 7.847938144329896e-05,
+      "loss": 0.9070594787597657,
+      "mean_token_accuracy": 0.7784084975719452,
+      "num_tokens": 4433753.0,
+      "step": 1720
+    },
+    {
+      "entropy": 0.8713727951049804,
+      "epoch": 1.486254295532646,
+      "grad_norm": 0.18062028288841248,
+      "learning_rate": 7.719072164948453e-05,
+      "loss": 0.8788843154907227,
+      "mean_token_accuracy": 0.7776244163513184,
+      "num_tokens": 4458675.0,
+      "step": 1730
+    },
+    {
+      "entropy": 0.8470430076122284,
+      "epoch": 1.4948453608247423,
+      "grad_norm": 0.17912915349006653,
+      "learning_rate": 7.590206185567009e-05,
+      "loss": 0.8447764396667481,
+      "mean_token_accuracy": 0.7814169287681579,
+      "num_tokens": 4484460.0,
+      "step": 1740
+    },
+    {
+      "entropy": 0.8573691666126251,
+      "epoch": 1.5034364261168385,
+      "grad_norm": 0.19001024961471558,
+      "learning_rate": 7.461340206185566e-05,
+      "loss": 0.8481529235839844,
+      "mean_token_accuracy": 0.7841818153858184,
+      "num_tokens": 4509367.0,
+      "step": 1750
+    },
+    {
+      "entropy": 0.812692153453827,
+      "epoch": 1.5120274914089347,
+      "grad_norm": 0.14844155311584473,
+      "learning_rate": 7.332474226804123e-05,
+      "loss": 0.7989925384521485,
+      "mean_token_accuracy": 0.7950545191764832,
+      "num_tokens": 4534267.0,
+      "step": 1760
+    },
+    {
+      "entropy": 0.8320570230484009,
+      "epoch": 1.5206185567010309,
+      "grad_norm": 0.15702161192893982,
+      "learning_rate": 7.20360824742268e-05,
+      "loss": 0.8325714111328125,
+      "mean_token_accuracy": 0.7877066612243653,
+      "num_tokens": 4560969.0,
+      "step": 1770
+    },
+    {
+      "entropy": 0.8388019800186157,
+      "epoch": 1.529209621993127,
+      "grad_norm": 0.18410871922969818,
+      "learning_rate": 7.074742268041236e-05,
+      "loss": 0.8359848976135253,
+      "mean_token_accuracy": 0.7866321623325347,
+      "num_tokens": 4588280.0,
+      "step": 1780
+    },
+    {
+      "entropy": 0.8161091566085815,
+      "epoch": 1.5378006872852232,
+      "grad_norm": 0.17120403051376343,
+      "learning_rate": 6.945876288659794e-05,
+      "loss": 0.7916454315185547,
+      "mean_token_accuracy": 0.7996809184551239,
+      "num_tokens": 4613497.0,
+      "step": 1790
+    },
+    {
+      "entropy": 0.7915269613265992,
+      "epoch": 1.5463917525773194,
+      "grad_norm": 0.1651919186115265,
+      "learning_rate": 6.81701030927835e-05,
+      "loss": 0.7937773704528809,
+      "mean_token_accuracy": 0.7961802840232849,
+      "num_tokens": 4638371.0,
+      "step": 1800
+    },
+    {
+      "entropy": 0.8150420367717743,
+      "epoch": 1.5549828178694158,
+      "grad_norm": 0.17061056196689606,
+      "learning_rate": 6.688144329896906e-05,
+      "loss": 0.8088832855224609,
+      "mean_token_accuracy": 0.7914064228534698,
+      "num_tokens": 4663692.0,
+      "step": 1810
+    },
+    {
+      "entropy": 0.7606650114059448,
+      "epoch": 1.563573883161512,
+      "grad_norm": 0.20968209207057953,
+      "learning_rate": 6.559278350515463e-05,
+      "loss": 0.753024673461914,
+      "mean_token_accuracy": 0.8017209947109223,
+      "num_tokens": 4688045.0,
+      "step": 1820
+    },
+    {
+      "entropy": 0.8470539689064026,
+      "epoch": 1.5721649484536082,
+      "grad_norm": 0.19549283385276794,
+      "learning_rate": 6.43041237113402e-05,
+      "loss": 0.8378950119018554,
+      "mean_token_accuracy": 0.785133320093155,
+      "num_tokens": 4713328.0,
+      "step": 1830
+    },
+    {
+      "entropy": 0.8347568809986115,
+      "epoch": 1.5807560137457046,
+      "grad_norm": 0.19542083144187927,
+      "learning_rate": 6.301546391752577e-05,
+      "loss": 0.8309958457946778,
+      "mean_token_accuracy": 0.7830386936664582,
+      "num_tokens": 4736645.0,
+      "step": 1840
+    },
+    {
+      "entropy": 0.8486950516700744,
+      "epoch": 1.5893470790378008,
+      "grad_norm": 0.17605172097682953,
+      "learning_rate": 6.172680412371133e-05,
+      "loss": 0.8492973327636719,
+      "mean_token_accuracy": 0.7818349421024322,
+      "num_tokens": 4762593.0,
+      "step": 1850
+    },
+    {
+      "entropy": 0.8633189976215363,
+      "epoch": 1.597938144329897,
+      "grad_norm": 0.1746501475572586,
+      "learning_rate": 6.0438144329896904e-05,
+      "loss": 0.8715569496154785,
+      "mean_token_accuracy": 0.7832526326179504,
+      "num_tokens": 4790512.0,
+      "step": 1860
+    },
+    {
+      "entropy": 0.8331901133060455,
+      "epoch": 1.6065292096219932,
+      "grad_norm": 0.18069683015346527,
+      "learning_rate": 5.9149484536082466e-05,
+      "loss": 0.8255064010620117,
+      "mean_token_accuracy": 0.7909268379211426,
+      "num_tokens": 4815026.0,
+      "step": 1870
+    },
+    {
+      "entropy": 0.8436095595359803,
+      "epoch": 1.6151202749140894,
+      "grad_norm": 0.17736031115055084,
+      "learning_rate": 5.786082474226804e-05,
+      "loss": 0.8569248199462891,
+      "mean_token_accuracy": 0.7808656811714172,
+      "num_tokens": 4839215.0,
+      "step": 1880
+    },
+    {
+      "entropy": 0.7866667926311492,
+      "epoch": 1.6237113402061856,
+      "grad_norm": 0.17926126718521118,
+      "learning_rate": 5.6572164948453604e-05,
+      "loss": 0.7782703399658203,
+      "mean_token_accuracy": 0.79602530002594,
+      "num_tokens": 4862402.0,
+      "step": 1890
+    },
+    {
+      "entropy": 0.8086029171943665,
+      "epoch": 1.6323024054982818,
+      "grad_norm": 0.18922077119350433,
+      "learning_rate": 5.5283505154639174e-05,
+      "loss": 0.793773603439331,
+      "mean_token_accuracy": 0.7902358531951904,
+      "num_tokens": 4884961.0,
+      "step": 1900
+    },
+    {
+      "entropy": 0.8036526799201965,
+      "epoch": 1.640893470790378,
+      "grad_norm": 0.16250303387641907,
+      "learning_rate": 5.3994845360824736e-05,
+      "loss": 0.8118386268615723,
+      "mean_token_accuracy": 0.7943076312541961,
+      "num_tokens": 4910669.0,
+      "step": 1910
+    },
+    {
+      "entropy": 0.8436183631420135,
+      "epoch": 1.6494845360824741,
+      "grad_norm": 0.20115436613559723,
+      "learning_rate": 5.27061855670103e-05,
+      "loss": 0.8377361297607422,
+      "mean_token_accuracy": 0.7822086751461029,
+      "num_tokens": 4936469.0,
+      "step": 1920
+    },
+    {
+      "entropy": 0.8387295484542847,
+      "epoch": 1.6580756013745703,
+      "grad_norm": 0.18081411719322205,
+      "learning_rate": 5.1417525773195874e-05,
+      "loss": 0.8227550506591796,
+      "mean_token_accuracy": 0.7908512234687806,
+      "num_tokens": 4961585.0,
+      "step": 1930
+    },
+    {
+      "entropy": 0.827156925201416,
+      "epoch": 1.6666666666666665,
+      "grad_norm": 0.16519112884998322,
+      "learning_rate": 5.012886597938144e-05,
+      "loss": 0.8146243095397949,
+      "mean_token_accuracy": 0.7934521496295929,
+      "num_tokens": 4986973.0,
+      "step": 1940
+    },
+    {
+      "entropy": 0.8766247212886811,
+      "epoch": 1.675257731958763,
+      "grad_norm": 0.18259386718273163,
+      "learning_rate": 4.8840206185567006e-05,
+      "loss": 0.9040776252746582,
+      "mean_token_accuracy": 0.7768758893013,
+      "num_tokens": 5015328.0,
+      "step": 1950
+    },
+    {
+      "entropy": 0.8589329719543457,
+      "epoch": 1.6838487972508591,
+      "grad_norm": 0.17805488407611847,
+      "learning_rate": 4.755154639175257e-05,
+      "loss": 0.843138599395752,
+      "mean_token_accuracy": 0.7871474087238312,
+      "num_tokens": 5040331.0,
+      "step": 1960
+    },
+    {
+      "entropy": 0.8477513492107391,
+      "epoch": 1.6924398625429553,
+      "grad_norm": 0.1912812739610672,
+      "learning_rate": 4.6262886597938144e-05,
+      "loss": 0.8690207481384278,
+      "mean_token_accuracy": 0.7804648995399475,
+      "num_tokens": 5067476.0,
+      "step": 1970
+    },
+    {
+      "entropy": 0.839428323507309,
+      "epoch": 1.7010309278350515,
+      "grad_norm": 0.17795996367931366,
+      "learning_rate": 4.4974226804123706e-05,
+      "loss": 0.8250746726989746,
+      "mean_token_accuracy": 0.7934922575950623,
+      "num_tokens": 5093307.0,
+      "step": 1980
+    },
+    {
+      "entropy": 0.8613415479660034,
+      "epoch": 1.709621993127148,
+      "grad_norm": 0.16119547188282013,
+      "learning_rate": 4.3685567010309276e-05,
+      "loss": 0.8428993225097656,
+      "mean_token_accuracy": 0.7848868429660797,
+      "num_tokens": 5119275.0,
+      "step": 1990
+    },
+    {
+      "entropy": 0.916008472442627,
+      "epoch": 1.718213058419244,
+      "grad_norm": 0.17801715433597565,
+      "learning_rate": 4.239690721649484e-05,
+      "loss": 0.9225644111633301,
+      "mean_token_accuracy": 0.7754484415054321,
+      "num_tokens": 5145206.0,
+      "step": 2000
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 2328,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 8417798278571520.0,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-2000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2af45870b4c4d611381846df6bdd4a32cb292a7b5897b79b7e5f196be57c596
+size 5585

checkpoint-2328/README.md

@@ -0,0 +1,209 @@
+---
+base_model: HuggingFaceTB/SmolLM2-135M-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:HuggingFaceTB/SmolLM2-135M-Instruct
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

checkpoint-2328/adapter_config.json

@@ -0,0 +1,43 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "HuggingFaceTB/SmolLM2-135M-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "k_proj",
+    "q_proj",
+    "v_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-2328/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7fc9ea20edd42cffd8c5b32349900876962b89d0cf0b52aa2d24603f2324ff2a
+size 7404368

checkpoint-2328/chat_template.jinja

@@ -0,0 +1,6 @@
+{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system
+You are a helpful AI assistant named SmolLM, trained by Hugging Face<|im_end|>
+' }}{% endif %}{{'<|im_start|>' + message['role'] + '
+' + message['content'] + '<|im_end|>' + '
+'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
+' }}{% endif %}

checkpoint-2328/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c531d5c0a682388d0333585d06d904c7a8cd92db47c6559ae30bea0d39ab20c9
+size 14950667

checkpoint-2328/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5673270aeec65920561e25573ba9faecb0ca3976d65cd2f6489d7752232bd40
+size 14645

checkpoint-2328/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:834a2e6b633182438229fc6544de1f73bd698865a519299ffd79b8b9dff673ae
+size 1383

checkpoint-2328/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a491c550d4a09236e07f87e9a8ccc2e31bb1a97085c661c7e88845ead685832c
+size 1465

checkpoint-2328/tokenizer_config.json

@@ -0,0 +1,17 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|im_start|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "is_local": false,
+  "model_max_length": 8192,
+  "pad_token": "<|im_end|>",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "<|endoftext|>",
+  "vocab_size": 49152
+}