Training in progress, step 8800

adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:72ca83f2e58139442e0fa82f75f99341b3fa68d264df6c740675c12dc5800842
+oid sha256:9e35d9491720d9e4b3645080434e252d16a48abe69885184ea34a2f11ec6b605
 size 319977229

checkpoint-6600/adapter_model/adapter_model/README.md

@@ -1,20 +0,0 @@
----
-library_name: peft
----
-## Training procedure
-
-
-The following `bitsandbytes` quantization config was used during training:
-- load_in_8bit: False
-- load_in_4bit: True
-- llm_int8_threshold: 6.0
-- llm_int8_skip_modules: None
-- llm_int8_enable_fp32_cpu_offload: False
-- llm_int8_has_fp16_weight: False
-- bnb_4bit_quant_type: nf4
-- bnb_4bit_use_double_quant: True
-- bnb_4bit_compute_dtype: bfloat16
-### Framework versions
-
-
-- PEFT 0.4.0

checkpoint-6600/adapter_model/adapter_model/adapter_config.json

@@ -1,26 +0,0 @@
-{
-  "auto_mapping": null,
-  "base_model_name_or_path": "pankajmathur/orca_mini_v3_7b",
-  "bias": "none",
-  "fan_in_fan_out": false,
-  "inference_mode": true,
-  "init_lora_weights": true,
-  "layers_pattern": null,
-  "layers_to_transform": null,
-  "lora_alpha": 16.0,
-  "lora_dropout": 0.1,
-  "modules_to_save": null,
-  "peft_type": "LORA",
-  "r": 64,
-  "revision": null,
-  "target_modules": [
-    "gate_proj",
-    "k_proj",
-    "down_proj",
-    "up_proj",
-    "o_proj",
-    "v_proj",
-    "q_proj"
-  ],
-  "task_type": "CAUSAL_LM"
-}

checkpoint-6600/adapter_model/adapter_model/adapter_model.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:acaf6023a66bfe9eb2692b7a2d808e465670c18e7d11d715f587f3b14ec28504
-size 319977229

checkpoint-7800/adapter_model/adapter_model/README.md

@@ -37,6 +37,17 @@ The following `bitsandbytes` quantization config was used during training:
 - bnb_4bit_use_double_quant: True
 - bnb_4bit_compute_dtype: bfloat16
 
+The following `bitsandbytes` quantization config was used during training:
+- load_in_8bit: False
+- load_in_4bit: True
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: nf4
+- bnb_4bit_use_double_quant: True
+- bnb_4bit_compute_dtype: bfloat16
+
 The following `bitsandbytes` quantization config was used during training:
 - load_in_8bit: False
 - load_in_4bit: True
@@ -52,5 +63,6 @@ The following `bitsandbytes` quantization config was used during training:
 - PEFT 0.4.0
 - PEFT 0.4.0
 - PEFT 0.4.0
+- PEFT 0.4.0
 
 - PEFT 0.4.0

checkpoint-7800/adapter_model/adapter_model/adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:75334ddde69c176d2ab795d826607ac387040334d49a9fd216e6fe5f164e8ec5
+oid sha256:72ca83f2e58139442e0fa82f75f99341b3fa68d264df6c740675c12dc5800842
 size 319977229

{checkpoint-6600 → checkpoint-8800}/adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:acaf6023a66bfe9eb2692b7a2d808e465670c18e7d11d715f587f3b14ec28504
+oid sha256:9e35d9491720d9e4b3645080434e252d16a48abe69885184ea34a2f11ec6b605
 size 319977229

{checkpoint-6600 → checkpoint-8800}/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:95d41756132a5bb626f9e9786d4b67b7301681be7ccd63bcf137c143998aa9c4
+oid sha256:d7e499f67c1b099f240d65342d10e82eb2a01f258fc1f82aa57071b60441560f
 size 1279539973

{checkpoint-6600 → checkpoint-8800}/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:af0b1e85e5863d4156d8564a40d63a46923fd75f8bbdb43c8576cfc2d355b3dd
+oid sha256:d59859eb2318ce4e5ddb5f0d315123fcd694906c4a9c0b5b0fa865bee3e9866d
 size 14511

{checkpoint-6600 → checkpoint-8800}/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aa19c433c8c029403e57118df2ab52631b3fc535294c01cab201bdeb198ed0f4
+oid sha256:6c21e5e690d8e0cade5418c2fcaefb2d060d01ccdb64d9d6411a6d2dbbe882b0
 size 627

{checkpoint-6600 → checkpoint-8800}/trainer_state.json

@@ -1,8 +1,8 @@
 {
-  "best_metric": 0.7248952984809875,
-  "best_model_checkpoint": "experts/expert-4/checkpoint-6600",
-  "epoch": 0.7378011290593035,
-  "global_step": 6600,
+  "best_metric": 0.7206757664680481,
+  "best_model_checkpoint": "experts/expert-4/checkpoint-7800",
+  "epoch": 0.9837348387457381,
+  "global_step": 8800,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -6309,11 +6309,2112 @@
       "mmlu_eval_accuracy_world_religions": 0.631578947368421,
       "mmlu_loss": 0.8850108603545642,
       "step": 6600
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0002,
+      "loss": 0.7221,
+      "step": 6610
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0002,
+      "loss": 0.75,
+      "step": 6620
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0002,
+      "loss": 0.8407,
+      "step": 6630
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0002,
+      "loss": 0.6769,
+      "step": 6640
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0002,
+      "loss": 0.8187,
+      "step": 6650
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0002,
+      "loss": 0.8102,
+      "step": 6660
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0002,
+      "loss": 0.7222,
+      "step": 6670
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0002,
+      "loss": 0.7202,
+      "step": 6680
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0002,
+      "loss": 0.8201,
+      "step": 6690
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0002,
+      "loss": 0.5917,
+      "step": 6700
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0002,
+      "loss": 0.6594,
+      "step": 6710
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0002,
+      "loss": 0.6152,
+      "step": 6720
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0002,
+      "loss": 0.8191,
+      "step": 6730
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0002,
+      "loss": 0.7112,
+      "step": 6740
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0002,
+      "loss": 0.6968,
+      "step": 6750
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0002,
+      "loss": 0.651,
+      "step": 6760
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0002,
+      "loss": 0.8289,
+      "step": 6770
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0002,
+      "loss": 0.8347,
+      "step": 6780
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0002,
+      "loss": 0.7234,
+      "step": 6790
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0002,
+      "loss": 0.7227,
+      "step": 6800
+    },
+    {
+      "epoch": 0.76,
+      "eval_loss": 0.7229225039482117,
+      "eval_runtime": 133.823,
+      "eval_samples_per_second": 7.473,
+      "eval_steps_per_second": 3.736,
+      "step": 6800
+    },
+    {
+      "epoch": 0.76,
+      "mmlu_eval_accuracy": 0.5152214474699063,
+      "mmlu_eval_accuracy_abstract_algebra": 0.36363636363636365,
+      "mmlu_eval_accuracy_anatomy": 0.7142857142857143,
+      "mmlu_eval_accuracy_astronomy": 0.5625,
+      "mmlu_eval_accuracy_business_ethics": 0.45454545454545453,
+      "mmlu_eval_accuracy_clinical_knowledge": 0.5172413793103449,
+      "mmlu_eval_accuracy_college_biology": 0.4375,
+      "mmlu_eval_accuracy_college_chemistry": 0.125,
+      "mmlu_eval_accuracy_college_computer_science": 0.6363636363636364,
+      "mmlu_eval_accuracy_college_mathematics": 0.18181818181818182,
+      "mmlu_eval_accuracy_college_medicine": 0.36363636363636365,
+      "mmlu_eval_accuracy_college_physics": 0.36363636363636365,
+      "mmlu_eval_accuracy_computer_security": 0.45454545454545453,
+      "mmlu_eval_accuracy_conceptual_physics": 0.4230769230769231,
+      "mmlu_eval_accuracy_econometrics": 0.3333333333333333,
+      "mmlu_eval_accuracy_electrical_engineering": 0.3125,
+      "mmlu_eval_accuracy_elementary_mathematics": 0.36585365853658536,
+      "mmlu_eval_accuracy_formal_logic": 0.21428571428571427,
+      "mmlu_eval_accuracy_global_facts": 0.4,
+      "mmlu_eval_accuracy_high_school_biology": 0.375,
+      "mmlu_eval_accuracy_high_school_chemistry": 0.22727272727272727,
+      "mmlu_eval_accuracy_high_school_computer_science": 0.5555555555555556,
+      "mmlu_eval_accuracy_high_school_european_history": 0.5555555555555556,
+      "mmlu_eval_accuracy_high_school_geography": 0.8636363636363636,
+      "mmlu_eval_accuracy_high_school_government_and_politics": 0.7142857142857143,
+      "mmlu_eval_accuracy_high_school_macroeconomics": 0.46511627906976744,
+      "mmlu_eval_accuracy_high_school_mathematics": 0.27586206896551724,
+      "mmlu_eval_accuracy_high_school_microeconomics": 0.5384615384615384,
+      "mmlu_eval_accuracy_high_school_physics": 0.23529411764705882,
+      "mmlu_eval_accuracy_high_school_psychology": 0.8166666666666667,
+      "mmlu_eval_accuracy_high_school_statistics": 0.4782608695652174,
+      "mmlu_eval_accuracy_high_school_us_history": 0.6363636363636364,
+      "mmlu_eval_accuracy_high_school_world_history": 0.6538461538461539,
+      "mmlu_eval_accuracy_human_aging": 0.7391304347826086,
+      "mmlu_eval_accuracy_human_sexuality": 0.4166666666666667,
+      "mmlu_eval_accuracy_international_law": 0.9230769230769231,
+      "mmlu_eval_accuracy_jurisprudence": 0.45454545454545453,
+      "mmlu_eval_accuracy_logical_fallacies": 0.6111111111111112,
+      "mmlu_eval_accuracy_machine_learning": 0.36363636363636365,
+      "mmlu_eval_accuracy_management": 0.7272727272727273,
+      "mmlu_eval_accuracy_marketing": 0.8,
+      "mmlu_eval_accuracy_medical_genetics": 0.9090909090909091,
+      "mmlu_eval_accuracy_miscellaneous": 0.627906976744186,
+      "mmlu_eval_accuracy_moral_disputes": 0.4473684210526316,
+      "mmlu_eval_accuracy_moral_scenarios": 0.22,
+      "mmlu_eval_accuracy_nutrition": 0.6363636363636364,
+      "mmlu_eval_accuracy_philosophy": 0.5,
+      "mmlu_eval_accuracy_prehistory": 0.5142857142857142,
+      "mmlu_eval_accuracy_professional_accounting": 0.45161290322580644,
+      "mmlu_eval_accuracy_professional_law": 0.35294117647058826,
+      "mmlu_eval_accuracy_professional_medicine": 0.4838709677419355,
+      "mmlu_eval_accuracy_professional_psychology": 0.5507246376811594,
+      "mmlu_eval_accuracy_public_relations": 0.4166666666666667,
+      "mmlu_eval_accuracy_security_studies": 0.48148148148148145,
+      "mmlu_eval_accuracy_sociology": 0.8181818181818182,
+      "mmlu_eval_accuracy_us_foreign_policy": 0.9090909090909091,
+      "mmlu_eval_accuracy_virology": 0.5555555555555556,
+      "mmlu_eval_accuracy_world_religions": 0.8421052631578947,
+      "mmlu_loss": 1.011317597901541,
+      "step": 6800
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0002,
+      "loss": 0.7919,
+      "step": 6810
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0002,
+      "loss": 0.6663,
+      "step": 6820
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0002,
+      "loss": 0.7148,
+      "step": 6830
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0002,
+      "loss": 0.7675,
+      "step": 6840
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.0002,
+      "loss": 0.6595,
+      "step": 6850
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.0002,
+      "loss": 0.8117,
+      "step": 6860
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.0002,
+      "loss": 0.7273,
+      "step": 6870
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.0002,
+      "loss": 0.7276,
+      "step": 6880
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.0002,
+      "loss": 0.6568,
+      "step": 6890
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.0002,
+      "loss": 0.6933,
+      "step": 6900
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.0002,
+      "loss": 0.5793,
+      "step": 6910
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.0002,
+      "loss": 0.6824,
+      "step": 6920
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.0002,
+      "loss": 0.7155,
+      "step": 6930
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.0002,
+      "loss": 0.854,
+      "step": 6940
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.0002,
+      "loss": 0.7699,
+      "step": 6950
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.0002,
+      "loss": 0.8371,
+      "step": 6960
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.0002,
+      "loss": 0.7292,
+      "step": 6970
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.0002,
+      "loss": 0.6799,
+      "step": 6980
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.0002,
+      "loss": 0.7101,
+      "step": 6990
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.0002,
+      "loss": 0.6587,
+      "step": 7000
+    },
+    {
+      "epoch": 0.78,
+      "eval_loss": 0.7265365123748779,
+      "eval_runtime": 133.844,
+      "eval_samples_per_second": 7.471,
+      "eval_steps_per_second": 3.736,
+      "step": 7000
+    },
+    {
+      "epoch": 0.78,
+      "mmlu_eval_accuracy": 0.4880448490136911,
+      "mmlu_eval_accuracy_abstract_algebra": 0.36363636363636365,
+      "mmlu_eval_accuracy_anatomy": 0.7142857142857143,
+      "mmlu_eval_accuracy_astronomy": 0.4375,
+      "mmlu_eval_accuracy_business_ethics": 0.36363636363636365,
+      "mmlu_eval_accuracy_clinical_knowledge": 0.5517241379310345,
+      "mmlu_eval_accuracy_college_biology": 0.5,
+      "mmlu_eval_accuracy_college_chemistry": 0.125,
+      "mmlu_eval_accuracy_college_computer_science": 0.2727272727272727,
+      "mmlu_eval_accuracy_college_mathematics": 0.2727272727272727,
+      "mmlu_eval_accuracy_college_medicine": 0.45454545454545453,
+      "mmlu_eval_accuracy_college_physics": 0.45454545454545453,
+      "mmlu_eval_accuracy_computer_security": 0.45454545454545453,
+      "mmlu_eval_accuracy_conceptual_physics": 0.38461538461538464,
+      "mmlu_eval_accuracy_econometrics": 0.25,
+      "mmlu_eval_accuracy_electrical_engineering": 0.1875,
+      "mmlu_eval_accuracy_elementary_mathematics": 0.34146341463414637,
+      "mmlu_eval_accuracy_formal_logic": 0.14285714285714285,
+      "mmlu_eval_accuracy_global_facts": 0.6,
+      "mmlu_eval_accuracy_high_school_biology": 0.375,
+      "mmlu_eval_accuracy_high_school_chemistry": 0.2727272727272727,
+      "mmlu_eval_accuracy_high_school_computer_science": 0.5555555555555556,
+      "mmlu_eval_accuracy_high_school_european_history": 0.6111111111111112,
+      "mmlu_eval_accuracy_high_school_geography": 0.8181818181818182,
+      "mmlu_eval_accuracy_high_school_government_and_politics": 0.47619047619047616,
+      "mmlu_eval_accuracy_high_school_macroeconomics": 0.3488372093023256,
+      "mmlu_eval_accuracy_high_school_mathematics": 0.2413793103448276,
+      "mmlu_eval_accuracy_high_school_microeconomics": 0.5,
+      "mmlu_eval_accuracy_high_school_physics": 0.29411764705882354,
+      "mmlu_eval_accuracy_high_school_psychology": 0.7833333333333333,
+      "mmlu_eval_accuracy_high_school_statistics": 0.43478260869565216,
+      "mmlu_eval_accuracy_high_school_us_history": 0.8181818181818182,
+      "mmlu_eval_accuracy_high_school_world_history": 0.6153846153846154,
+      "mmlu_eval_accuracy_human_aging": 0.6956521739130435,
+      "mmlu_eval_accuracy_human_sexuality": 0.3333333333333333,
+      "mmlu_eval_accuracy_international_law": 0.9230769230769231,
+      "mmlu_eval_accuracy_jurisprudence": 0.45454545454545453,
+      "mmlu_eval_accuracy_logical_fallacies": 0.6666666666666666,
+      "mmlu_eval_accuracy_machine_learning": 0.2727272727272727,
+      "mmlu_eval_accuracy_management": 0.7272727272727273,
+      "mmlu_eval_accuracy_marketing": 0.76,
+      "mmlu_eval_accuracy_medical_genetics": 0.9090909090909091,
+      "mmlu_eval_accuracy_miscellaneous": 0.6395348837209303,
+      "mmlu_eval_accuracy_moral_disputes": 0.5263157894736842,
+      "mmlu_eval_accuracy_moral_scenarios": 0.22,
+      "mmlu_eval_accuracy_nutrition": 0.5454545454545454,
+      "mmlu_eval_accuracy_philosophy": 0.4411764705882353,
+      "mmlu_eval_accuracy_prehistory": 0.42857142857142855,
+      "mmlu_eval_accuracy_professional_accounting": 0.41935483870967744,
+      "mmlu_eval_accuracy_professional_law": 0.3352941176470588,
+      "mmlu_eval_accuracy_professional_medicine": 0.45161290322580644,
+      "mmlu_eval_accuracy_professional_psychology": 0.4927536231884058,
+      "mmlu_eval_accuracy_public_relations": 0.5833333333333334,
+      "mmlu_eval_accuracy_security_studies": 0.48148148148148145,
+      "mmlu_eval_accuracy_sociology": 0.7272727272727273,
+      "mmlu_eval_accuracy_us_foreign_policy": 0.6363636363636364,
+      "mmlu_eval_accuracy_virology": 0.5,
+      "mmlu_eval_accuracy_world_religions": 0.631578947368421,
+      "mmlu_loss": 1.021762224094208,
+      "step": 7000
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.0002,
+      "loss": 0.7768,
+      "step": 7010
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.0002,
+      "loss": 0.8015,
+      "step": 7020
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.0002,
+      "loss": 0.7361,
+      "step": 7030
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.0002,
+      "loss": 0.6737,
+      "step": 7040
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.0002,
+      "loss": 0.7194,
+      "step": 7050
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.0002,
+      "loss": 0.7331,
+      "step": 7060
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.0002,
+      "loss": 0.719,
+      "step": 7070
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.0002,
+      "loss": 0.71,
+      "step": 7080
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.0002,
+      "loss": 0.7005,
+      "step": 7090
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.0002,
+      "loss": 0.7349,
+      "step": 7100
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.0002,
+      "loss": 0.8054,
+      "step": 7110
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.0002,
+      "loss": 0.7939,
+      "step": 7120
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.0002,
+      "loss": 0.7347,
+      "step": 7130
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.0002,
+      "loss": 0.7617,
+      "step": 7140
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.0002,
+      "loss": 0.6904,
+      "step": 7150
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.0002,
+      "loss": 0.6306,
+      "step": 7160
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.0002,
+      "loss": 0.8292,
+      "step": 7170
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.0002,
+      "loss": 0.727,
+      "step": 7180
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.0002,
+      "loss": 0.7253,
+      "step": 7190
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.0002,
+      "loss": 0.7563,
+      "step": 7200
+    },
+    {
+      "epoch": 0.8,
+      "eval_loss": 0.7270277142524719,
+      "eval_runtime": 133.8707,
+      "eval_samples_per_second": 7.47,
+      "eval_steps_per_second": 3.735,
+      "step": 7200
+    },
+    {
+      "epoch": 0.8,
+      "mmlu_eval_accuracy": 0.4911456222026462,
+      "mmlu_eval_accuracy_abstract_algebra": 0.45454545454545453,
+      "mmlu_eval_accuracy_anatomy": 0.5714285714285714,
+      "mmlu_eval_accuracy_astronomy": 0.3125,
+      "mmlu_eval_accuracy_business_ethics": 0.36363636363636365,
+      "mmlu_eval_accuracy_clinical_knowledge": 0.5517241379310345,
+      "mmlu_eval_accuracy_college_biology": 0.4375,
+      "mmlu_eval_accuracy_college_chemistry": 0.25,
+      "mmlu_eval_accuracy_college_computer_science": 0.2727272727272727,
+      "mmlu_eval_accuracy_college_mathematics": 0.18181818181818182,
+      "mmlu_eval_accuracy_college_medicine": 0.45454545454545453,
+      "mmlu_eval_accuracy_college_physics": 0.45454545454545453,
+      "mmlu_eval_accuracy_computer_security": 0.5454545454545454,
+      "mmlu_eval_accuracy_conceptual_physics": 0.4230769230769231,
+      "mmlu_eval_accuracy_econometrics": 0.4166666666666667,
+      "mmlu_eval_accuracy_electrical_engineering": 0.3125,
+      "mmlu_eval_accuracy_elementary_mathematics": 0.36585365853658536,
+      "mmlu_eval_accuracy_formal_logic": 0.21428571428571427,
+      "mmlu_eval_accuracy_global_facts": 0.6,
+      "mmlu_eval_accuracy_high_school_biology": 0.46875,
+      "mmlu_eval_accuracy_high_school_chemistry": 0.22727272727272727,
+      "mmlu_eval_accuracy_high_school_computer_science": 0.5555555555555556,
+      "mmlu_eval_accuracy_high_school_european_history": 0.5,
+      "mmlu_eval_accuracy_high_school_geography": 0.8181818181818182,
+      "mmlu_eval_accuracy_high_school_government_and_politics": 0.6666666666666666,
+      "mmlu_eval_accuracy_high_school_macroeconomics": 0.4418604651162791,
+      "mmlu_eval_accuracy_high_school_mathematics": 0.20689655172413793,
+      "mmlu_eval_accuracy_high_school_microeconomics": 0.5769230769230769,
+      "mmlu_eval_accuracy_high_school_physics": 0.35294117647058826,
+      "mmlu_eval_accuracy_high_school_psychology": 0.8166666666666667,
+      "mmlu_eval_accuracy_high_school_statistics": 0.30434782608695654,
+      "mmlu_eval_accuracy_high_school_us_history": 0.6363636363636364,
+      "mmlu_eval_accuracy_high_school_world_history": 0.6538461538461539,
+      "mmlu_eval_accuracy_human_aging": 0.5652173913043478,
+      "mmlu_eval_accuracy_human_sexuality": 0.3333333333333333,
+      "mmlu_eval_accuracy_international_law": 0.9230769230769231,
+      "mmlu_eval_accuracy_jurisprudence": 0.5454545454545454,
+      "mmlu_eval_accuracy_logical_fallacies": 0.6666666666666666,
+      "mmlu_eval_accuracy_machine_learning": 0.18181818181818182,
+      "mmlu_eval_accuracy_management": 0.6363636363636364,
+      "mmlu_eval_accuracy_marketing": 0.8,
+      "mmlu_eval_accuracy_medical_genetics": 0.9090909090909091,
+      "mmlu_eval_accuracy_miscellaneous": 0.6046511627906976,
+      "mmlu_eval_accuracy_moral_disputes": 0.5,
+      "mmlu_eval_accuracy_moral_scenarios": 0.22,
+      "mmlu_eval_accuracy_nutrition": 0.5757575757575758,
+      "mmlu_eval_accuracy_philosophy": 0.5294117647058824,
+      "mmlu_eval_accuracy_prehistory": 0.4857142857142857,
+      "mmlu_eval_accuracy_professional_accounting": 0.3225806451612903,
+      "mmlu_eval_accuracy_professional_law": 0.37058823529411766,
+      "mmlu_eval_accuracy_professional_medicine": 0.3870967741935484,
+      "mmlu_eval_accuracy_professional_psychology": 0.5072463768115942,
+      "mmlu_eval_accuracy_public_relations": 0.5,
+      "mmlu_eval_accuracy_security_studies": 0.48148148148148145,
+      "mmlu_eval_accuracy_sociology": 0.6818181818181818,
+      "mmlu_eval_accuracy_us_foreign_policy": 0.7272727272727273,
+      "mmlu_eval_accuracy_virology": 0.5,
+      "mmlu_eval_accuracy_world_religions": 0.631578947368421,
+      "mmlu_loss": 1.171269353702859,
+      "step": 7200
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 0.0002,
+      "loss": 0.7924,
+      "step": 7210
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 0.0002,
+      "loss": 0.7404,
+      "step": 7220
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 0.0002,
+      "loss": 0.7767,
+      "step": 7230
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 0.0002,
+      "loss": 0.7569,
+      "step": 7240
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 0.0002,
+      "loss": 0.7199,
+      "step": 7250
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 0.0002,
+      "loss": 0.7314,
+      "step": 7260
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 0.0002,
+      "loss": 0.7551,
+      "step": 7270
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 0.0002,
+      "loss": 0.811,
+      "step": 7280
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 0.0002,
+      "loss": 0.7722,
+      "step": 7290
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 0.0002,
+      "loss": 0.7613,
+      "step": 7300
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 0.0002,
+      "loss": 0.6748,
+      "step": 7310
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 0.0002,
+      "loss": 0.7915,
+      "step": 7320
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 0.0002,
+      "loss": 0.8202,
+      "step": 7330
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 0.0002,
+      "loss": 0.6924,
+      "step": 7340
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 0.0002,
+      "loss": 0.7208,
+      "step": 7350
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 0.0002,
+      "loss": 0.6896,
+      "step": 7360
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 0.0002,
+      "loss": 0.7421,
+      "step": 7370
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 0.0002,
+      "loss": 0.7677,
+      "step": 7380
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 0.0002,
+      "loss": 0.7429,
+      "step": 7390
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 0.0002,
+      "loss": 0.6691,
+      "step": 7400
+    },
+    {
+      "epoch": 0.83,
+      "eval_loss": 0.7271261215209961,
+      "eval_runtime": 133.7069,
+      "eval_samples_per_second": 7.479,
+      "eval_steps_per_second": 3.74,
+      "step": 7400
+    },
+    {
+      "epoch": 0.83,
+      "mmlu_eval_accuracy": 0.5056019574032259,
+      "mmlu_eval_accuracy_abstract_algebra": 0.45454545454545453,
+      "mmlu_eval_accuracy_anatomy": 0.5714285714285714,
+      "mmlu_eval_accuracy_astronomy": 0.4375,
+      "mmlu_eval_accuracy_business_ethics": 0.5454545454545454,
+      "mmlu_eval_accuracy_clinical_knowledge": 0.5172413793103449,
+      "mmlu_eval_accuracy_college_biology": 0.5,
+      "mmlu_eval_accuracy_college_chemistry": 0.0,
+      "mmlu_eval_accuracy_college_computer_science": 0.18181818181818182,
+      "mmlu_eval_accuracy_college_mathematics": 0.2727272727272727,
+      "mmlu_eval_accuracy_college_medicine": 0.36363636363636365,
+      "mmlu_eval_accuracy_college_physics": 0.2727272727272727,
+      "mmlu_eval_accuracy_computer_security": 0.6363636363636364,
+      "mmlu_eval_accuracy_conceptual_physics": 0.6153846153846154,
+      "mmlu_eval_accuracy_econometrics": 0.16666666666666666,
+      "mmlu_eval_accuracy_electrical_engineering": 0.25,
+      "mmlu_eval_accuracy_elementary_mathematics": 0.36585365853658536,
+      "mmlu_eval_accuracy_formal_logic": 0.14285714285714285,
+      "mmlu_eval_accuracy_global_facts": 0.5,
+      "mmlu_eval_accuracy_high_school_biology": 0.5,
+      "mmlu_eval_accuracy_high_school_chemistry": 0.2727272727272727,
+      "mmlu_eval_accuracy_high_school_computer_science": 0.7777777777777778,
+      "mmlu_eval_accuracy_high_school_european_history": 0.6111111111111112,
+      "mmlu_eval_accuracy_high_school_geography": 0.8181818181818182,
+      "mmlu_eval_accuracy_high_school_government_and_politics": 0.5238095238095238,
+      "mmlu_eval_accuracy_high_school_macroeconomics": 0.46511627906976744,
+      "mmlu_eval_accuracy_high_school_mathematics": 0.1724137931034483,
+      "mmlu_eval_accuracy_high_school_microeconomics": 0.5,
+      "mmlu_eval_accuracy_high_school_physics": 0.5294117647058824,
+      "mmlu_eval_accuracy_high_school_psychology": 0.7833333333333333,
+      "mmlu_eval_accuracy_high_school_statistics": 0.391304347826087,
+      "mmlu_eval_accuracy_high_school_us_history": 0.8181818181818182,
+      "mmlu_eval_accuracy_high_school_world_history": 0.5769230769230769,
+      "mmlu_eval_accuracy_human_aging": 0.7391304347826086,
+      "mmlu_eval_accuracy_human_sexuality": 0.25,
+      "mmlu_eval_accuracy_international_law": 0.7692307692307693,
+      "mmlu_eval_accuracy_jurisprudence": 0.45454545454545453,
+      "mmlu_eval_accuracy_logical_fallacies": 0.6666666666666666,
+      "mmlu_eval_accuracy_machine_learning": 0.2727272727272727,
+      "mmlu_eval_accuracy_management": 0.5454545454545454,
+      "mmlu_eval_accuracy_marketing": 0.84,
+      "mmlu_eval_accuracy_medical_genetics": 0.9090909090909091,
+      "mmlu_eval_accuracy_miscellaneous": 0.6511627906976745,
+      "mmlu_eval_accuracy_moral_disputes": 0.5526315789473685,
+      "mmlu_eval_accuracy_moral_scenarios": 0.27,
+      "mmlu_eval_accuracy_nutrition": 0.5757575757575758,
+      "mmlu_eval_accuracy_philosophy": 0.5,
+      "mmlu_eval_accuracy_prehistory": 0.4857142857142857,
+      "mmlu_eval_accuracy_professional_accounting": 0.41935483870967744,
+      "mmlu_eval_accuracy_professional_law": 0.3411764705882353,
+      "mmlu_eval_accuracy_professional_medicine": 0.5161290322580645,
+      "mmlu_eval_accuracy_professional_psychology": 0.5362318840579711,
+      "mmlu_eval_accuracy_public_relations": 0.5,
+      "mmlu_eval_accuracy_security_studies": 0.5185185185185185,
+      "mmlu_eval_accuracy_sociology": 0.7727272727272727,
+      "mmlu_eval_accuracy_us_foreign_policy": 0.9090909090909091,
+      "mmlu_eval_accuracy_virology": 0.5,
+      "mmlu_eval_accuracy_world_religions": 0.7894736842105263,
+      "mmlu_loss": 1.2089649017112374,
+      "step": 7400
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 0.0002,
+      "loss": 0.6739,
+      "step": 7410
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 0.0002,
+      "loss": 0.7322,
+      "step": 7420
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 0.0002,
+      "loss": 0.7059,
+      "step": 7430
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 0.0002,
+      "loss": 0.7883,
+      "step": 7440
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 0.0002,
+      "loss": 0.817,
+      "step": 7450
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 0.0002,
+      "loss": 0.8002,
+      "step": 7460
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 0.0002,
+      "loss": 0.7203,
+      "step": 7470
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 0.0002,
+      "loss": 0.6422,
+      "step": 7480
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 0.0002,
+      "loss": 0.6782,
+      "step": 7490
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 0.0002,
+      "loss": 0.6946,
+      "step": 7500
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 0.0002,
+      "loss": 0.6521,
+      "step": 7510
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 0.0002,
+      "loss": 0.7206,
+      "step": 7520
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 0.0002,
+      "loss": 0.6546,
+      "step": 7530
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 0.0002,
+      "loss": 0.7544,
+      "step": 7540
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 0.0002,
+      "loss": 0.6293,
+      "step": 7550
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 0.0002,
+      "loss": 0.6536,
+      "step": 7560
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 0.0002,
+      "loss": 0.6702,
+      "step": 7570
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 0.0002,
+      "loss": 0.8209,
+      "step": 7580
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 0.0002,
+      "loss": 0.7015,
+      "step": 7590
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 0.0002,
+      "loss": 0.6716,
+      "step": 7600
+    },
+    {
+      "epoch": 0.85,
+      "eval_loss": 0.7208623886108398,
+      "eval_runtime": 133.8399,
+      "eval_samples_per_second": 7.472,
+      "eval_steps_per_second": 3.736,
+      "step": 7600
+    },
+    {
+      "epoch": 0.85,
+      "mmlu_eval_accuracy": 0.5215281301514161,
+      "mmlu_eval_accuracy_abstract_algebra": 0.36363636363636365,
+      "mmlu_eval_accuracy_anatomy": 0.5714285714285714,
+      "mmlu_eval_accuracy_astronomy": 0.5,
+      "mmlu_eval_accuracy_business_ethics": 0.6363636363636364,
+      "mmlu_eval_accuracy_clinical_knowledge": 0.5172413793103449,
+      "mmlu_eval_accuracy_college_biology": 0.4375,
+      "mmlu_eval_accuracy_college_chemistry": 0.125,
+      "mmlu_eval_accuracy_college_computer_science": 0.45454545454545453,
+      "mmlu_eval_accuracy_college_mathematics": 0.18181818181818182,
+      "mmlu_eval_accuracy_college_medicine": 0.5,
+      "mmlu_eval_accuracy_college_physics": 0.45454545454545453,
+      "mmlu_eval_accuracy_computer_security": 0.5454545454545454,
+      "mmlu_eval_accuracy_conceptual_physics": 0.4230769230769231,
+      "mmlu_eval_accuracy_econometrics": 0.4166666666666667,
+      "mmlu_eval_accuracy_electrical_engineering": 0.3125,
+      "mmlu_eval_accuracy_elementary_mathematics": 0.4146341463414634,
+      "mmlu_eval_accuracy_formal_logic": 0.2857142857142857,
+      "mmlu_eval_accuracy_global_facts": 0.6,
+      "mmlu_eval_accuracy_high_school_biology": 0.40625,
+      "mmlu_eval_accuracy_high_school_chemistry": 0.36363636363636365,
+      "mmlu_eval_accuracy_high_school_computer_science": 0.6666666666666666,
+      "mmlu_eval_accuracy_high_school_european_history": 0.6666666666666666,
+      "mmlu_eval_accuracy_high_school_geography": 0.9090909090909091,
+      "mmlu_eval_accuracy_high_school_government_and_politics": 0.5714285714285714,
+      "mmlu_eval_accuracy_high_school_macroeconomics": 0.3953488372093023,
+      "mmlu_eval_accuracy_high_school_mathematics": 0.1724137931034483,
+      "mmlu_eval_accuracy_high_school_microeconomics": 0.5384615384615384,
+      "mmlu_eval_accuracy_high_school_physics": 0.4117647058823529,
+      "mmlu_eval_accuracy_high_school_psychology": 0.85,
+      "mmlu_eval_accuracy_high_school_statistics": 0.391304347826087,
+      "mmlu_eval_accuracy_high_school_us_history": 0.6363636363636364,
+      "mmlu_eval_accuracy_high_school_world_history": 0.6538461538461539,
+      "mmlu_eval_accuracy_human_aging": 0.6956521739130435,
+      "mmlu_eval_accuracy_human_sexuality": 0.4166666666666667,
+      "mmlu_eval_accuracy_international_law": 0.8461538461538461,
+      "mmlu_eval_accuracy_jurisprudence": 0.45454545454545453,
+      "mmlu_eval_accuracy_logical_fallacies": 0.6111111111111112,
+      "mmlu_eval_accuracy_machine_learning": 0.18181818181818182,
+      "mmlu_eval_accuracy_management": 0.6363636363636364,
+      "mmlu_eval_accuracy_marketing": 0.84,
+      "mmlu_eval_accuracy_medical_genetics": 1.0,
+      "mmlu_eval_accuracy_miscellaneous": 0.6511627906976745,
+      "mmlu_eval_accuracy_moral_disputes": 0.5526315789473685,
+      "mmlu_eval_accuracy_moral_scenarios": 0.29,
+      "mmlu_eval_accuracy_nutrition": 0.696969696969697,
+      "mmlu_eval_accuracy_philosophy": 0.5,
+      "mmlu_eval_accuracy_prehistory": 0.4857142857142857,
+      "mmlu_eval_accuracy_professional_accounting": 0.2903225806451613,
+      "mmlu_eval_accuracy_professional_law": 0.32941176470588235,
+      "mmlu_eval_accuracy_professional_medicine": 0.5161290322580645,
+      "mmlu_eval_accuracy_professional_psychology": 0.5797101449275363,
+      "mmlu_eval_accuracy_public_relations": 0.5,
+      "mmlu_eval_accuracy_security_studies": 0.4444444444444444,
+      "mmlu_eval_accuracy_sociology": 0.7272727272727273,
+      "mmlu_eval_accuracy_us_foreign_policy": 0.8181818181818182,
+      "mmlu_eval_accuracy_virology": 0.5,
+      "mmlu_eval_accuracy_world_religions": 0.7894736842105263,
+      "mmlu_loss": 1.0419133053046916,
+      "step": 7600
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 0.0002,
+      "loss": 0.6791,
+      "step": 7610
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 0.0002,
+      "loss": 0.7639,
+      "step": 7620
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 0.0002,
+      "loss": 0.7893,
+      "step": 7630
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 0.0002,
+      "loss": 0.7624,
+      "step": 7640
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 0.0002,
+      "loss": 0.7497,
+      "step": 7650
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 0.0002,
+      "loss": 0.683,
+      "step": 7660
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 0.0002,
+      "loss": 0.7794,
+      "step": 7670
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 0.0002,
+      "loss": 0.7462,
+      "step": 7680
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 0.0002,
+      "loss": 0.799,
+      "step": 7690
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 0.0002,
+      "loss": 0.6985,
+      "step": 7700
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 0.0002,
+      "loss": 0.688,
+      "step": 7710
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 0.0002,
+      "loss": 0.683,
+      "step": 7720
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 0.0002,
+      "loss": 0.6134,
+      "step": 7730
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 0.0002,
+      "loss": 0.7178,
+      "step": 7740
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 0.0002,
+      "loss": 0.7174,
+      "step": 7750
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 0.0002,
+      "loss": 0.772,
+      "step": 7760
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 0.0002,
+      "loss": 0.7135,
+      "step": 7770
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 0.0002,
+      "loss": 0.7751,
+      "step": 7780
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 0.0002,
+      "loss": 0.8658,
+      "step": 7790
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 0.0002,
+      "loss": 0.7098,
+      "step": 7800
+    },
+    {
+      "epoch": 0.87,
+      "eval_loss": 0.7206757664680481,
+      "eval_runtime": 133.9229,
+      "eval_samples_per_second": 7.467,
+      "eval_steps_per_second": 3.733,
+      "step": 7800
+    },
+    {
+      "epoch": 0.87,
+      "mmlu_eval_accuracy": 0.5147796098176309,
+      "mmlu_eval_accuracy_abstract_algebra": 0.36363636363636365,
+      "mmlu_eval_accuracy_anatomy": 0.5714285714285714,
+      "mmlu_eval_accuracy_astronomy": 0.5,
+      "mmlu_eval_accuracy_business_ethics": 0.45454545454545453,
+      "mmlu_eval_accuracy_clinical_knowledge": 0.4482758620689655,
+      "mmlu_eval_accuracy_college_biology": 0.5625,
+      "mmlu_eval_accuracy_college_chemistry": 0.125,
+      "mmlu_eval_accuracy_college_computer_science": 0.36363636363636365,
+      "mmlu_eval_accuracy_college_mathematics": 0.18181818181818182,
+      "mmlu_eval_accuracy_college_medicine": 0.4090909090909091,
+      "mmlu_eval_accuracy_college_physics": 0.36363636363636365,
+      "mmlu_eval_accuracy_computer_security": 0.6363636363636364,
+      "mmlu_eval_accuracy_conceptual_physics": 0.46153846153846156,
+      "mmlu_eval_accuracy_econometrics": 0.3333333333333333,
+      "mmlu_eval_accuracy_electrical_engineering": 0.1875,
+      "mmlu_eval_accuracy_elementary_mathematics": 0.3902439024390244,
+      "mmlu_eval_accuracy_formal_logic": 0.2857142857142857,
+      "mmlu_eval_accuracy_global_facts": 0.4,
+      "mmlu_eval_accuracy_high_school_biology": 0.40625,
+      "mmlu_eval_accuracy_high_school_chemistry": 0.4090909090909091,
+      "mmlu_eval_accuracy_high_school_computer_science": 0.6666666666666666,
+      "mmlu_eval_accuracy_high_school_european_history": 0.6666666666666666,
+      "mmlu_eval_accuracy_high_school_geography": 0.8181818181818182,
+      "mmlu_eval_accuracy_high_school_government_and_politics": 0.7142857142857143,
+      "mmlu_eval_accuracy_high_school_macroeconomics": 0.4186046511627907,
+      "mmlu_eval_accuracy_high_school_mathematics": 0.13793103448275862,
+      "mmlu_eval_accuracy_high_school_microeconomics": 0.6923076923076923,
+      "mmlu_eval_accuracy_high_school_physics": 0.35294117647058826,
+      "mmlu_eval_accuracy_high_school_psychology": 0.85,
+      "mmlu_eval_accuracy_high_school_statistics": 0.391304347826087,
+      "mmlu_eval_accuracy_high_school_us_history": 0.7272727272727273,
+      "mmlu_eval_accuracy_high_school_world_history": 0.7307692307692307,
+      "mmlu_eval_accuracy_human_aging": 0.6086956521739131,
+      "mmlu_eval_accuracy_human_sexuality": 0.3333333333333333,
+      "mmlu_eval_accuracy_international_law": 0.9230769230769231,
+      "mmlu_eval_accuracy_jurisprudence": 0.45454545454545453,
+      "mmlu_eval_accuracy_logical_fallacies": 0.6111111111111112,
+      "mmlu_eval_accuracy_machine_learning": 0.18181818181818182,
+      "mmlu_eval_accuracy_management": 0.6363636363636364,
+      "mmlu_eval_accuracy_marketing": 0.8,
+      "mmlu_eval_accuracy_medical_genetics": 0.8181818181818182,
+      "mmlu_eval_accuracy_miscellaneous": 0.6744186046511628,
+      "mmlu_eval_accuracy_moral_disputes": 0.47368421052631576,
+      "mmlu_eval_accuracy_moral_scenarios": 0.22,
+      "mmlu_eval_accuracy_nutrition": 0.6363636363636364,
+      "mmlu_eval_accuracy_philosophy": 0.5,
+      "mmlu_eval_accuracy_prehistory": 0.4857142857142857,
+      "mmlu_eval_accuracy_professional_accounting": 0.3870967741935484,
+      "mmlu_eval_accuracy_professional_law": 0.37058823529411766,
+      "mmlu_eval_accuracy_professional_medicine": 0.6129032258064516,
+      "mmlu_eval_accuracy_professional_psychology": 0.5507246376811594,
+      "mmlu_eval_accuracy_public_relations": 0.5833333333333334,
+      "mmlu_eval_accuracy_security_studies": 0.48148148148148145,
+      "mmlu_eval_accuracy_sociology": 0.7272727272727273,
+      "mmlu_eval_accuracy_us_foreign_policy": 0.9090909090909091,
+      "mmlu_eval_accuracy_virology": 0.5,
+      "mmlu_eval_accuracy_world_religions": 0.8421052631578947,
+      "mmlu_loss": 0.9636794671455189,
+      "step": 7800
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 0.0002,
+      "loss": 0.7146,
+      "step": 7810
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 0.0002,
+      "loss": 0.7836,
+      "step": 7820
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 0.0002,
+      "loss": 0.6701,
+      "step": 7830
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 0.0002,
+      "loss": 0.6226,
+      "step": 7840
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 0.0002,
+      "loss": 0.7832,
+      "step": 7850
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 0.0002,
+      "loss": 0.7421,
+      "step": 7860
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 0.0002,
+      "loss": 0.8075,
+      "step": 7870
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 0.0002,
+      "loss": 0.713,
+      "step": 7880
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 0.0002,
+      "loss": 0.7896,
+      "step": 7890
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 0.0002,
+      "loss": 0.7236,
+      "step": 7900
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 0.0002,
+      "loss": 0.8057,
+      "step": 7910
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 0.0002,
+      "loss": 0.6468,
+      "step": 7920
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 0.0002,
+      "loss": 0.7553,
+      "step": 7930
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 0.0002,
+      "loss": 0.7463,
+      "step": 7940
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 0.0002,
+      "loss": 0.7863,
+      "step": 7950
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 0.0002,
+      "loss": 0.6481,
+      "step": 7960
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 0.0002,
+      "loss": 0.7012,
+      "step": 7970
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 0.0002,
+      "loss": 0.7489,
+      "step": 7980
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 0.0002,
+      "loss": 0.6534,
+      "step": 7990
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 0.0002,
+      "loss": 0.7703,
+      "step": 8000
+    },
+    {
+      "epoch": 0.89,
+      "eval_loss": 0.7257216572761536,
+      "eval_runtime": 133.8997,
+      "eval_samples_per_second": 7.468,
+      "eval_steps_per_second": 3.734,
+      "step": 8000
+    },
+    {
+      "epoch": 0.89,
+      "mmlu_eval_accuracy": 0.4968322236943019,
+      "mmlu_eval_accuracy_abstract_algebra": 0.18181818181818182,
+      "mmlu_eval_accuracy_anatomy": 0.7857142857142857,
+      "mmlu_eval_accuracy_astronomy": 0.5625,
+      "mmlu_eval_accuracy_business_ethics": 0.6363636363636364,
+      "mmlu_eval_accuracy_clinical_knowledge": 0.41379310344827586,
+      "mmlu_eval_accuracy_college_biology": 0.375,
+      "mmlu_eval_accuracy_college_chemistry": 0.125,
+      "mmlu_eval_accuracy_college_computer_science": 0.2727272727272727,
+      "mmlu_eval_accuracy_college_mathematics": 0.45454545454545453,
+      "mmlu_eval_accuracy_college_medicine": 0.36363636363636365,
+      "mmlu_eval_accuracy_college_physics": 0.2727272727272727,
+      "mmlu_eval_accuracy_computer_security": 0.36363636363636365,
+      "mmlu_eval_accuracy_conceptual_physics": 0.5384615384615384,
+      "mmlu_eval_accuracy_econometrics": 0.25,
+      "mmlu_eval_accuracy_electrical_engineering": 0.375,
+      "mmlu_eval_accuracy_elementary_mathematics": 0.2926829268292683,
+      "mmlu_eval_accuracy_formal_logic": 0.14285714285714285,
+      "mmlu_eval_accuracy_global_facts": 0.3,
+      "mmlu_eval_accuracy_high_school_biology": 0.375,
+      "mmlu_eval_accuracy_high_school_chemistry": 0.4090909090909091,
+      "mmlu_eval_accuracy_high_school_computer_science": 0.6666666666666666,
+      "mmlu_eval_accuracy_high_school_european_history": 0.6111111111111112,
+      "mmlu_eval_accuracy_high_school_geography": 0.8181818181818182,
+      "mmlu_eval_accuracy_high_school_government_and_politics": 0.5714285714285714,
+      "mmlu_eval_accuracy_high_school_macroeconomics": 0.4186046511627907,
+      "mmlu_eval_accuracy_high_school_mathematics": 0.3448275862068966,
+      "mmlu_eval_accuracy_high_school_microeconomics": 0.3076923076923077,
+      "mmlu_eval_accuracy_high_school_physics": 0.35294117647058826,
+      "mmlu_eval_accuracy_high_school_psychology": 0.7166666666666667,
+      "mmlu_eval_accuracy_high_school_statistics": 0.391304347826087,
+      "mmlu_eval_accuracy_high_school_us_history": 0.6818181818181818,
+      "mmlu_eval_accuracy_high_school_world_history": 0.5769230769230769,
+      "mmlu_eval_accuracy_human_aging": 0.782608695652174,
+      "mmlu_eval_accuracy_human_sexuality": 0.5,
+      "mmlu_eval_accuracy_international_law": 0.7692307692307693,
+      "mmlu_eval_accuracy_jurisprudence": 0.2727272727272727,
+      "mmlu_eval_accuracy_logical_fallacies": 0.5555555555555556,
+      "mmlu_eval_accuracy_machine_learning": 0.2727272727272727,
+      "mmlu_eval_accuracy_management": 0.6363636363636364,
+      "mmlu_eval_accuracy_marketing": 0.92,
+      "mmlu_eval_accuracy_medical_genetics": 1.0,
+      "mmlu_eval_accuracy_miscellaneous": 0.6744186046511628,
+      "mmlu_eval_accuracy_moral_disputes": 0.4473684210526316,
+      "mmlu_eval_accuracy_moral_scenarios": 0.26,
+      "mmlu_eval_accuracy_nutrition": 0.6666666666666666,
+      "mmlu_eval_accuracy_philosophy": 0.5294117647058824,
+      "mmlu_eval_accuracy_prehistory": 0.45714285714285713,
+      "mmlu_eval_accuracy_professional_accounting": 0.41935483870967744,
+      "mmlu_eval_accuracy_professional_law": 0.3176470588235294,
+      "mmlu_eval_accuracy_professional_medicine": 0.5161290322580645,
+      "mmlu_eval_accuracy_professional_psychology": 0.5072463768115942,
+      "mmlu_eval_accuracy_public_relations": 0.5,
+      "mmlu_eval_accuracy_security_studies": 0.48148148148148145,
+      "mmlu_eval_accuracy_sociology": 0.8181818181818182,
+      "mmlu_eval_accuracy_us_foreign_policy": 0.7272727272727273,
+      "mmlu_eval_accuracy_virology": 0.4444444444444444,
+      "mmlu_eval_accuracy_world_religions": 0.8947368421052632,
+      "mmlu_loss": 1.2022641924825102,
+      "step": 8000
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 0.0002,
+      "loss": 0.7554,
+      "step": 8010
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 0.0002,
+      "loss": 0.6063,
+      "step": 8020
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 0.0002,
+      "loss": 0.6717,
+      "step": 8030
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 0.0002,
+      "loss": 0.681,
+      "step": 8040
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 0.0002,
+      "loss": 0.7477,
+      "step": 8050
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 0.0002,
+      "loss": 0.6894,
+      "step": 8060
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 0.0002,
+      "loss": 0.6284,
+      "step": 8070
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 0.0002,
+      "loss": 0.8124,
+      "step": 8080
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 0.0002,
+      "loss": 0.7501,
+      "step": 8090
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 0.0002,
+      "loss": 0.8261,
+      "step": 8100
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 0.0002,
+      "loss": 0.8236,
+      "step": 8110
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 0.0002,
+      "loss": 0.6545,
+      "step": 8120
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 0.0002,
+      "loss": 0.6958,
+      "step": 8130
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 0.0002,
+      "loss": 0.7055,
+      "step": 8140
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 0.0002,
+      "loss": 0.6359,
+      "step": 8150
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 0.0002,
+      "loss": 0.6509,
+      "step": 8160
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 0.0002,
+      "loss": 0.8168,
+      "step": 8170
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 0.0002,
+      "loss": 0.6494,
+      "step": 8180
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 0.0002,
+      "loss": 0.7379,
+      "step": 8190
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 0.0002,
+      "loss": 0.7671,
+      "step": 8200
+    },
+    {
+      "epoch": 0.92,
+      "eval_loss": 0.7237617373466492,
+      "eval_runtime": 133.8533,
+      "eval_samples_per_second": 7.471,
+      "eval_steps_per_second": 3.735,
+      "step": 8200
+    },
+    {
+      "epoch": 0.92,
+      "mmlu_eval_accuracy": 0.5125238323825849,
+      "mmlu_eval_accuracy_abstract_algebra": 0.36363636363636365,
+      "mmlu_eval_accuracy_anatomy": 0.7142857142857143,
+      "mmlu_eval_accuracy_astronomy": 0.5,
+      "mmlu_eval_accuracy_business_ethics": 0.45454545454545453,
+      "mmlu_eval_accuracy_clinical_knowledge": 0.5862068965517241,
+      "mmlu_eval_accuracy_college_biology": 0.4375,
+      "mmlu_eval_accuracy_college_chemistry": 0.125,
+      "mmlu_eval_accuracy_college_computer_science": 0.36363636363636365,
+      "mmlu_eval_accuracy_college_mathematics": 0.2727272727272727,
+      "mmlu_eval_accuracy_college_medicine": 0.5454545454545454,
+      "mmlu_eval_accuracy_college_physics": 0.36363636363636365,
+      "mmlu_eval_accuracy_computer_security": 0.5454545454545454,
+      "mmlu_eval_accuracy_conceptual_physics": 0.5769230769230769,
+      "mmlu_eval_accuracy_econometrics": 0.25,
+      "mmlu_eval_accuracy_electrical_engineering": 0.375,
+      "mmlu_eval_accuracy_elementary_mathematics": 0.2926829268292683,
+      "mmlu_eval_accuracy_formal_logic": 0.14285714285714285,
+      "mmlu_eval_accuracy_global_facts": 0.6,
+      "mmlu_eval_accuracy_high_school_biology": 0.46875,
+      "mmlu_eval_accuracy_high_school_chemistry": 0.36363636363636365,
+      "mmlu_eval_accuracy_high_school_computer_science": 0.6666666666666666,
+      "mmlu_eval_accuracy_high_school_european_history": 0.6111111111111112,
+      "mmlu_eval_accuracy_high_school_geography": 0.7272727272727273,
+      "mmlu_eval_accuracy_high_school_government_and_politics": 0.5714285714285714,
+      "mmlu_eval_accuracy_high_school_macroeconomics": 0.3953488372093023,
+      "mmlu_eval_accuracy_high_school_mathematics": 0.41379310344827586,
+      "mmlu_eval_accuracy_high_school_microeconomics": 0.6153846153846154,
+      "mmlu_eval_accuracy_high_school_physics": 0.47058823529411764,
+      "mmlu_eval_accuracy_high_school_psychology": 0.8,
+      "mmlu_eval_accuracy_high_school_statistics": 0.43478260869565216,
+      "mmlu_eval_accuracy_high_school_us_history": 0.6363636363636364,
+      "mmlu_eval_accuracy_high_school_world_history": 0.6153846153846154,
+      "mmlu_eval_accuracy_human_aging": 0.6086956521739131,
+      "mmlu_eval_accuracy_human_sexuality": 0.5,
+      "mmlu_eval_accuracy_international_law": 0.8461538461538461,
+      "mmlu_eval_accuracy_jurisprudence": 0.2727272727272727,
+      "mmlu_eval_accuracy_logical_fallacies": 0.6666666666666666,
+      "mmlu_eval_accuracy_machine_learning": 0.18181818181818182,
+      "mmlu_eval_accuracy_management": 0.7272727272727273,
+      "mmlu_eval_accuracy_marketing": 0.8,
+      "mmlu_eval_accuracy_medical_genetics": 0.8181818181818182,
+      "mmlu_eval_accuracy_miscellaneous": 0.6627906976744186,
+      "mmlu_eval_accuracy_moral_disputes": 0.42105263157894735,
+      "mmlu_eval_accuracy_moral_scenarios": 0.29,
+      "mmlu_eval_accuracy_nutrition": 0.6666666666666666,
+      "mmlu_eval_accuracy_philosophy": 0.4411764705882353,
+      "mmlu_eval_accuracy_prehistory": 0.5714285714285714,
+      "mmlu_eval_accuracy_professional_accounting": 0.3225806451612903,
+      "mmlu_eval_accuracy_professional_law": 0.36470588235294116,
+      "mmlu_eval_accuracy_professional_medicine": 0.45161290322580644,
+      "mmlu_eval_accuracy_professional_psychology": 0.5217391304347826,
+      "mmlu_eval_accuracy_public_relations": 0.4166666666666667,
+      "mmlu_eval_accuracy_security_studies": 0.48148148148148145,
+      "mmlu_eval_accuracy_sociology": 0.8636363636363636,
+      "mmlu_eval_accuracy_us_foreign_policy": 0.7272727272727273,
+      "mmlu_eval_accuracy_virology": 0.5,
+      "mmlu_eval_accuracy_world_religions": 0.7894736842105263,
+      "mmlu_loss": 0.9977992222919165,
+      "step": 8200
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 0.0002,
+      "loss": 0.6726,
+      "step": 8210
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 0.0002,
+      "loss": 0.7613,
+      "step": 8220
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 0.0002,
+      "loss": 0.7592,
+      "step": 8230
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 0.0002,
+      "loss": 0.7735,
+      "step": 8240
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 0.0002,
+      "loss": 0.7698,
+      "step": 8250
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 0.0002,
+      "loss": 0.6664,
+      "step": 8260
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 0.0002,
+      "loss": 0.6488,
+      "step": 8270
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 0.0002,
+      "loss": 0.7968,
+      "step": 8280
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 0.0002,
+      "loss": 0.7509,
+      "step": 8290
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 0.0002,
+      "loss": 0.713,
+      "step": 8300
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 0.0002,
+      "loss": 0.7517,
+      "step": 8310
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 0.0002,
+      "loss": 0.6979,
+      "step": 8320
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 0.0002,
+      "loss": 0.6627,
+      "step": 8330
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 0.0002,
+      "loss": 0.6101,
+      "step": 8340
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 0.0002,
+      "loss": 0.7809,
+      "step": 8350
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 0.0002,
+      "loss": 0.696,
+      "step": 8360
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 0.0002,
+      "loss": 0.7072,
+      "step": 8370
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 0.0002,
+      "loss": 0.6693,
+      "step": 8380
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 0.0002,
+      "loss": 0.6619,
+      "step": 8390
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 0.0002,
+      "loss": 0.7257,
+      "step": 8400
+    },
+    {
+      "epoch": 0.94,
+      "eval_loss": 0.7216758131980896,
+      "eval_runtime": 133.7373,
+      "eval_samples_per_second": 7.477,
+      "eval_steps_per_second": 3.739,
+      "step": 8400
+    },
+    {
+      "epoch": 0.94,
+      "mmlu_eval_accuracy": 0.526984584707121,
+      "mmlu_eval_accuracy_abstract_algebra": 0.45454545454545453,
+      "mmlu_eval_accuracy_anatomy": 0.5714285714285714,
+      "mmlu_eval_accuracy_astronomy": 0.4375,
+      "mmlu_eval_accuracy_business_ethics": 0.5454545454545454,
+      "mmlu_eval_accuracy_clinical_knowledge": 0.5172413793103449,
+      "mmlu_eval_accuracy_college_biology": 0.5,
+      "mmlu_eval_accuracy_college_chemistry": 0.125,
+      "mmlu_eval_accuracy_college_computer_science": 0.36363636363636365,
+      "mmlu_eval_accuracy_college_mathematics": 0.2727272727272727,
+      "mmlu_eval_accuracy_college_medicine": 0.45454545454545453,
+      "mmlu_eval_accuracy_college_physics": 0.45454545454545453,
+      "mmlu_eval_accuracy_computer_security": 0.5454545454545454,
+      "mmlu_eval_accuracy_conceptual_physics": 0.4230769230769231,
+      "mmlu_eval_accuracy_econometrics": 0.25,
+      "mmlu_eval_accuracy_electrical_engineering": 0.1875,
+      "mmlu_eval_accuracy_elementary_mathematics": 0.34146341463414637,
+      "mmlu_eval_accuracy_formal_logic": 0.21428571428571427,
+      "mmlu_eval_accuracy_global_facts": 0.6,
+      "mmlu_eval_accuracy_high_school_biology": 0.46875,
+      "mmlu_eval_accuracy_high_school_chemistry": 0.4090909090909091,
+      "mmlu_eval_accuracy_high_school_computer_science": 0.6666666666666666,
+      "mmlu_eval_accuracy_high_school_european_history": 0.7222222222222222,
+      "mmlu_eval_accuracy_high_school_geography": 0.7727272727272727,
+      "mmlu_eval_accuracy_high_school_government_and_politics": 0.6666666666666666,
+      "mmlu_eval_accuracy_high_school_macroeconomics": 0.46511627906976744,
+      "mmlu_eval_accuracy_high_school_mathematics": 0.2413793103448276,
+      "mmlu_eval_accuracy_high_school_microeconomics": 0.5384615384615384,
+      "mmlu_eval_accuracy_high_school_physics": 0.4117647058823529,
+      "mmlu_eval_accuracy_high_school_psychology": 0.8,
+      "mmlu_eval_accuracy_high_school_statistics": 0.5217391304347826,
+      "mmlu_eval_accuracy_high_school_us_history": 0.7727272727272727,
+      "mmlu_eval_accuracy_high_school_world_history": 0.7307692307692307,
+      "mmlu_eval_accuracy_human_aging": 0.6521739130434783,
+      "mmlu_eval_accuracy_human_sexuality": 0.4166666666666667,
+      "mmlu_eval_accuracy_international_law": 0.9230769230769231,
+      "mmlu_eval_accuracy_jurisprudence": 0.45454545454545453,
+      "mmlu_eval_accuracy_logical_fallacies": 0.6666666666666666,
+      "mmlu_eval_accuracy_machine_learning": 0.36363636363636365,
+      "mmlu_eval_accuracy_management": 0.6363636363636364,
+      "mmlu_eval_accuracy_marketing": 0.8,
+      "mmlu_eval_accuracy_medical_genetics": 0.8181818181818182,
+      "mmlu_eval_accuracy_miscellaneous": 0.686046511627907,
+      "mmlu_eval_accuracy_moral_disputes": 0.5263157894736842,
+      "mmlu_eval_accuracy_moral_scenarios": 0.22,
+      "mmlu_eval_accuracy_nutrition": 0.6060606060606061,
+      "mmlu_eval_accuracy_philosophy": 0.5,
+      "mmlu_eval_accuracy_prehistory": 0.6285714285714286,
+      "mmlu_eval_accuracy_professional_accounting": 0.41935483870967744,
+      "mmlu_eval_accuracy_professional_law": 0.4176470588235294,
+      "mmlu_eval_accuracy_professional_medicine": 0.5161290322580645,
+      "mmlu_eval_accuracy_professional_psychology": 0.5507246376811594,
+      "mmlu_eval_accuracy_public_relations": 0.5,
+      "mmlu_eval_accuracy_security_studies": 0.5555555555555556,
+      "mmlu_eval_accuracy_sociology": 0.7727272727272727,
+      "mmlu_eval_accuracy_us_foreign_policy": 0.7272727272727273,
+      "mmlu_eval_accuracy_virology": 0.4444444444444444,
+      "mmlu_eval_accuracy_world_religions": 0.7894736842105263,
+      "mmlu_loss": 1.0874772744530485,
+      "step": 8400
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 0.0002,
+      "loss": 0.741,
+      "step": 8410
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 0.0002,
+      "loss": 0.7208,
+      "step": 8420
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 0.0002,
+      "loss": 0.7545,
+      "step": 8430
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 0.0002,
+      "loss": 0.6342,
+      "step": 8440
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 0.0002,
+      "loss": 0.6757,
+      "step": 8450
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 0.0002,
+      "loss": 0.6329,
+      "step": 8460
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 0.0002,
+      "loss": 0.6294,
+      "step": 8470
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 0.0002,
+      "loss": 0.7249,
+      "step": 8480
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 0.0002,
+      "loss": 0.748,
+      "step": 8490
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 0.0002,
+      "loss": 0.7092,
+      "step": 8500
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 0.0002,
+      "loss": 0.6631,
+      "step": 8510
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 0.0002,
+      "loss": 0.7569,
+      "step": 8520
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 0.0002,
+      "loss": 0.7655,
+      "step": 8530
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 0.0002,
+      "loss": 0.6968,
+      "step": 8540
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 0.0002,
+      "loss": 0.6705,
+      "step": 8550
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 0.0002,
+      "loss": 0.6807,
+      "step": 8560
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 0.0002,
+      "loss": 0.7453,
+      "step": 8570
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 0.0002,
+      "loss": 0.6911,
+      "step": 8580
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 0.0002,
+      "loss": 0.6233,
+      "step": 8590
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 0.0002,
+      "loss": 0.7262,
+      "step": 8600
+    },
+    {
+      "epoch": 0.96,
+      "eval_loss": 0.7232338786125183,
+      "eval_runtime": 133.8059,
+      "eval_samples_per_second": 7.474,
+      "eval_steps_per_second": 3.737,
+      "step": 8600
+    },
+    {
+      "epoch": 0.96,
+      "mmlu_eval_accuracy": 0.5137482992996992,
+      "mmlu_eval_accuracy_abstract_algebra": 0.45454545454545453,
+      "mmlu_eval_accuracy_anatomy": 0.6428571428571429,
+      "mmlu_eval_accuracy_astronomy": 0.5625,
+      "mmlu_eval_accuracy_business_ethics": 0.6363636363636364,
+      "mmlu_eval_accuracy_clinical_knowledge": 0.4827586206896552,
+      "mmlu_eval_accuracy_college_biology": 0.375,
+      "mmlu_eval_accuracy_college_chemistry": 0.25,
+      "mmlu_eval_accuracy_college_computer_science": 0.45454545454545453,
+      "mmlu_eval_accuracy_college_mathematics": 0.36363636363636365,
+      "mmlu_eval_accuracy_college_medicine": 0.4090909090909091,
+      "mmlu_eval_accuracy_college_physics": 0.45454545454545453,
+      "mmlu_eval_accuracy_computer_security": 0.45454545454545453,
+      "mmlu_eval_accuracy_conceptual_physics": 0.6153846153846154,
+      "mmlu_eval_accuracy_econometrics": 0.16666666666666666,
+      "mmlu_eval_accuracy_electrical_engineering": 0.3125,
+      "mmlu_eval_accuracy_elementary_mathematics": 0.34146341463414637,
+      "mmlu_eval_accuracy_formal_logic": 0.21428571428571427,
+      "mmlu_eval_accuracy_global_facts": 0.5,
+      "mmlu_eval_accuracy_high_school_biology": 0.4375,
+      "mmlu_eval_accuracy_high_school_chemistry": 0.36363636363636365,
+      "mmlu_eval_accuracy_high_school_computer_science": 0.5555555555555556,
+      "mmlu_eval_accuracy_high_school_european_history": 0.6666666666666666,
+      "mmlu_eval_accuracy_high_school_geography": 0.8181818181818182,
+      "mmlu_eval_accuracy_high_school_government_and_politics": 0.6190476190476191,
+      "mmlu_eval_accuracy_high_school_macroeconomics": 0.4186046511627907,
+      "mmlu_eval_accuracy_high_school_mathematics": 0.27586206896551724,
+      "mmlu_eval_accuracy_high_school_microeconomics": 0.46153846153846156,
+      "mmlu_eval_accuracy_high_school_physics": 0.35294117647058826,
+      "mmlu_eval_accuracy_high_school_psychology": 0.7833333333333333,
+      "mmlu_eval_accuracy_high_school_statistics": 0.391304347826087,
+      "mmlu_eval_accuracy_high_school_us_history": 0.6818181818181818,
+      "mmlu_eval_accuracy_high_school_world_history": 0.6923076923076923,
+      "mmlu_eval_accuracy_human_aging": 0.6521739130434783,
+      "mmlu_eval_accuracy_human_sexuality": 0.4166666666666667,
+      "mmlu_eval_accuracy_international_law": 0.9230769230769231,
+      "mmlu_eval_accuracy_jurisprudence": 0.36363636363636365,
+      "mmlu_eval_accuracy_logical_fallacies": 0.6111111111111112,
+      "mmlu_eval_accuracy_machine_learning": 0.2727272727272727,
+      "mmlu_eval_accuracy_management": 0.5454545454545454,
+      "mmlu_eval_accuracy_marketing": 0.84,
+      "mmlu_eval_accuracy_medical_genetics": 0.8181818181818182,
+      "mmlu_eval_accuracy_miscellaneous": 0.6511627906976745,
+      "mmlu_eval_accuracy_moral_disputes": 0.47368421052631576,
+      "mmlu_eval_accuracy_moral_scenarios": 0.22,
+      "mmlu_eval_accuracy_nutrition": 0.6666666666666666,
+      "mmlu_eval_accuracy_philosophy": 0.47058823529411764,
+      "mmlu_eval_accuracy_prehistory": 0.5142857142857142,
+      "mmlu_eval_accuracy_professional_accounting": 0.4838709677419355,
+      "mmlu_eval_accuracy_professional_law": 0.3411764705882353,
+      "mmlu_eval_accuracy_professional_medicine": 0.4838709677419355,
+      "mmlu_eval_accuracy_professional_psychology": 0.5507246376811594,
+      "mmlu_eval_accuracy_public_relations": 0.4166666666666667,
+      "mmlu_eval_accuracy_security_studies": 0.48148148148148145,
+      "mmlu_eval_accuracy_sociology": 0.7727272727272727,
+      "mmlu_eval_accuracy_us_foreign_policy": 0.8181818181818182,
+      "mmlu_eval_accuracy_virology": 0.4444444444444444,
+      "mmlu_eval_accuracy_world_religions": 0.8421052631578947,
+      "mmlu_loss": 1.1483409754912473,
+      "step": 8600
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 0.0002,
+      "loss": 0.6649,
+      "step": 8610
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 0.0002,
+      "loss": 0.7852,
+      "step": 8620
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 0.0002,
+      "loss": 0.6614,
+      "step": 8630
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 0.0002,
+      "loss": 0.7296,
+      "step": 8640
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 0.0002,
+      "loss": 0.7971,
+      "step": 8650
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 0.0002,
+      "loss": 0.7291,
+      "step": 8660
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 0.0002,
+      "loss": 0.6949,
+      "step": 8670
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 0.0002,
+      "loss": 0.7349,
+      "step": 8680
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 0.0002,
+      "loss": 0.6495,
+      "step": 8690
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 0.0002,
+      "loss": 0.7464,
+      "step": 8700
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 0.0002,
+      "loss": 0.5793,
+      "step": 8710
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 0.0002,
+      "loss": 0.7244,
+      "step": 8720
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 0.0002,
+      "loss": 0.8785,
+      "step": 8730
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 0.0002,
+      "loss": 0.7174,
+      "step": 8740
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 0.0002,
+      "loss": 0.8183,
+      "step": 8750
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 0.0002,
+      "loss": 0.7137,
+      "step": 8760
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 0.0002,
+      "loss": 0.7427,
+      "step": 8770
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 0.0002,
+      "loss": 0.6908,
+      "step": 8780
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 0.0002,
+      "loss": 0.7299,
+      "step": 8790
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 0.0002,
+      "loss": 0.7078,
+      "step": 8800
+    },
+    {
+      "epoch": 0.98,
+      "eval_loss": 0.7242326736450195,
+      "eval_runtime": 133.88,
+      "eval_samples_per_second": 7.469,
+      "eval_steps_per_second": 3.735,
+      "step": 8800
+    },
+    {
+      "epoch": 0.98,
+      "mmlu_eval_accuracy": 0.47832951297756615,
+      "mmlu_eval_accuracy_abstract_algebra": 0.2727272727272727,
+      "mmlu_eval_accuracy_anatomy": 0.6428571428571429,
+      "mmlu_eval_accuracy_astronomy": 0.4375,
+      "mmlu_eval_accuracy_business_ethics": 0.45454545454545453,
+      "mmlu_eval_accuracy_clinical_knowledge": 0.4482758620689655,
+      "mmlu_eval_accuracy_college_biology": 0.5625,
+      "mmlu_eval_accuracy_college_chemistry": 0.0,
+      "mmlu_eval_accuracy_college_computer_science": 0.2727272727272727,
+      "mmlu_eval_accuracy_college_mathematics": 0.18181818181818182,
+      "mmlu_eval_accuracy_college_medicine": 0.45454545454545453,
+      "mmlu_eval_accuracy_college_physics": 0.2727272727272727,
+      "mmlu_eval_accuracy_computer_security": 0.45454545454545453,
+      "mmlu_eval_accuracy_conceptual_physics": 0.5,
+      "mmlu_eval_accuracy_econometrics": 0.08333333333333333,
+      "mmlu_eval_accuracy_electrical_engineering": 0.1875,
+      "mmlu_eval_accuracy_elementary_mathematics": 0.34146341463414637,
+      "mmlu_eval_accuracy_formal_logic": 0.07142857142857142,
+      "mmlu_eval_accuracy_global_facts": 0.4,
+      "mmlu_eval_accuracy_high_school_biology": 0.5,
+      "mmlu_eval_accuracy_high_school_chemistry": 0.3181818181818182,
+      "mmlu_eval_accuracy_high_school_computer_science": 0.6666666666666666,
+      "mmlu_eval_accuracy_high_school_european_history": 0.4444444444444444,
+      "mmlu_eval_accuracy_high_school_geography": 0.7727272727272727,
+      "mmlu_eval_accuracy_high_school_government_and_politics": 0.6190476190476191,
+      "mmlu_eval_accuracy_high_school_macroeconomics": 0.4186046511627907,
+      "mmlu_eval_accuracy_high_school_mathematics": 0.13793103448275862,
+      "mmlu_eval_accuracy_high_school_microeconomics": 0.5,
+      "mmlu_eval_accuracy_high_school_physics": 0.5294117647058824,
+      "mmlu_eval_accuracy_high_school_psychology": 0.8166666666666667,
+      "mmlu_eval_accuracy_high_school_statistics": 0.391304347826087,
+      "mmlu_eval_accuracy_high_school_us_history": 0.6818181818181818,
+      "mmlu_eval_accuracy_high_school_world_history": 0.6538461538461539,
+      "mmlu_eval_accuracy_human_aging": 0.6521739130434783,
+      "mmlu_eval_accuracy_human_sexuality": 0.4166666666666667,
+      "mmlu_eval_accuracy_international_law": 0.9230769230769231,
+      "mmlu_eval_accuracy_jurisprudence": 0.36363636363636365,
+      "mmlu_eval_accuracy_logical_fallacies": 0.5555555555555556,
+      "mmlu_eval_accuracy_machine_learning": 0.18181818181818182,
+      "mmlu_eval_accuracy_management": 0.5454545454545454,
+      "mmlu_eval_accuracy_marketing": 0.8,
+      "mmlu_eval_accuracy_medical_genetics": 0.8181818181818182,
+      "mmlu_eval_accuracy_miscellaneous": 0.686046511627907,
+      "mmlu_eval_accuracy_moral_disputes": 0.5789473684210527,
+      "mmlu_eval_accuracy_moral_scenarios": 0.22,
+      "mmlu_eval_accuracy_nutrition": 0.5454545454545454,
+      "mmlu_eval_accuracy_philosophy": 0.4117647058823529,
+      "mmlu_eval_accuracy_prehistory": 0.6285714285714286,
+      "mmlu_eval_accuracy_professional_accounting": 0.3225806451612903,
+      "mmlu_eval_accuracy_professional_law": 0.3588235294117647,
+      "mmlu_eval_accuracy_professional_medicine": 0.45161290322580644,
+      "mmlu_eval_accuracy_professional_psychology": 0.4782608695652174,
+      "mmlu_eval_accuracy_public_relations": 0.5833333333333334,
+      "mmlu_eval_accuracy_security_studies": 0.48148148148148145,
+      "mmlu_eval_accuracy_sociology": 0.7727272727272727,
+      "mmlu_eval_accuracy_us_foreign_policy": 0.8181818181818182,
+      "mmlu_eval_accuracy_virology": 0.4444444444444444,
+      "mmlu_eval_accuracy_world_religions": 0.7368421052631579,
+      "mmlu_loss": 1.19566303125872,
+      "step": 8800
     }
   ],
   "max_steps": 10000,
   "num_train_epochs": 2,
-  "total_flos": 6.548099673173852e+17,
+  "total_flos": 8.729415955939e+17,
   "trial_name": null,
   "trial_params": null
 }