Upload folder using huggingface_hub

adapter_config.json

@@ -30,13 +30,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
+    "o_proj",
     "gate_proj",
-    "k_proj",
-    "up_proj",
-    "down_proj",
     "q_proj",
-    "o_proj",
-    "v_proj"
+    "v_proj",
+    "down_proj",
+    "k_proj",
+    "up_proj"
   ],
   "target_parameters": null,
   "task_type": "CAUSAL_LM",

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8e8b9d18a741945d57ae9c50da19b27c7eae565676488b34040cc2ed3b73a710
+oid sha256:7680f780544af3904e7232d54a0ff4bc26477fe15049cb99453cb60e94b2f079
 size 36981856

optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6b34f6db49d513fcbf099dad7bf2cc4bb5d2188e1757be60413d2baa22cd7f39
+oid sha256:ed4c64e15dad7e171d2580c38ba6922a406dd5b02c87cf007ecb20e986582510
 size 74194619

ref/adapter_config.json

@@ -30,13 +30,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
+    "o_proj",
     "gate_proj",
-    "k_proj",
-    "up_proj",
-    "down_proj",
     "q_proj",
-    "o_proj",
-    "v_proj"
+    "v_proj",
+    "down_proj",
+    "k_proj",
+    "up_proj"
   ],
   "target_parameters": null,
   "task_type": "CAUSAL_LM",

ref/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2bd76acdb9568aa0eaa808e5fdf632aef392350216ede4f4af8e7daeb17647b8
+oid sha256:d696a715e6ccdd8eb59060020be0bd578738383f6e181a3fe76f362d014a253b
 size 73911112

rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f4a9f217e852f439efa6bd32fde98d6867f11aa6ea13ddc021ba10af6a0b0934
+oid sha256:2534e434cd5abbb8f7668d3eab0549db0ef95d6a797a3efa86b712e8e32266a7
 size 14645

trainer_state.json

@@ -12,7 +12,7 @@
     {
       "entropy": 1.6346618384122849,
       "epoch": 0.050314465408805034,
-      "grad_norm": 3.6875,
+      "grad_norm": 3.734375,
       "learning_rate": 5e-05,
       "logits/chosen": -0.7411658949470903,
       "logits/rejected": -0.20507352810005436,
@@ -28,1065 +28,1065 @@
       "step": 1
     },
     {
-      "entropy": 1.4591333270072937,
+      "entropy": 1.4584085196256638,
       "epoch": 0.10062893081761007,
-      "grad_norm": 3.375,
+      "grad_norm": 3.40625,
       "learning_rate": 4.9166666666666665e-05,
-      "logits/chosen": -0.9232633579440521,
-      "logits/rejected": -0.42599087972727157,
-      "logps/chosen": -153.28020477294922,
-      "logps/rejected": -180.01216506958008,
-      "loss": 0.5030958652496338,
+      "logits/chosen": -0.9251237438184291,
+      "logits/rejected": -0.42817166651451355,
+      "logps/chosen": -153.27350616455078,
+      "logps/rejected": -180.06759071350098,
+      "loss": 0.5006560683250427,
       "mean_token_accuracy": 0.583746887743473,
       "num_tokens": 12841.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.22939472552388906,
-      "rewards/margins": 0.42953281849622726,
-      "rewards/rejected": -0.20013809762895107,
+      "rewards/chosen": 0.23006458766758442,
+      "rewards/margins": 0.4357452392578125,
+      "rewards/rejected": -0.20568065904080868,
       "step": 2
     },
     {
-      "entropy": 1.4469595104455948,
+      "entropy": 1.4456398040056229,
       "epoch": 0.1509433962264151,
-      "grad_norm": 2.59375,
+      "grad_norm": 2.609375,
       "learning_rate": 4.8333333333333334e-05,
-      "logits/chosen": -1.0695778754695249,
-      "logits/rejected": -0.5926211969909361,
-      "logps/chosen": -146.45265579223633,
-      "logps/rejected": -187.52983283996582,
-      "loss": 0.38490599393844604,
+      "logits/chosen": -1.0732471831941308,
+      "logits/rejected": -0.5970107323243525,
+      "logps/chosen": -146.46685123443604,
+      "logps/rejected": -187.66933250427246,
+      "loss": 0.3808254599571228,
       "mean_token_accuracy": 0.6039908826351166,
       "num_tokens": 18816.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.34354821499437094,
-      "rewards/margins": 0.7708233185112476,
-      "rewards/rejected": -0.4272750951349735,
+      "rewards/chosen": 0.34212866611778736,
+      "rewards/margins": 0.7833537347614765,
+      "rewards/rejected": -0.4412250593304634,
       "step": 3
     },
     {
-      "entropy": 1.518604964017868,
+      "entropy": 1.5161051750183105,
       "epoch": 0.20125786163522014,
-      "grad_norm": 2.140625,
+      "grad_norm": 2.15625,
       "learning_rate": 4.75e-05,
-      "logits/chosen": -0.935088641030341,
-      "logits/rejected": -0.3929027095055847,
-      "logps/chosen": -118.26776504516602,
-      "logps/rejected": -181.97826480865479,
-      "loss": 0.34729430079460144,
+      "logits/chosen": -0.9412079692678976,
+      "logits/rejected": -0.4002159728435217,
+      "logps/chosen": -118.29372596740723,
+      "logps/rejected": -182.18238067626953,
+      "loss": 0.34213775396347046,
       "mean_token_accuracy": 0.5912635922431946,
       "num_tokens": 25522.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.3668609778396785,
-      "rewards/margins": 0.9369952641427517,
-      "rewards/rejected": -0.5701342783868313,
+      "rewards/chosen": 0.3642648714594543,
+      "rewards/margins": 0.954810731112957,
+      "rewards/rejected": -0.5905458554625511,
       "step": 4
     },
     {
-      "entropy": 1.436258003115654,
+      "entropy": 1.4330662935972214,
       "epoch": 0.25157232704402516,
-      "grad_norm": 1.7578125,
+      "grad_norm": 1.7421875,
       "learning_rate": 4.666666666666667e-05,
-      "logits/chosen": -1.230789349374424,
-      "logits/rejected": -0.6072302734593342,
-      "logps/chosen": -118.5812578201294,
-      "logps/rejected": -193.36974906921387,
-      "loss": 0.21708804368972778,
+      "logits/chosen": -1.2388932583587922,
+      "logits/rejected": -0.6178736694127079,
+      "logps/chosen": -118.57859134674072,
+      "logps/rejected": -193.721284866333,
+      "loss": 0.21044230461120605,
       "mean_token_accuracy": 0.5765212289988995,
       "num_tokens": 32576.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.4549223016947508,
-      "rewards/margins": 1.485500916838646,
-      "rewards/rejected": -1.0305786281824112,
+      "rewards/chosen": 0.45518894493579865,
+      "rewards/margins": 1.520921140909195,
+      "rewards/rejected": -1.065732218325138,
       "step": 5
     },
     {
-      "entropy": 1.4615814238786697,
+      "entropy": 1.4573922604322433,
       "epoch": 0.3018867924528302,
-      "grad_norm": 1.515625,
+      "grad_norm": 1.484375,
       "learning_rate": 4.5833333333333334e-05,
-      "logits/chosen": -1.2629337189976264,
-      "logits/rejected": -0.4917354064790751,
-      "logps/chosen": -120.08106422424316,
-      "logps/rejected": -203.18199729919434,
-      "loss": 0.1837618350982666,
+      "logits/chosen": -1.2737800530058234,
+      "logits/rejected": -0.5034645169102513,
+      "logps/chosen": -120.18647003173828,
+      "logps/rejected": -203.72421646118164,
+      "loss": 0.17699970304965973,
       "mean_token_accuracy": 0.5985999256372452,
       "num_tokens": 39703.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.42343464493751526,
-      "rewards/margins": 1.681808888912201,
-      "rewards/rejected": -1.2583742290735245,
+      "rewards/chosen": 0.41289406828582287,
+      "rewards/margins": 1.7254902124404907,
+      "rewards/rejected": -1.3125961422920227,
       "step": 6
     },
     {
-      "entropy": 1.303115427494049,
+      "entropy": 1.2998351603746414,
       "epoch": 0.3522012578616352,
-      "grad_norm": 1.5703125,
+      "grad_norm": 1.546875,
       "learning_rate": 4.5e-05,
-      "logits/chosen": -1.3031889189529362,
-      "logits/rejected": -0.5977883441101891,
-      "logps/chosen": -108.28875637054443,
-      "logps/rejected": -193.97905731201172,
-      "loss": 0.1922958791255951,
+      "logits/chosen": -1.3130657002820167,
+      "logits/rejected": -0.6094413558435448,
+      "logps/chosen": -108.3181095123291,
+      "logps/rejected": -194.44207382202148,
+      "loss": 0.18618769943714142,
       "mean_token_accuracy": 0.6349928826093674,
       "num_tokens": 47481.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.26323529286310077,
-      "rewards/margins": 1.6920193135738373,
-      "rewards/rejected": -1.428784005343914,
+      "rewards/chosen": 0.26029996760189533,
+      "rewards/margins": 1.7353856414556503,
+      "rewards/rejected": -1.4750856757164001,
       "step": 7
     },
     {
-      "entropy": 1.3591983914375305,
+      "entropy": 1.354924201965332,
       "epoch": 0.4025157232704403,
-      "grad_norm": 1.0234375,
+      "grad_norm": 1.015625,
       "learning_rate": 4.4166666666666665e-05,
-      "logits/chosen": -1.42433679119847,
-      "logits/rejected": -0.987506627964361,
-      "logps/chosen": -137.10870265960693,
-      "logps/rejected": -202.67957878112793,
-      "loss": 0.10500691831111908,
-      "mean_token_accuracy": 0.6057217642664909,
+      "logits/chosen": -1.437566920334179,
+      "logits/rejected": -1.0028015186250743,
+      "logps/chosen": -137.25361728668213,
+      "logps/rejected": -203.34245681762695,
+      "loss": 0.1016073077917099,
+      "mean_token_accuracy": 0.6037986874580383,
       "num_tokens": 54551.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.6465687770396471,
-      "rewards/margins": 2.516775831580162,
-      "rewards/rejected": -1.87020705640316,
+      "rewards/chosen": 0.6320773344486952,
+      "rewards/margins": 2.5685721784830093,
+      "rewards/rejected": -1.9364948570728302,
       "step": 8
     },
     {
-      "entropy": 1.305082157254219,
+      "entropy": 1.3007782101631165,
       "epoch": 0.4528301886792453,
-      "grad_norm": 0.70703125,
+      "grad_norm": 0.6796875,
       "learning_rate": 4.3333333333333334e-05,
-      "logits/chosen": -1.4197879173426369,
-      "logits/rejected": -0.9261144061875269,
-      "logps/chosen": -133.74537086486816,
-      "logps/rejected": -214.5505771636963,
-      "loss": 0.07105977088212967,
-      "mean_token_accuracy": 0.6148578971624374,
+      "logits/chosen": -1.4337730887270794,
+      "logits/rejected": -0.9420388615307833,
+      "logps/chosen": -133.88286685943604,
+      "logps/rejected": -215.5273036956787,
+      "loss": 0.06624128669500351,
+      "mean_token_accuracy": 0.6150126904249191,
       "num_tokens": 61082.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.7314652749337256,
-      "rewards/margins": 3.20961757004261,
-      "rewards/rejected": -2.4781522899866104,
+      "rewards/chosen": 0.7177156624384224,
+      "rewards/margins": 3.2935406416654587,
+      "rewards/rejected": -2.5758249759674072,
       "step": 9
     },
     {
-      "entropy": 1.3140722662210464,
+      "entropy": 1.3091522455215454,
       "epoch": 0.5031446540880503,
-      "grad_norm": 0.54296875,
+      "grad_norm": 0.515625,
       "learning_rate": 4.25e-05,
-      "logits/chosen": -1.7090569959939388,
-      "logits/rejected": -1.1499504201609827,
-      "logps/chosen": -114.76737594604492,
-      "logps/rejected": -215.3073024749756,
-      "loss": 0.04022533446550369,
+      "logits/chosen": -1.7237460889408442,
+      "logits/rejected": -1.1708205992219858,
+      "logps/chosen": -114.79162216186523,
+      "logps/rejected": -216.33137321472168,
+      "loss": 0.03747009485960007,
       "mean_token_accuracy": 0.5977272689342499,
       "num_tokens": 68063.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.7901867162436247,
-      "rewards/margins": 3.9837906509637833,
-      "rewards/rejected": -3.1936039328575134,
+      "rewards/chosen": 0.7877620831131935,
+      "rewards/margins": 4.083773195743561,
+      "rewards/rejected": -3.296011045575142,
       "step": 10
     },
     {
-      "entropy": 1.2443669438362122,
+      "entropy": 1.2404007613658905,
       "epoch": 0.5534591194968553,
-      "grad_norm": 0.2734375,
+      "grad_norm": 0.259765625,
       "learning_rate": 4.166666666666667e-05,
-      "logits/chosen": -1.6404496463752554,
-      "logits/rejected": -1.1740173366915783,
-      "logps/chosen": -136.84611701965332,
-      "logps/rejected": -222.02164459228516,
-      "loss": 0.02123747020959854,
-      "mean_token_accuracy": 0.6184160858392715,
+      "logits/chosen": -1.655923360923379,
+      "logits/rejected": -1.1929389227396723,
+      "logps/chosen": -137.04386043548584,
+      "logps/rejected": -223.17731285095215,
+      "loss": 0.019634969532489777,
+      "mean_token_accuracy": 0.615639328956604,
       "num_tokens": 73558.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 1.1639278389047831,
-      "rewards/margins": 4.731138825416565,
-      "rewards/rejected": -3.567211002111435,
+      "rewards/chosen": 1.1441535080084577,
+      "rewards/margins": 4.826931297779083,
+      "rewards/rejected": -3.6827778220176697,
       "step": 11
     },
     {
-      "entropy": 1.2719001024961472,
+      "entropy": 1.2675090283155441,
       "epoch": 0.6037735849056604,
-      "grad_norm": 0.177734375,
+      "grad_norm": 0.16796875,
       "learning_rate": 4.0833333333333334e-05,
-      "logits/chosen": -1.7234823190264654,
-      "logits/rejected": -1.2490433686033002,
-      "logps/chosen": -133.34080696105957,
-      "logps/rejected": -222.08237648010254,
-      "loss": 0.01416418794542551,
-      "mean_token_accuracy": 0.6046521663665771,
+      "logits/chosen": -1.738670325722853,
+      "logits/rejected": -1.2716987398570991,
+      "logps/chosen": -133.57194137573242,
+      "logps/rejected": -223.28443908691406,
+      "loss": 0.013121644034981728,
+      "mean_token_accuracy": 0.6074652224779129,
       "num_tokens": 79421.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 1.091646859422326,
-      "rewards/margins": 4.951511919498444,
-      "rewards/rejected": -3.8598650693893433,
+      "rewards/chosen": 1.0685334280133247,
+      "rewards/margins": 5.048604816198349,
+      "rewards/rejected": -3.980071395635605,
       "step": 12
     },
     {
-      "entropy": 1.2260268032550812,
+      "entropy": 1.221432313323021,
       "epoch": 0.6540880503144654,
-      "grad_norm": 0.4765625,
+      "grad_norm": 0.466796875,
       "learning_rate": 4e-05,
-      "logits/chosen": -1.8490552369674467,
-      "logits/rejected": -1.3316055711821662,
-      "logps/chosen": -130.4850263595581,
-      "logps/rejected": -238.74424743652344,
-      "loss": 0.031002508476376534,
-      "mean_token_accuracy": 0.623911663889885,
+      "logits/chosen": -1.8638311019017777,
+      "logits/rejected": -1.3540506586835501,
+      "logps/chosen": -130.8041524887085,
+      "logps/rejected": -240.23817443847656,
+      "loss": 0.029251903295516968,
+      "mean_token_accuracy": 0.6208610609173775,
       "num_tokens": 86682.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.5156277692876756,
-      "rewards/margins": 5.114428728818893,
-      "rewards/rejected": -4.598800882697105,
+      "rewards/chosen": 0.4837151520187035,
+      "rewards/margins": 5.231908708810806,
+      "rewards/rejected": -4.748193636536598,
       "step": 13
     },
     {
-      "entropy": 1.1846471428871155,
+      "entropy": 1.179955169558525,
       "epoch": 0.7044025157232704,
-      "grad_norm": 0.2734375,
+      "grad_norm": 0.263671875,
       "learning_rate": 3.9166666666666665e-05,
-      "logits/chosen": -1.6988998158408573,
-      "logits/rejected": -1.412297648550121,
-      "logps/chosen": -134.32691383361816,
-      "logps/rejected": -220.49063682556152,
-      "loss": 0.021130766719579697,
-      "mean_token_accuracy": 0.6293000131845474,
+      "logits/chosen": -1.714330251919895,
+      "logits/rejected": -1.4322710140173394,
+      "logps/chosen": -134.82413864135742,
+      "logps/rejected": -221.86817169189453,
+      "loss": 0.019698552787303925,
+      "mean_token_accuracy": 0.6337382346391678,
       "num_tokens": 93035.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.28157730400562286,
-      "rewards/margins": 4.996448755264282,
-      "rewards/rejected": -4.714871436357498,
+      "rewards/chosen": 0.23185482621192932,
+      "rewards/margins": 5.084479838609695,
+      "rewards/rejected": -4.852624952793121,
       "step": 14
     },
     {
-      "entropy": 1.1781003028154373,
+      "entropy": 1.1727432310581207,
       "epoch": 0.7547169811320755,
-      "grad_norm": 0.0517578125,
+      "grad_norm": 0.046142578125,
       "learning_rate": 3.8333333333333334e-05,
-      "logits/chosen": -1.8902387361462176,
-      "logits/rejected": -1.4900272791975393,
-      "logps/chosen": -132.44914436340332,
-      "logps/rejected": -247.65433311462402,
-      "loss": 0.002454916015267372,
-      "mean_token_accuracy": 0.6452845111489296,
+      "logits/chosen": -1.907136892939249,
+      "logits/rejected": -1.514815267146274,
+      "logps/chosen": -132.96124839782715,
+      "logps/rejected": -249.50230979919434,
+      "loss": 0.002145277801901102,
+      "mean_token_accuracy": 0.6479089632630348,
       "num_tokens": 98572.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 1.2324292734265327,
-      "rewards/margins": 7.052622675895691,
-      "rewards/rejected": -5.820193350315094,
+      "rewards/chosen": 1.1812188103795052,
+      "rewards/margins": 7.1862099170684814,
+      "rewards/rejected": -6.004990994930267,
       "step": 15
     },
     {
-      "entropy": 1.2133459448814392,
+      "entropy": 1.2092190980911255,
       "epoch": 0.8050314465408805,
-      "grad_norm": 0.20703125,
+      "grad_norm": 0.203125,
       "learning_rate": 3.7500000000000003e-05,
-      "logits/chosen": -1.8413987197790387,
-      "logits/rejected": -1.4087605133817744,
-      "logps/chosen": -139.3100233078003,
-      "logps/rejected": -246.31277465820312,
-      "loss": 0.01590706966817379,
-      "mean_token_accuracy": 0.6142129823565483,
+      "logits/chosen": -1.8553556408587935,
+      "logits/rejected": -1.4260795074385058,
+      "logps/chosen": -139.758376121521,
+      "logps/rejected": -248.12162590026855,
+      "loss": 0.015183546580374241,
+      "mean_token_accuracy": 0.6128971949219704,
       "num_tokens": 106014.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.3513742520008236,
-      "rewards/margins": 6.107073247432709,
-      "rewards/rejected": -5.755699068307877,
+      "rewards/chosen": 0.30653896857984364,
+      "rewards/margins": 6.243123233318329,
+      "rewards/rejected": -5.9365842044353485,
       "step": 16
     },
     {
-      "entropy": 1.2062125205993652,
+      "entropy": 1.2008470296859741,
       "epoch": 0.8553459119496856,
-      "grad_norm": 0.1396484375,
+      "grad_norm": 0.134765625,
       "learning_rate": 3.6666666666666666e-05,
-      "logits/chosen": -1.9615212533907105,
-      "logits/rejected": -1.3313654933874406,
-      "logps/chosen": -127.33739471435547,
-      "logps/rejected": -249.73816299438477,
-      "loss": 0.00578045891597867,
+      "logits/chosen": -1.9781900924423292,
+      "logits/rejected": -1.3513694447951352,
+      "logps/chosen": -127.7047929763794,
+      "logps/rejected": -251.46310424804688,
+      "loss": 0.005312850698828697,
       "mean_token_accuracy": 0.6195648014545441,
       "num_tokens": 112297.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.9851933747413568,
-      "rewards/margins": 7.103353261947632,
-      "rewards/rejected": -6.118159830570221,
+      "rewards/chosen": 0.948453530203551,
+      "rewards/margins": 7.2391074895858765,
+      "rewards/rejected": -6.290653884410858,
       "step": 17
     },
     {
-      "entropy": 1.198799878358841,
+      "entropy": 1.1938088834285736,
       "epoch": 0.9056603773584906,
-      "grad_norm": 0.030517578125,
+      "grad_norm": 0.0283203125,
       "learning_rate": 3.5833333333333335e-05,
-      "logits/chosen": -2.082073810115763,
-      "logits/rejected": -1.6705133301771156,
-      "logps/chosen": -128.168438911438,
-      "logps/rejected": -246.99494552612305,
-      "loss": 0.001798766665160656,
-      "mean_token_accuracy": 0.6067424267530441,
+      "logits/chosen": -2.0970170230360243,
+      "logits/rejected": -1.6893045219197895,
+      "logps/chosen": -128.71086406707764,
+      "logps/rejected": -248.6879997253418,
+      "loss": 0.0016415835125371814,
+      "mean_token_accuracy": 0.6066833287477493,
       "num_tokens": 118259.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.6913285218179226,
-      "rewards/margins": 7.232239246368408,
-      "rewards/rejected": -6.540910840034485,
+      "rewards/chosen": 0.6370860114693642,
+      "rewards/margins": 7.347302317619324,
+      "rewards/rejected": -6.710216283798218,
       "step": 18
     },
     {
-      "entropy": 1.1391501128673553,
+      "entropy": 1.1347751468420029,
       "epoch": 0.9559748427672956,
-      "grad_norm": 0.023193359375,
+      "grad_norm": 0.021240234375,
       "learning_rate": 3.5e-05,
-      "logits/chosen": -2.1592130772558074,
-      "logits/rejected": -1.7971208229649984,
-      "logps/chosen": -136.7722806930542,
-      "logps/rejected": -235.51027870178223,
-      "loss": 0.0014428942231461406,
-      "mean_token_accuracy": 0.6118658147752285,
+      "logits/chosen": -2.174784405399738,
+      "logits/rejected": -1.8171304190186441,
+      "logps/chosen": -137.3084011077881,
+      "logps/rejected": -237.20912742614746,
+      "loss": 0.0013000022154301405,
+      "mean_token_accuracy": 0.6105500273406506,
       "num_tokens": 123618.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 1.2391198351979256,
-      "rewards/margins": 7.5219069719314575,
-      "rewards/rejected": -6.282787263393402,
+      "rewards/chosen": 1.1855077669024467,
+      "rewards/margins": 7.6381800174713135,
+      "rewards/rejected": -6.452672183513641,
       "step": 19
     },
     {
-      "entropy": 1.2272680827549525,
+      "entropy": 1.2229801756995065,
       "epoch": 1.0,
-      "grad_norm": 0.037353515625,
+      "grad_norm": 0.034912109375,
       "learning_rate": 3.4166666666666666e-05,
-      "logits/chosen": -1.927751023508857,
-      "logits/rejected": -1.4586340540864533,
-      "logps/chosen": -140.12276676722936,
-      "logps/rejected": -256.6956220354353,
-      "loss": 0.0020336967427283525,
-      "mean_token_accuracy": 0.6004699298313686,
+      "logits/chosen": -1.9402861332224905,
+      "logits/rejected": -1.478508916635409,
+      "logps/chosen": -140.62159075055803,
+      "logps/rejected": -258.58056640625,
+      "loss": 0.0018078959546983242,
+      "mean_token_accuracy": 0.6002635615212577,
       "num_tokens": 130000.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.13402164301701955,
-      "rewards/margins": 6.993039471762521,
-      "rewards/rejected": -7.12706116267613,
+      "rewards/chosen": -0.18390404965196336,
+      "rewards/margins": 7.131651401519775,
+      "rewards/rejected": -7.31555564062936,
       "step": 20
     },
     {
-      "entropy": 1.1386558711528778,
+      "entropy": 1.1338201016187668,
       "epoch": 1.050314465408805,
-      "grad_norm": 0.00396728515625,
+      "grad_norm": 0.00341796875,
       "learning_rate": 3.3333333333333335e-05,
-      "logits/chosen": -2.222644827049755,
-      "logits/rejected": -1.7477538407871338,
-      "logps/chosen": -132.11598300933838,
-      "logps/rejected": -268.2633934020996,
-      "loss": 0.00020816404139623046,
-      "mean_token_accuracy": 0.6361734122037888,
+      "logits/chosen": -2.2381289981693855,
+      "logits/rejected": -1.7685196120349025,
+      "logps/chosen": -132.71670532226562,
+      "logps/rejected": -270.32165908813477,
+      "loss": 0.00017723625933285803,
+      "mean_token_accuracy": 0.640245221555233,
       "num_tokens": 135799.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 1.087248451076448,
-      "rewards/margins": 8.924234092235565,
-      "rewards/rejected": -7.8369855880737305,
+      "rewards/chosen": 1.0271762115880847,
+      "rewards/margins": 9.069988369941711,
+      "rewards/rejected": -8.04281210899353,
       "step": 21
     },
     {
-      "entropy": 1.188778430223465,
+      "entropy": 1.184071958065033,
       "epoch": 1.10062893081761,
-      "grad_norm": 0.06005859375,
+      "grad_norm": 0.06201171875,
       "learning_rate": 3.2500000000000004e-05,
-      "logits/chosen": -2.190930237663479,
-      "logits/rejected": -1.5339520761713392,
-      "logps/chosen": -136.57733154296875,
-      "logps/rejected": -276.55401611328125,
-      "loss": 0.0026830965653061867,
-      "mean_token_accuracy": 0.5980619117617607,
+      "logits/chosen": -2.2030959691219643,
+      "logits/rejected": -1.549622772314418,
+      "logps/chosen": -137.2278184890747,
+      "logps/rejected": -278.6941947937012,
+      "loss": 0.002708145882934332,
+      "mean_token_accuracy": 0.5984649807214737,
       "num_tokens": 142686.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.16770659349276684,
-      "rewards/margins": 8.182168662548065,
-      "rewards/rejected": -8.014461994171143,
+      "rewards/chosen": 0.10265790205448866,
+      "rewards/margins": 8.331138014793396,
+      "rewards/rejected": -8.228479981422424,
       "step": 22
     },
     {
-      "entropy": 1.1210491955280304,
+      "entropy": 1.1163400262594223,
       "epoch": 1.150943396226415,
-      "grad_norm": 0.01141357421875,
+      "grad_norm": 0.010498046875,
       "learning_rate": 3.1666666666666666e-05,
-      "logits/chosen": -2.2553734561327348,
-      "logits/rejected": -1.9669641015596422,
-      "logps/chosen": -121.1500768661499,
-      "logps/rejected": -249.81173515319824,
-      "loss": 0.0006848493940196931,
-      "mean_token_accuracy": 0.5800603404641151,
+      "logits/chosen": -2.2692944342560915,
+      "logits/rejected": -1.9815728330371412,
+      "logps/chosen": -121.65661811828613,
+      "logps/rejected": -251.65609550476074,
+      "loss": 0.0006153068970888853,
+      "mean_token_accuracy": 0.5803752839565277,
       "num_tokens": 148888.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.6980049340054393,
-      "rewards/margins": 8.123494029045105,
-      "rewards/rejected": -7.425489127635956,
+      "rewards/chosen": 0.6473507843911648,
+      "rewards/margins": 8.257275819778442,
+      "rewards/rejected": -7.609925091266632,
       "step": 23
     },
     {
-      "entropy": 1.2056218385696411,
+      "entropy": 1.2022811472415924,
       "epoch": 1.20125786163522,
-      "grad_norm": 0.0213623046875,
+      "grad_norm": 0.0196533203125,
       "learning_rate": 3.0833333333333335e-05,
-      "logits/chosen": -2.0574360723332568,
-      "logits/rejected": -1.5004657659813077,
-      "logps/chosen": -141.36074924468994,
-      "logps/rejected": -285.9758243560791,
-      "loss": 0.0011920351535081863,
-      "mean_token_accuracy": 0.6025349199771881,
+      "logits/chosen": -2.0684997205168574,
+      "logits/rejected": -1.5187979793505884,
+      "logps/chosen": -142.0221881866455,
+      "logps/rejected": -288.0162467956543,
+      "loss": 0.0010658408282324672,
+      "mean_token_accuracy": 0.6041584983468056,
       "num_tokens": 155066.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.3790994770824909,
-      "rewards/margins": 9.149889826774597,
-      "rewards/rejected": -8.77079039812088,
+      "rewards/chosen": 0.3129555657505989,
+      "rewards/margins": 9.287788093090057,
+      "rewards/rejected": -8.974832653999329,
       "step": 24
     },
     {
-      "entropy": 1.0760537832975388,
+      "entropy": 1.0715540498495102,
       "epoch": 1.251572327044025,
-      "grad_norm": 0.0164794921875,
+      "grad_norm": 0.0152587890625,
       "learning_rate": 3e-05,
-      "logits/chosen": -2.243240417111155,
-      "logits/rejected": -1.6917496542346284,
-      "logps/chosen": -123.56195449829102,
-      "logps/rejected": -266.39801025390625,
-      "loss": 0.0008477906812913716,
-      "mean_token_accuracy": 0.6462969928979874,
+      "logits/chosen": -2.253688589577229,
+      "logits/rejected": -1.709912522981535,
+      "logps/chosen": -124.24794864654541,
+      "logps/rejected": -268.31462478637695,
+      "loss": 0.0007783680921420455,
+      "mean_token_accuracy": 0.6487728357315063,
       "num_tokens": 161896.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.5940647050738335,
-      "rewards/margins": 8.513349890708923,
-      "rewards/rejected": -7.919285297393799,
+      "rewards/chosen": 0.5254653260344639,
+      "rewards/margins": 8.636412143707275,
+      "rewards/rejected": -8.110946834087372,
       "step": 25
     },
     {
-      "entropy": 1.1326584815979004,
+      "entropy": 1.1292345225811005,
       "epoch": 1.3018867924528301,
-      "grad_norm": 0.0177001953125,
+      "grad_norm": 0.01611328125,
       "learning_rate": 2.916666666666667e-05,
-      "logits/chosen": -2.025026339871026,
-      "logits/rejected": -1.6378500703330563,
-      "logps/chosen": -120.81675624847412,
-      "logps/rejected": -253.07993507385254,
-      "loss": 0.0010006717639043927,
-      "mean_token_accuracy": 0.6229780465364456,
+      "logits/chosen": -2.03905456801847,
+      "logits/rejected": -1.653255032290945,
+      "logps/chosen": -121.39779376983643,
+      "logps/rejected": -255.01618576049805,
+      "loss": 0.0008846853161230683,
+      "mean_token_accuracy": 0.622931219637394,
       "num_tokens": 168755.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.3348179815802723,
-      "rewards/margins": 7.985908567905426,
-      "rewards/rejected": -7.651090621948242,
+      "rewards/chosen": 0.2767142332158983,
+      "rewards/margins": 8.121429800987244,
+      "rewards/rejected": -7.844715654850006,
       "step": 26
     },
     {
-      "entropy": 1.1113788187503815,
+      "entropy": 1.107276238501072,
       "epoch": 1.3522012578616351,
-      "grad_norm": 0.00836181640625,
+      "grad_norm": 0.0076904296875,
       "learning_rate": 2.8333333333333335e-05,
-      "logits/chosen": -2.2742666534990894,
-      "logits/rejected": -1.9221175464707185,
-      "logps/chosen": -143.62575721740723,
-      "logps/rejected": -270.15936851501465,
-      "loss": 0.00037246744614094496,
-      "mean_token_accuracy": 0.6221201568841934,
+      "logits/chosen": -2.286326659906524,
+      "logits/rejected": -1.9377510490705079,
+      "logps/chosen": -144.30573749542236,
+      "logps/rejected": -272.37299728393555,
+      "loss": 0.00033011281630024314,
+      "mean_token_accuracy": 0.619342528283596,
       "num_tokens": 175381.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 1.0057550426572561,
-      "rewards/margins": 9.40854561328888,
-      "rewards/rejected": -8.402790665626526,
+      "rewards/chosen": 0.9377570524811745,
+      "rewards/margins": 9.561910688877106,
+      "rewards/rejected": -8.6241534948349,
       "step": 27
     },
     {
-      "entropy": 1.1610162109136581,
+      "entropy": 1.157460778951645,
       "epoch": 1.4025157232704402,
-      "grad_norm": 0.00518798828125,
+      "grad_norm": 0.0047607421875,
       "learning_rate": 2.7500000000000004e-05,
-      "logits/chosen": -2.122237574979544,
-      "logits/rejected": -1.8119841137964543,
-      "logps/chosen": -124.42662048339844,
-      "logps/rejected": -272.423641204834,
-      "loss": 0.0002692067064344883,
-      "mean_token_accuracy": 0.6206282004714012,
+      "logits/chosen": -2.1326494507814266,
+      "logits/rejected": -1.8268033175884986,
+      "logps/chosen": -125.0985517501831,
+      "logps/rejected": -274.5887870788574,
+      "loss": 0.0002390609442954883,
+      "mean_token_accuracy": 0.620079979300499,
       "num_tokens": 181771.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.9137327093631029,
-      "rewards/margins": 9.385262191295624,
-      "rewards/rejected": -8.47152954339981,
+      "rewards/chosen": 0.8465396109968424,
+      "rewards/margins": 9.534583747386932,
+      "rewards/rejected": -8.688044130802155,
       "step": 28
     },
     {
-      "entropy": 1.1231490969657898,
+      "entropy": 1.1201048269867897,
       "epoch": 1.4528301886792452,
-      "grad_norm": 0.010009765625,
+      "grad_norm": 0.0093994140625,
       "learning_rate": 2.6666666666666667e-05,
-      "logits/chosen": -2.3010927487353845,
-      "logits/rejected": -1.9966991698314587,
-      "logps/chosen": -139.3568572998047,
-      "logps/rejected": -258.50311279296875,
-      "loss": 0.0005843475228175521,
-      "mean_token_accuracy": 0.6224977001547813,
+      "logits/chosen": -2.311015681209517,
+      "logits/rejected": -2.0115759556684885,
+      "logps/chosen": -140.10973072052002,
+      "logps/rejected": -260.4351615905762,
+      "loss": 0.0005316605675034225,
+      "mean_token_accuracy": 0.6181787773966789,
       "num_tokens": 188644.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.0714789517223835,
-      "rewards/margins": 8.368709564208984,
-      "rewards/rejected": -8.440188646316528,
+      "rewards/chosen": -0.1467662937939167,
+      "rewards/margins": 8.486627459526062,
+      "rewards/rejected": -8.633393704891205,
       "step": 29
     },
     {
-      "entropy": 1.1115453839302063,
+      "entropy": 1.1074048355221748,
       "epoch": 1.5031446540880502,
-      "grad_norm": 0.0267333984375,
+      "grad_norm": 0.024658203125,
       "learning_rate": 2.5833333333333336e-05,
-      "logits/chosen": -2.2130385961990293,
-      "logits/rejected": -1.8950639322000826,
-      "logps/chosen": -119.8697099685669,
-      "logps/rejected": -246.79789543151855,
-      "loss": 0.0016802519094198942,
-      "mean_token_accuracy": 0.6181407496333122,
+      "logits/chosen": -2.2242952052635343,
+      "logits/rejected": -1.9084164743872978,
+      "logps/chosen": -120.53641414642334,
+      "logps/rejected": -248.61384201049805,
+      "loss": 0.0015231292927637696,
+      "mean_token_accuracy": 0.6136712729930878,
       "num_tokens": 196391.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.025272462284192443,
-      "rewards/margins": 7.992879867553711,
-      "rewards/rejected": -7.967607498168945,
+      "rewards/chosen": -0.0413979523582384,
+      "rewards/margins": 8.107804000377655,
+      "rewards/rejected": -8.149202108383179,
       "step": 30
     },
     {
-      "entropy": 1.086497575044632,
+      "entropy": 1.0835556164383888,
       "epoch": 1.5534591194968552,
-      "grad_norm": 0.03271484375,
+      "grad_norm": 0.0308837890625,
       "learning_rate": 2.5e-05,
-      "logits/chosen": -2.2244215972877353,
-      "logits/rejected": -1.7872997006786144,
-      "logps/chosen": -159.92728996276855,
-      "logps/rejected": -284.250545501709,
-      "loss": 0.0013625127030536532,
-      "mean_token_accuracy": 0.6142807975411415,
+      "logits/chosen": -2.2358070307390454,
+      "logits/rejected": -1.8015257262782527,
+      "logps/chosen": -160.49627590179443,
+      "logps/rejected": -286.36713790893555,
+      "loss": 0.0012202183715999126,
+      "mean_token_accuracy": 0.6140667796134949,
       "num_tokens": 203518.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.06530415918678045,
-      "rewards/margins": 8.824142456054688,
-      "rewards/rejected": -8.88944661617279,
+      "rewards/chosen": -0.12220276962034404,
+      "rewards/margins": 8.978903114795685,
+      "rewards/rejected": -9.10110592842102,
       "step": 31
     },
     {
-      "entropy": 1.136437065899372,
+      "entropy": 1.1327729299664497,
       "epoch": 1.6037735849056602,
-      "grad_norm": 0.0081787109375,
+      "grad_norm": 0.0078125,
       "learning_rate": 2.4166666666666667e-05,
-      "logits/chosen": -2.211966660335218,
-      "logits/rejected": -1.7966088455890876,
-      "logps/chosen": -143.01754474639893,
-      "logps/rejected": -294.5742607116699,
-      "loss": 0.0003461150045040995,
-      "mean_token_accuracy": 0.6098548360168934,
+      "logits/chosen": -2.222914372779498,
+      "logits/rejected": -1.8115026450741711,
+      "logps/chosen": -143.74180603027344,
+      "logps/rejected": -296.8906936645508,
+      "loss": 0.0003141126944683492,
+      "mean_token_accuracy": 0.6098875515162945,
       "num_tokens": 209850.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.2523078881204128,
-      "rewards/margins": 10.28915536403656,
-      "rewards/rejected": -10.036847531795502,
+      "rewards/chosen": 0.1798817589879036,
+      "rewards/margins": 10.448372840881348,
+      "rewards/rejected": -10.268491089344025,
       "step": 32
     },
     {
-      "entropy": 1.130326434969902,
+      "entropy": 1.125333271920681,
       "epoch": 1.6540880503144653,
-      "grad_norm": 0.01904296875,
+      "grad_norm": 0.017578125,
       "learning_rate": 2.3333333333333336e-05,
-      "logits/chosen": -2.4035100083486487,
-      "logits/rejected": -1.7515896245587153,
-      "logps/chosen": -125.12364768981934,
-      "logps/rejected": -263.9906826019287,
-      "loss": 0.0005816172924824059,
-      "mean_token_accuracy": 0.6256594210863113,
+      "logits/chosen": -2.4135272067915143,
+      "logits/rejected": -1.7676647261719525,
+      "logps/chosen": -125.86479663848877,
+      "logps/rejected": -266.06410026550293,
+      "loss": 0.0005237428122200072,
+      "mean_token_accuracy": 0.6229566335678101,
       "num_tokens": 215637.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.7825962388888001,
-      "rewards/margins": 9.555868089199066,
-      "rewards/rejected": -8.77327173948288,
+      "rewards/chosen": 0.7084813113324344,
+      "rewards/margins": 9.689094841480255,
+      "rewards/rejected": -8.980613589286804,
       "step": 33
     },
     {
-      "entropy": 1.0617773309350014,
+      "entropy": 1.058401882648468,
       "epoch": 1.7044025157232703,
-      "grad_norm": 0.0081787109375,
+      "grad_norm": 0.007354736328125,
       "learning_rate": 2.25e-05,
-      "logits/chosen": -2.450243334633306,
-      "logits/rejected": -2.0934474514571817,
-      "logps/chosen": -125.30186462402344,
-      "logps/rejected": -256.15345764160156,
-      "loss": 0.0004422894271556288,
-      "mean_token_accuracy": 0.6317535266280174,
+      "logits/chosen": -2.46378613488542,
+      "logits/rejected": -2.1071431291236653,
+      "logps/chosen": -125.6760368347168,
+      "logps/rejected": -258.0559844970703,
+      "loss": 0.00038854233571328223,
+      "mean_token_accuracy": 0.6303330659866333,
       "num_tokens": 221051.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 1.3803571164608002,
-      "rewards/margins": 9.975262939929962,
-      "rewards/rejected": -8.594905972480774,
+      "rewards/chosen": 1.3429398629814386,
+      "rewards/margins": 10.128098785877228,
+      "rewards/rejected": -8.785158812999725,
       "step": 34
     },
     {
-      "entropy": 1.1356039494276047,
+      "entropy": 1.1321996748447418,
       "epoch": 1.7547169811320755,
-      "grad_norm": 0.0032958984375,
+      "grad_norm": 0.0030670166015625,
       "learning_rate": 2.1666666666666667e-05,
-      "logits/chosen": -2.364045478373609,
-      "logits/rejected": -2.100374343065235,
-      "logps/chosen": -131.08294582366943,
-      "logps/rejected": -274.32538986206055,
-      "loss": 0.00013096419570501894,
-      "mean_token_accuracy": 0.6208470910787582,
+      "logits/chosen": -2.3747636222967063,
+      "logits/rejected": -2.115993388323722,
+      "logps/chosen": -131.67159175872803,
+      "logps/rejected": -276.36437797546387,
+      "loss": 0.00012030061770929024,
+      "mean_token_accuracy": 0.6251056790351868,
       "num_tokens": 226426.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.9078209921717644,
-      "rewards/margins": 10.476734340190887,
-      "rewards/rejected": -9.568913280963898,
+      "rewards/chosen": 0.8489564098417759,
+      "rewards/margins": 10.621768474578857,
+      "rewards/rejected": -9.772812008857727,
       "step": 35
     },
     {
-      "entropy": 1.1196807771921158,
+      "entropy": 1.116398274898529,
       "epoch": 1.8050314465408805,
-      "grad_norm": 0.0201416015625,
+      "grad_norm": 0.0181884765625,
       "learning_rate": 2.0833333333333336e-05,
-      "logits/chosen": -2.4489547712147877,
-      "logits/rejected": -1.96198692605538,
-      "logps/chosen": -116.7791223526001,
-      "logps/rejected": -270.7232780456543,
-      "loss": 0.0009249980794265866,
-      "mean_token_accuracy": 0.623729519546032,
+      "logits/chosen": -2.4568723077785557,
+      "logits/rejected": -1.9738618373268513,
+      "logps/chosen": -117.32646560668945,
+      "logps/rejected": -272.7527599334717,
+      "loss": 0.0008142158621922135,
+      "mean_token_accuracy": 0.6250453069806099,
       "num_tokens": 233419.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.3964777053333819,
-      "rewards/margins": 9.61779636144638,
-      "rewards/rejected": -9.221318662166595,
+      "rewards/chosen": 0.3417433723807335,
+      "rewards/margins": 9.766010344028473,
+      "rewards/rejected": -9.424266993999481,
       "step": 36
     },
     {
-      "entropy": 1.0988834276795387,
+      "entropy": 1.0965562090277672,
       "epoch": 1.8553459119496856,
-      "grad_norm": 0.002960205078125,
+      "grad_norm": 0.0027313232421875,
       "learning_rate": 2e-05,
-      "logits/chosen": -2.347542994631864,
-      "logits/rejected": -1.9687581260709852,
-      "logps/chosen": -144.3069133758545,
-      "logps/rejected": -294.52979278564453,
-      "loss": 0.0001527789863757789,
-      "mean_token_accuracy": 0.5927288830280304,
+      "logits/chosen": -2.3551532120785383,
+      "logits/rejected": -1.9812547363037738,
+      "logps/chosen": -144.9142713546753,
+      "logps/rejected": -296.3636951446533,
+      "loss": 0.00013589797890745103,
+      "mean_token_accuracy": 0.5917238146066666,
       "num_tokens": 240673.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.041503352113068104,
-      "rewards/margins": 10.00435733795166,
-      "rewards/rejected": -10.045860767364502,
+      "rewards/chosen": -0.10223913192749023,
+      "rewards/margins": 10.127011775970459,
+      "rewards/rejected": -10.22925090789795,
       "step": 37
     },
     {
-      "entropy": 1.09148870408535,
+      "entropy": 1.088953472673893,
       "epoch": 1.9056603773584906,
-      "grad_norm": 0.0035552978515625,
+      "grad_norm": 0.0032806396484375,
       "learning_rate": 1.9166666666666667e-05,
-      "logits/chosen": -2.319408068295181,
-      "logits/rejected": -2.07876476569641,
-      "logps/chosen": -140.95322799682617,
-      "logps/rejected": -283.8196105957031,
-      "loss": 0.0001463937369408086,
-      "mean_token_accuracy": 0.6046550124883652,
+      "logits/chosen": -2.326897647347544,
+      "logits/rejected": -2.0923411183004155,
+      "logps/chosen": -141.62347412109375,
+      "logps/rejected": -285.8790645599365,
+      "loss": 0.0001322527095908299,
+      "mean_token_accuracy": 0.6075941771268845,
       "num_tokens": 245974.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.7766995467245579,
-      "rewards/margins": 10.631062924861908,
-      "rewards/rejected": -9.85436338186264,
+      "rewards/chosen": 0.7096749469637871,
+      "rewards/margins": 10.769983649253845,
+      "rewards/rejected": -10.060308814048767,
       "step": 38
     },
     {
-      "entropy": 1.1747687011957169,
+      "entropy": 1.1708777844905853,
       "epoch": 1.9559748427672956,
-      "grad_norm": 0.00439453125,
+      "grad_norm": 0.003997802734375,
       "learning_rate": 1.8333333333333333e-05,
-      "logits/chosen": -2.1828008029912604,
-      "logits/rejected": -1.726034681820153,
-      "logps/chosen": -126.46953868865967,
-      "logps/rejected": -270.92620849609375,
-      "loss": 0.0002291620330652222,
-      "mean_token_accuracy": 0.6064154356718063,
+      "logits/chosen": -2.1918049696005726,
+      "logits/rejected": -1.7380488443097255,
+      "logps/chosen": -127.21480464935303,
+      "logps/rejected": -273.00823402404785,
+      "loss": 0.00020453613251447678,
+      "mean_token_accuracy": 0.6081509068608284,
       "num_tokens": 253184.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.02468252368271351,
-      "rewards/margins": 9.081076204776764,
-      "rewards/rejected": -9.105758786201477,
+      "rewards/chosen": -0.09920912818051875,
+      "rewards/margins": 9.214752078056335,
+      "rewards/rejected": -9.313961207866669,
       "step": 39
     },
     {
-      "entropy": 1.1650119509015764,
+      "entropy": 1.1617241672107153,
       "epoch": 2.0,
-      "grad_norm": 0.02099609375,
+      "grad_norm": 0.0191650390625,
       "learning_rate": 1.75e-05,
-      "logits/chosen": -2.2664670930140036,
-      "logits/rejected": -1.7445590165072045,
-      "logps/chosen": -153.27142769949776,
-      "logps/rejected": -289.8901563371931,
-      "loss": 0.0010365444468334317,
+      "logits/chosen": -2.2749118295696342,
+      "logits/rejected": -1.7584887991939784,
+      "logps/chosen": -154.0042724609375,
+      "logps/rejected": -292.14568001883373,
+      "loss": 0.0009170390549115837,
       "mean_token_accuracy": 0.5644707764898028,
       "num_tokens": 260000.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.37358812987804413,
-      "rewards/margins": 8.982958453042167,
-      "rewards/rejected": -9.356546606336321,
+      "rewards/chosen": -0.44687261964593616,
+      "rewards/margins": 9.13522618157523,
+      "rewards/rejected": -9.582098756517683,
       "step": 40
     },
     {
-      "entropy": 1.1251012831926346,
+      "entropy": 1.1219489574432373,
       "epoch": 2.050314465408805,
-      "grad_norm": 0.00433349609375,
+      "grad_norm": 0.00396728515625,
       "learning_rate": 1.6666666666666667e-05,
-      "logits/chosen": -2.386099018384776,
-      "logits/rejected": -1.8694206899386892,
-      "logps/chosen": -120.46521186828613,
-      "logps/rejected": -271.48500633239746,
-      "loss": 0.0002081924758385867,
+      "logits/chosen": -2.3933433594531426,
+      "logits/rejected": -1.8782996338947386,
+      "logps/chosen": -121.1051197052002,
+      "logps/rejected": -273.1605224609375,
+      "loss": 0.00018580301548354328,
       "mean_token_accuracy": 0.6062684953212738,
       "num_tokens": 266060.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.5738538741134107,
-      "rewards/margins": 9.70900011062622,
-      "rewards/rejected": -9.13514620065689,
+      "rewards/chosen": 0.5098631188739091,
+      "rewards/margins": 9.81256103515625,
+      "rewards/rejected": -9.302697837352753,
       "step": 41
     },
     {
-      "entropy": 1.1459600627422333,
+      "entropy": 1.142582356929779,
       "epoch": 2.10062893081761,
-      "grad_norm": 0.0054931640625,
+      "grad_norm": 0.00506591796875,
       "learning_rate": 1.5833333333333333e-05,
-      "logits/chosen": -2.3829266191138174,
-      "logits/rejected": -2.0208899907551676,
-      "logps/chosen": -131.44946670532227,
-      "logps/rejected": -271.2663917541504,
-      "loss": 0.00030793892801739275,
-      "mean_token_accuracy": 0.6211965084075928,
+      "logits/chosen": -2.3932424547453,
+      "logits/rejected": -2.034844354918984,
+      "logps/chosen": -132.19951725006104,
+      "logps/rejected": -273.3875484466553,
+      "loss": 0.00027762039098888636,
+      "mean_token_accuracy": 0.6254889070987701,
       "num_tokens": 272769.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.10255242511630058,
-      "rewards/margins": 9.451286256313324,
-      "rewards/rejected": -9.34873378276825,
+      "rewards/chosen": 0.027547355741262436,
+      "rewards/margins": 9.588396728038788,
+      "rewards/rejected": -9.560849368572235,
       "step": 42
     },
     {
-      "entropy": 1.0864861086010933,
+      "entropy": 1.0831276252865791,
       "epoch": 2.150943396226415,
-      "grad_norm": 0.0211181640625,
+      "grad_norm": 0.0201416015625,
       "learning_rate": 1.5e-05,
-      "logits/chosen": -2.313994988239532,
-      "logits/rejected": -1.9055890774453226,
-      "logps/chosen": -140.04273986816406,
-      "logps/rejected": -295.6635856628418,
-      "loss": 0.0007015014416538179,
-      "mean_token_accuracy": 0.623003862798214,
+      "logits/chosen": -2.3213323581687466,
+      "logits/rejected": -1.9188541617869985,
+      "logps/chosen": -140.96678638458252,
+      "logps/rejected": -298.01341819763184,
+      "loss": 0.0006403317674994469,
+      "mean_token_accuracy": 0.6212100088596344,
       "num_tokens": 279603.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.03781823627650738,
-      "rewards/margins": 10.340316832065582,
-      "rewards/rejected": -10.37813514471054,
+      "rewards/chosen": -0.13022289611399174,
+      "rewards/margins": 10.482895195484161,
+      "rewards/rejected": -10.61311811208725,
       "step": 43
     },
     {
-      "entropy": 1.119643673300743,
+      "entropy": 1.1165131032466888,
       "epoch": 2.20125786163522,
-      "grad_norm": 0.004425048828125,
+      "grad_norm": 0.004119873046875,
       "learning_rate": 1.4166666666666668e-05,
-      "logits/chosen": -2.3158235133601206,
-      "logits/rejected": -1.804865597293398,
-      "logps/chosen": -136.55340480804443,
-      "logps/rejected": -278.1607780456543,
-      "loss": 0.00020963302813470364,
-      "mean_token_accuracy": 0.6163425669074059,
+      "logits/chosen": -2.325539811977018,
+      "logits/rejected": -1.819132333499079,
+      "logps/chosen": -137.1863489151001,
+      "logps/rejected": -280.5503120422363,
+      "loss": 0.00018718022329267114,
+      "mean_token_accuracy": 0.6205575913190842,
       "num_tokens": 286436.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.15877847000956535,
-      "rewards/margins": 9.487533926963806,
-      "rewards/rejected": -9.328755378723145,
+      "rewards/chosen": 0.0954840648919344,
+      "rewards/margins": 9.663193047046661,
+      "rewards/rejected": -9.567708969116211,
       "step": 44
     },
     {
-      "entropy": 1.0829864144325256,
+      "entropy": 1.0798147842288017,
       "epoch": 2.251572327044025,
-      "grad_norm": 0.006805419921875,
+      "grad_norm": 0.006195068359375,
       "learning_rate": 1.3333333333333333e-05,
-      "logits/chosen": -2.4553567046643434,
-      "logits/rejected": -1.9043197747016694,
-      "logps/chosen": -137.17022514343262,
-      "logps/rejected": -297.57119941711426,
-      "loss": 0.00036281984648667276,
-      "mean_token_accuracy": 0.6070330739021301,
+      "logits/chosen": -2.462720523269614,
+      "logits/rejected": -1.9150426029399608,
+      "logps/chosen": -137.90959072113037,
+      "logps/rejected": -299.62617683410645,
+      "loss": 0.00032327763619832695,
+      "mean_token_accuracy": 0.6055086851119995,
       "num_tokens": 294223.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.006409928202629089,
-      "rewards/margins": 10.255102336406708,
-      "rewards/rejected": -10.261512219905853,
+      "rewards/chosen": -0.0803464986383915,
+      "rewards/margins": 10.386663496494293,
+      "rewards/rejected": -10.467009961605072,
       "step": 45
     },
     {
-      "entropy": 1.044785775244236,
+      "entropy": 1.0406965985894203,
       "epoch": 2.30188679245283,
-      "grad_norm": 0.003997802734375,
+      "grad_norm": 0.00360107421875,
       "learning_rate": 1.25e-05,
-      "logits/chosen": -2.341592993550264,
-      "logits/rejected": -2.0601840101843556,
-      "logps/chosen": -113.64177894592285,
-      "logps/rejected": -268.7923049926758,
-      "loss": 0.0001799453020794317,
-      "mean_token_accuracy": 0.6692526936531067,
+      "logits/chosen": -2.351859813790029,
+      "logits/rejected": -2.079019230872164,
+      "logps/chosen": -114.20672798156738,
+      "logps/rejected": -270.77587890625,
+      "loss": 0.0001596831134520471,
+      "mean_token_accuracy": 0.6743896827101707,
       "num_tokens": 299634.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 1.3361206632107496,
-      "rewards/margins": 10.512819230556488,
-      "rewards/rejected": -9.176698446273804,
+      "rewards/chosen": 1.2796257368754596,
+      "rewards/margins": 10.65468156337738,
+      "rewards/rejected": -9.375055730342865,
       "step": 46
     },
     {
-      "entropy": 1.1353266835212708,
+      "entropy": 1.1319249421358109,
       "epoch": 2.352201257861635,
-      "grad_norm": 0.0035400390625,
+      "grad_norm": 0.00323486328125,
       "learning_rate": 1.1666666666666668e-05,
-      "logits/chosen": -2.327246213935827,
-      "logits/rejected": -1.8727321034834783,
-      "logps/chosen": -161.3322401046753,
-      "logps/rejected": -302.8222484588623,
-      "loss": 0.00013208847667556256,
-      "mean_token_accuracy": 0.6015190705657005,
+      "logits/chosen": -2.3348495432588066,
+      "logits/rejected": -1.8859447778318736,
+      "logps/chosen": -162.34913635253906,
+      "logps/rejected": -305.2253665924072,
+      "loss": 0.00012244780373293906,
+      "mean_token_accuracy": 0.5992361158132553,
       "num_tokens": 305920.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.028157714928966016,
-      "rewards/margins": 10.57133662700653,
-      "rewards/rejected": -10.543178856372833,
+      "rewards/chosen": -0.0735319098457694,
+      "rewards/margins": 10.709958910942078,
+      "rewards/rejected": -10.783490896224976,
       "step": 47
     },
     {
-      "entropy": 1.1054219380021095,
+      "entropy": 1.1012349873781204,
       "epoch": 2.40251572327044,
-      "grad_norm": 0.006011962890625,
+      "grad_norm": 0.00555419921875,
       "learning_rate": 1.0833333333333334e-05,
-      "logits/chosen": -2.3692881200690303,
-      "logits/rejected": -1.953993051539208,
-      "logps/chosen": -133.50615692138672,
-      "logps/rejected": -267.50397300720215,
-      "loss": 0.0003233029565308243,
-      "mean_token_accuracy": 0.6168623343110085,
+      "logits/chosen": -2.376434497997167,
+      "logits/rejected": -1.9649504169742804,
+      "logps/chosen": -134.29817581176758,
+      "logps/rejected": -269.4762592315674,
+      "loss": 0.00029120981344021857,
+      "mean_token_accuracy": 0.615195669233799,
       "num_tokens": 313346.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.12370538981485879,
-      "rewards/margins": 8.729556858539581,
-      "rewards/rejected": -8.605851471424103,
+      "rewards/chosen": 0.04450349509716034,
+      "rewards/margins": 8.847583532333374,
+      "rewards/rejected": -8.803080201148987,
       "step": 48
     },
     {
-      "entropy": 1.0848723128437996,
+      "entropy": 1.0822330936789513,
       "epoch": 2.452830188679245,
-      "grad_norm": 0.004974365234375,
+      "grad_norm": 0.004486083984375,
       "learning_rate": 1e-05,
-      "logits/chosen": -2.4114885162664828,
-      "logits/rejected": -1.933475836984313,
-      "logps/chosen": -138.67891216278076,
-      "logps/rejected": -284.93113708496094,
-      "loss": 0.0002774967870209366,
-      "mean_token_accuracy": 0.5793358683586121,
+      "logits/chosen": -2.4181823111443506,
+      "logits/rejected": -1.9432117626083543,
+      "logps/chosen": -139.22668647766113,
+      "logps/rejected": -286.98141860961914,
+      "loss": 0.00024342790129594505,
+      "mean_token_accuracy": 0.5807452276349068,
       "num_tokens": 320662.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.1479677117895335,
-      "rewards/margins": 9.403626084327698,
-      "rewards/rejected": -9.551593720912933,
+      "rewards/chosen": -0.20274513261392713,
+      "rewards/margins": 9.553876638412476,
+      "rewards/rejected": -9.756621778011322,
       "step": 49
     },
     {
-      "entropy": 1.1292646825313568,
+      "entropy": 1.1255912110209465,
       "epoch": 2.50314465408805,
-      "grad_norm": 0.0113525390625,
+      "grad_norm": 0.01043701171875,
       "learning_rate": 9.166666666666666e-06,
-      "logits/chosen": -2.3793244079557065,
-      "logits/rejected": -2.0578088532827836,
-      "logps/chosen": -151.90766048431396,
-      "logps/rejected": -276.3668155670166,
-      "loss": 0.0005854924093000591,
-      "mean_token_accuracy": 0.59138548001647,
+      "logits/chosen": -2.390428515839434,
+      "logits/rejected": -2.070867890248319,
+      "logps/chosen": -152.57851219177246,
+      "logps/rejected": -278.1038990020752,
+      "loss": 0.0005284082726575434,
+      "mean_token_accuracy": 0.5914618484675884,
       "num_tokens": 326611.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.3165478724986315,
-      "rewards/margins": 9.21978086233139,
-      "rewards/rejected": -8.903232991695404,
+      "rewards/chosen": 0.24946272000670433,
+      "rewards/margins": 9.3264040350914,
+      "rewards/rejected": -9.076941430568695,
       "step": 50
     },
     {
-      "entropy": 1.1128623560070992,
+      "entropy": 1.10995664447546,
       "epoch": 2.5534591194968552,
-      "grad_norm": 0.00640869140625,
+      "grad_norm": 0.005950927734375,
       "learning_rate": 8.333333333333334e-06,
-      "logits/chosen": -2.3023183294428042,
-      "logits/rejected": -1.967437557097666,
-      "logps/chosen": -116.14246654510498,
-      "logps/rejected": -265.47961044311523,
-      "loss": 0.000282328634057194,
-      "mean_token_accuracy": 0.6275674253702164,
+      "logits/chosen": -2.3087954809493545,
+      "logits/rejected": -1.9792274410316972,
+      "logps/chosen": -116.59206485748291,
+      "logps/rejected": -267.238338470459,
+      "loss": 0.00025631688185967505,
+      "mean_token_accuracy": 0.625831313431263,
       "num_tokens": 332814.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.9145755774807185,
-      "rewards/margins": 9.993305206298828,
-      "rewards/rejected": -9.078729569911957,
+      "rewards/chosen": 0.8696157680824399,
+      "rewards/margins": 10.124218106269836,
+      "rewards/rejected": -9.254602372646332,
       "step": 51
     },
     {
-      "entropy": 1.1205245852470398,
+      "entropy": 1.117017239332199,
       "epoch": 2.6037735849056602,
-      "grad_norm": 0.001739501953125,
+      "grad_norm": 0.00152587890625,
       "learning_rate": 7.5e-06,
-      "logits/chosen": -2.405199971394472,
-      "logits/rejected": -1.9401018785140567,
-      "logps/chosen": -120.2322769165039,
-      "logps/rejected": -278.4016227722168,
-      "loss": 9.171015699394047e-05,
-      "mean_token_accuracy": 0.611074797809124,
+      "logits/chosen": -2.4137662915261817,
+      "logits/rejected": -1.9523699949115683,
+      "logps/chosen": -120.80679893493652,
+      "logps/rejected": -280.64561653137207,
+      "loss": 7.924844976514578e-05,
+      "mean_token_accuracy": 0.6106409505009651,
       "num_tokens": 339547.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.8479831870645285,
-      "rewards/margins": 10.514736771583557,
-      "rewards/rejected": -9.66675341129303,
+      "rewards/chosen": 0.790530975908041,
+      "rewards/margins": 10.681683659553528,
+      "rewards/rejected": -9.891152799129486,
       "step": 52
     },
     {
-      "entropy": 1.1072203814983368,
+      "entropy": 1.1054791137576103,
       "epoch": 2.6540880503144653,
-      "grad_norm": 0.017822265625,
+      "grad_norm": 0.0164794921875,
       "learning_rate": 6.666666666666667e-06,
-      "logits/chosen": -2.2667942885965546,
-      "logits/rejected": -1.900450488756554,
-      "logps/chosen": -115.35403633117676,
-      "logps/rejected": -267.29714012145996,
-      "loss": 0.0008604128961451352,
-      "mean_token_accuracy": 0.5969742015004158,
+      "logits/chosen": -2.273507778205582,
+      "logits/rejected": -1.9109036313937824,
+      "logps/chosen": -115.94961833953857,
+      "logps/rejected": -269.0294952392578,
+      "loss": 0.0007658082176931202,
+      "mean_token_accuracy": 0.5990233793854713,
       "num_tokens": 346647.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.317084601148963,
-      "rewards/margins": 9.336723506450653,
-      "rewards/rejected": -9.019638895988464,
+      "rewards/chosen": 0.25752640701830387,
+      "rewards/margins": 9.45040088891983,
+      "rewards/rejected": -9.192874610424042,
       "step": 53
     },
     {
-      "entropy": 1.1071247309446335,
+      "entropy": 1.103651024401188,
       "epoch": 2.7044025157232703,
-      "grad_norm": 0.0029754638671875,
+      "grad_norm": 0.002593994140625,
       "learning_rate": 5.833333333333334e-06,
-      "logits/chosen": -2.324148264155996,
-      "logits/rejected": -1.7913556462377824,
-      "logps/chosen": -151.82556343078613,
-      "logps/rejected": -299.4059410095215,
-      "loss": 0.00012161008635303006,
-      "mean_token_accuracy": 0.6138368621468544,
+      "logits/chosen": -2.333249501998797,
+      "logits/rejected": -1.8058473440003249,
+      "logps/chosen": -152.4753885269165,
+      "logps/rejected": -301.55738639831543,
+      "loss": 0.00010592853504931554,
+      "mean_token_accuracy": 0.6078185737133026,
       "num_tokens": 353155.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.3809825163334608,
-      "rewards/margins": 10.480114579200745,
-      "rewards/rejected": -10.09913194179535,
+      "rewards/chosen": 0.3159999940544367,
+      "rewards/margins": 10.630276560783386,
+      "rewards/rejected": -10.314276397228241,
       "step": 54
     },
     {
-      "entropy": 1.16407360881567,
+      "entropy": 1.160094790160656,
       "epoch": 2.7547169811320753,
-      "grad_norm": 0.0017242431640625,
+      "grad_norm": 0.0015716552734375,
       "learning_rate": 5e-06,
-      "logits/chosen": -2.322006496808721,
-      "logits/rejected": -1.882509489706578,
-      "logps/chosen": -140.3326654434204,
-      "logps/rejected": -293.94667053222656,
-      "loss": 7.282369915628806e-05,
-      "mean_token_accuracy": 0.617231510579586,
+      "logits/chosen": -2.3308505618468462,
+      "logits/rejected": -1.8965502590056609,
+      "logps/chosen": -141.07257843017578,
+      "logps/rejected": -296.07042503356934,
+      "loss": 6.342934648273513e-05,
+      "mean_token_accuracy": 0.6171156838536263,
       "num_tokens": 358538.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.7881479188799858,
-      "rewards/margins": 10.648730993270874,
-      "rewards/rejected": -9.860582947731018,
+      "rewards/chosen": 0.7141566202044487,
+      "rewards/margins": 10.787114977836609,
+      "rewards/rejected": -10.072958290576935,
       "step": 55
     },
     {
-      "entropy": 1.1090919971466064,
+      "entropy": 1.1058026999235153,
       "epoch": 2.8050314465408803,
-      "grad_norm": 0.0311279296875,
+      "grad_norm": 0.030517578125,
       "learning_rate": 4.166666666666667e-06,
-      "logits/chosen": -2.31211133409438,
-      "logits/rejected": -1.8826621248132671,
-      "logps/chosen": -121.12357425689697,
-      "logps/rejected": -252.39348220825195,
-      "loss": 0.0018161134794354439,
-      "mean_token_accuracy": 0.6220331639051437,
+      "logits/chosen": -2.322696784138965,
+      "logits/rejected": -1.894479697226797,
+      "logps/chosen": -121.85036373138428,
+      "logps/rejected": -254.1634178161621,
+      "loss": 0.00172812445089221,
+      "mean_token_accuracy": 0.6225817948579788,
       "num_tokens": 364730.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.17907649464905262,
-      "rewards/margins": 8.474798560142517,
-      "rewards/rejected": -8.295722126960754,
+      "rewards/chosen": 0.10639754496514797,
+      "rewards/margins": 8.579113006591797,
+      "rewards/rejected": -8.472715437412262,
       "step": 56
     },
     {
-      "entropy": 1.1059659719467163,
+      "entropy": 1.102943792939186,
       "epoch": 2.8553459119496853,
-      "grad_norm": 0.0093994140625,
+      "grad_norm": 0.00872802734375,
       "learning_rate": 3.3333333333333333e-06,
-      "logits/chosen": -2.1642121086590733,
-      "logits/rejected": -1.9605235069705975,
-      "logps/chosen": -142.26798820495605,
-      "logps/rejected": -266.46795082092285,
-      "loss": 0.0004599351086653769,
-      "mean_token_accuracy": 0.6000082641839981,
+      "logits/chosen": -2.173708624672683,
+      "logits/rejected": -1.970646926930764,
+      "logps/chosen": -142.89206504821777,
+      "logps/rejected": -268.407958984375,
+      "loss": 0.00041806488297879696,
+      "mean_token_accuracy": 0.5970962047576904,
       "num_tokens": 371209.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.3405567053705454,
-      "rewards/margins": 9.269334852695465,
-      "rewards/rejected": -8.928778231143951,
+      "rewards/chosen": 0.27814904414117336,
+      "rewards/margins": 9.40092819929123,
+      "rewards/rejected": -9.122779071331024,
       "step": 57
     },
     {
-      "entropy": 1.1505941599607468,
+      "entropy": 1.1477283239364624,
       "epoch": 2.9056603773584904,
-      "grad_norm": 0.005279541015625,
+      "grad_norm": 0.00469970703125,
       "learning_rate": 2.5e-06,
-      "logits/chosen": -2.2861668483136004,
-      "logits/rejected": -1.8705101269968154,
-      "logps/chosen": -136.88138389587402,
-      "logps/rejected": -289.31751251220703,
-      "loss": 0.0002126125618815422,
-      "mean_token_accuracy": 0.6103544309735298,
+      "logits/chosen": -2.293317013131777,
+      "logits/rejected": -1.8812048808544868,
+      "logps/chosen": -137.3569221496582,
+      "logps/rejected": -291.4598960876465,
+      "loss": 0.00018517834541853517,
+      "mean_token_accuracy": 0.6110436543822289,
       "num_tokens": 378077.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.38323727436363697,
-      "rewards/margins": 10.188703656196594,
-      "rewards/rejected": -9.805466413497925,
+      "rewards/chosen": 0.33568347059190273,
+      "rewards/margins": 10.355388283729553,
+      "rewards/rejected": -10.01970487833023,
       "step": 58
     },
     {
-      "entropy": 1.0992570891976357,
+      "entropy": 1.0955762341618538,
       "epoch": 2.9559748427672954,
-      "grad_norm": 0.00213623046875,
+      "grad_norm": 0.00193023681640625,
       "learning_rate": 1.6666666666666667e-06,
-      "logits/chosen": -2.298579599751336,
-      "logits/rejected": -2.0673505464671744,
-      "logps/chosen": -141.01217555999756,
-      "logps/rejected": -277.6583061218262,
-      "loss": 0.00010025675874203444,
-      "mean_token_accuracy": 0.6147632747888565,
+      "logits/chosen": -2.3105184309249727,
+      "logits/rejected": -2.0821060341980577,
+      "logps/chosen": -141.80660915374756,
+      "logps/rejected": -279.77105140686035,
+      "loss": 8.936067024478689e-05,
+      "mean_token_accuracy": 0.6105019077658653,
       "num_tokens": 384115.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.6212236363207921,
-      "rewards/margins": 10.357111930847168,
-      "rewards/rejected": -9.735888302326202,
+      "rewards/chosen": 0.5417802967131138,
+      "rewards/margins": 10.488943040370941,
+      "rewards/rejected": -9.947162747383118,
       "step": 59
     },
     {
-      "entropy": 1.094216755458287,
+      "entropy": 1.0900011318070548,
       "epoch": 3.0,
-      "grad_norm": 0.02587890625,
+      "grad_norm": 0.024169921875,
       "learning_rate": 8.333333333333333e-07,
-      "logits/chosen": -2.3206561180335696,
-      "logits/rejected": -1.7881133994765854,
-      "logps/chosen": -137.9519805908203,
-      "logps/rejected": -263.429922921317,
-      "loss": 0.0014771231217309833,
-      "mean_token_accuracy": 0.610472559928894,
+      "logits/chosen": -2.332351866206725,
+      "logits/rejected": -1.8031474708962778,
+      "logps/chosen": -138.6800994873047,
+      "logps/rejected": -265.439217703683,
+      "loss": 0.0013465541414916515,
+      "mean_token_accuracy": 0.6026440688541957,
       "num_tokens": 390000.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.13421392613755806,
-      "rewards/margins": 8.44434186390468,
-      "rewards/rejected": -8.578555720193046,
+      "rewards/chosen": -0.2070258174623762,
+      "rewards/margins": 8.572459425245013,
+      "rewards/rejected": -8.77948522567749,
       "step": 60
     }
   ],