diff --git "a/checkpoint-1833/trainer_state.json" "b/checkpoint-1833/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/checkpoint-1833/trainer_state.json"
@@ -0,0 +1,2779 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 1833,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.016366612111292964,
+      "grad_norm": NaN,
+      "learning_rate": 1.6304347826086957e-06,
+      "logits/chosen": -1.2515310049057007,
+      "logits/rejected": -0.8573959469795227,
+      "logps/chosen": -455.1468811035156,
+      "logps/rejected": -387.0008239746094,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": 0.0002114105736836791,
+      "rewards/margins": 0.00180077797267586,
+      "rewards/rejected": -0.0015893673989921808,
+      "step": 10
+    },
+    {
+      "epoch": 0.03273322422258593,
+      "grad_norm": 10.154256820678711,
+      "learning_rate": 3.2608695652173914e-06,
+      "logits/chosen": -1.145971417427063,
+      "logits/rejected": -1.082183837890625,
+      "logps/chosen": -446.249755859375,
+      "logps/rejected": -426.87237548828125,
+      "loss": 0.6942,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.0005732345744036138,
+      "rewards/margins": -0.002082281280308962,
+      "rewards/rejected": 0.0015090465312823653,
+      "step": 20
+    },
+    {
+      "epoch": 0.049099836333878884,
+      "grad_norm": 8.393162727355957,
+      "learning_rate": 5.4347826086956525e-06,
+      "logits/chosen": -0.9614235162734985,
+      "logits/rejected": -0.8527517318725586,
+      "logps/chosen": -460.367431640625,
+      "logps/rejected": -433.61285400390625,
+      "loss": 0.6934,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 0.0062791816890239716,
+      "rewards/margins": -0.00045429245801642537,
+      "rewards/rejected": 0.006733474787324667,
+      "step": 30
+    },
+    {
+      "epoch": 0.06546644844517185,
+      "grad_norm": 9.398829460144043,
+      "learning_rate": 8.15217391304348e-06,
+      "logits/chosen": -1.171631097793579,
+      "logits/rejected": -1.0616101026535034,
+      "logps/chosen": -421.583984375,
+      "logps/rejected": -424.787109375,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.020799484103918076,
+      "rewards/margins": 0.006221642717719078,
+      "rewards/rejected": 0.014577841386198997,
+      "step": 40
+    },
+    {
+      "epoch": 0.08183306055646482,
+      "grad_norm": 9.43005657196045,
+      "learning_rate": 1.0869565217391305e-05,
+      "logits/chosen": -1.2450587749481201,
+      "logits/rejected": -1.1417832374572754,
+      "logps/chosen": -476.83538818359375,
+      "logps/rejected": -444.6556091308594,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.042164258658885956,
+      "rewards/margins": 0.010725794360041618,
+      "rewards/rejected": 0.03143846243619919,
+      "step": 50
+    },
+    {
+      "epoch": 0.09819967266775777,
+      "grad_norm": 8.825221061706543,
+      "learning_rate": 1.331521739130435e-05,
+      "logits/chosen": -1.3223823308944702,
+      "logits/rejected": NaN,
+      "logps/chosen": -383.4811706542969,
+      "logps/rejected": -454.43780517578125,
+      "loss": 0.6965,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.03668512776494026,
+      "rewards/margins": -0.003995636478066444,
+      "rewards/rejected": 0.04068076238036156,
+      "step": 60
+    },
+    {
+      "epoch": 0.11456628477905073,
+      "grad_norm": 9.5093994140625,
+      "learning_rate": 1.6032608695652173e-05,
+      "logits/chosen": -1.2538468837738037,
+      "logits/rejected": -1.2771422863006592,
+      "logps/chosen": -468.79400634765625,
+      "logps/rejected": -402.565673828125,
+      "loss": 0.6864,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.04952714219689369,
+      "rewards/margins": 0.01824185810983181,
+      "rewards/rejected": 0.03128528222441673,
+      "step": 70
+    },
+    {
+      "epoch": 0.1309328968903437,
+      "grad_norm": 7.945922374725342,
+      "learning_rate": 1.8750000000000002e-05,
+      "logits/chosen": -1.386731743812561,
+      "logits/rejected": -0.8966825604438782,
+      "logps/chosen": -446.54736328125,
+      "logps/rejected": -430.45025634765625,
+      "loss": 0.6719,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.06767664849758148,
+      "rewards/margins": 0.048766594380140305,
+      "rewards/rejected": 0.01891004480421543,
+      "step": 80
+    },
+    {
+      "epoch": 0.14729950900163666,
+      "grad_norm": 8.496461868286133,
+      "learning_rate": 2.1467391304347828e-05,
+      "logits/chosen": -1.3730494976043701,
+      "logits/rejected": -1.1826181411743164,
+      "logps/chosen": -413.87835693359375,
+      "logps/rejected": -385.1138916015625,
+      "loss": 0.69,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.07122664153575897,
+      "rewards/margins": 0.014941570349037647,
+      "rewards/rejected": 0.0562850646674633,
+      "step": 90
+    },
+    {
+      "epoch": 0.16366612111292964,
+      "grad_norm": 10.744088172912598,
+      "learning_rate": 2.4184782608695653e-05,
+      "logits/chosen": -1.2254825830459595,
+      "logits/rejected": -1.1136434078216553,
+      "logps/chosen": -495.34295654296875,
+      "logps/rejected": -435.0003967285156,
+      "loss": 0.6618,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.1682208627462387,
+      "rewards/margins": 0.08198796212673187,
+      "rewards/rejected": 0.08623288571834564,
+      "step": 100
+    },
+    {
+      "epoch": 0.18003273322422259,
+      "grad_norm": 11.016289710998535,
+      "learning_rate": 2.6902173913043476e-05,
+      "logits/chosen": -1.3138881921768188,
+      "logits/rejected": -1.0289314985275269,
+      "logps/chosen": -476.26153564453125,
+      "logps/rejected": -426.837646484375,
+      "loss": 0.6355,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.21744687855243683,
+      "rewards/margins": 0.15108919143676758,
+      "rewards/rejected": 0.06635768711566925,
+      "step": 110
+    },
+    {
+      "epoch": 0.19639934533551553,
+      "grad_norm": 11.619248390197754,
+      "learning_rate": 2.9619565217391305e-05,
+      "logits/chosen": -1.2751747369766235,
+      "logits/rejected": -1.1858090162277222,
+      "logps/chosen": -444.4823303222656,
+      "logps/rejected": -404.58123779296875,
+      "loss": 0.6716,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.2203199863433838,
+      "rewards/margins": 0.06618332862854004,
+      "rewards/rejected": 0.15413665771484375,
+      "step": 120
+    },
+    {
+      "epoch": 0.2127659574468085,
+      "grad_norm": 8.900009155273438,
+      "learning_rate": 3.233695652173913e-05,
+      "logits/chosen": -1.2365785837173462,
+      "logits/rejected": -1.2031922340393066,
+      "logps/chosen": -477.8089294433594,
+      "logps/rejected": -456.80120849609375,
+      "loss": 0.6794,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.20705166459083557,
+      "rewards/margins": 0.060810189694166183,
+      "rewards/rejected": 0.1462414562702179,
+      "step": 130
+    },
+    {
+      "epoch": 0.22913256955810146,
+      "grad_norm": 8.928816795349121,
+      "learning_rate": 3.505434782608696e-05,
+      "logits/chosen": -1.2421447038650513,
+      "logits/rejected": -1.136252522468567,
+      "logps/chosen": -547.4909057617188,
+      "logps/rejected": -460.38275146484375,
+      "loss": 0.6047,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.21343636512756348,
+      "rewards/margins": 0.24689535796642303,
+      "rewards/rejected": -0.033459004014730453,
+      "step": 140
+    },
+    {
+      "epoch": 0.24549918166939444,
+      "grad_norm": 11.977150917053223,
+      "learning_rate": 3.777173913043478e-05,
+      "logits/chosen": -1.338332176208496,
+      "logits/rejected": -1.1015567779541016,
+      "logps/chosen": -432.2693786621094,
+      "logps/rejected": -423.0946350097656,
+      "loss": 0.6946,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.15119442343711853,
+      "rewards/margins": 0.07870660722255707,
+      "rewards/rejected": 0.07248781621456146,
+      "step": 150
+    },
+    {
+      "epoch": 0.2618657937806874,
+      "grad_norm": 13.040871620178223,
+      "learning_rate": 4.0489130434782615e-05,
+      "logits/chosen": -1.0111101865768433,
+      "logits/rejected": NaN,
+      "logps/chosen": -472.6917419433594,
+      "logps/rejected": -426.6333923339844,
+      "loss": 0.6598,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.24765905737876892,
+      "rewards/margins": 0.13647916913032532,
+      "rewards/rejected": 0.11117993295192719,
+      "step": 160
+    },
+    {
+      "epoch": 0.27823240589198034,
+      "grad_norm": 8.028325080871582,
+      "learning_rate": 4.3206521739130434e-05,
+      "logits/chosen": -0.9152072668075562,
+      "logits/rejected": -0.8340185284614563,
+      "logps/chosen": -438.38555908203125,
+      "logps/rejected": -392.93426513671875,
+      "loss": 0.6272,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.3285876512527466,
+      "rewards/margins": 0.20840699970722198,
+      "rewards/rejected": 0.1201806515455246,
+      "step": 170
+    },
+    {
+      "epoch": 0.2945990180032733,
+      "grad_norm": 9.436712265014648,
+      "learning_rate": 4.592391304347826e-05,
+      "logits/chosen": -1.0703661441802979,
+      "logits/rejected": -1.0127884149551392,
+      "logps/chosen": -449.83905029296875,
+      "logps/rejected": -437.92822265625,
+      "loss": 0.6684,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.34186336398124695,
+      "rewards/margins": 0.14258861541748047,
+      "rewards/rejected": 0.19927474856376648,
+      "step": 180
+    },
+    {
+      "epoch": 0.3109656301145663,
+      "grad_norm": 8.914007186889648,
+      "learning_rate": 4.8641304347826086e-05,
+      "logits/chosen": -0.8762575387954712,
+      "logits/rejected": -0.5631248354911804,
+      "logps/chosen": -473.806396484375,
+      "logps/rejected": -443.6448669433594,
+      "loss": 0.6243,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.3772050440311432,
+      "rewards/margins": 0.24287982285022736,
+      "rewards/rejected": 0.13432522118091583,
+      "step": 190
+    },
+    {
+      "epoch": 0.32733224222585927,
+      "grad_norm": 14.76502513885498,
+      "learning_rate": 4.98483929654336e-05,
+      "logits/chosen": NaN,
+      "logits/rejected": -0.9821416139602661,
+      "logps/chosen": -447.65386962890625,
+      "logps/rejected": -400.1768493652344,
+      "loss": 0.6805,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.2871859073638916,
+      "rewards/margins": 0.17125040292739868,
+      "rewards/rejected": 0.11593550443649292,
+      "step": 200
+    },
+    {
+      "epoch": 0.3436988543371522,
+      "grad_norm": 11.80213451385498,
+      "learning_rate": 4.954517889630079e-05,
+      "logits/chosen": -1.2535641193389893,
+      "logits/rejected": -1.0489479303359985,
+      "logps/chosen": -457.010986328125,
+      "logps/rejected": -432.8558044433594,
+      "loss": 0.6769,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.24471895396709442,
+      "rewards/margins": 0.16205154359340668,
+      "rewards/rejected": 0.08266742527484894,
+      "step": 210
+    },
+    {
+      "epoch": 0.36006546644844517,
+      "grad_norm": 7.534430980682373,
+      "learning_rate": 4.924196482716799e-05,
+      "logits/chosen": -1.5050650835037231,
+      "logits/rejected": -1.5133517980575562,
+      "logps/chosen": -445.3924865722656,
+      "logps/rejected": -420.99114990234375,
+      "loss": 0.7003,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.18017885088920593,
+      "rewards/margins": 0.1387586146593094,
+      "rewards/rejected": 0.04142025485634804,
+      "step": 220
+    },
+    {
+      "epoch": 0.37643207855973815,
+      "grad_norm": 7.207632541656494,
+      "learning_rate": 4.893875075803517e-05,
+      "logits/chosen": -1.3496973514556885,
+      "logits/rejected": -1.2030962705612183,
+      "logps/chosen": -436.5039978027344,
+      "logps/rejected": -396.91534423828125,
+      "loss": 0.6192,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.19479091465473175,
+      "rewards/margins": 0.353401243686676,
+      "rewards/rejected": -0.15861034393310547,
+      "step": 230
+    },
+    {
+      "epoch": 0.39279869067103107,
+      "grad_norm": 9.134820938110352,
+      "learning_rate": 4.8665858095815645e-05,
+      "logits/chosen": -1.2978178262710571,
+      "logits/rejected": -1.1864020824432373,
+      "logps/chosen": -486.7569885253906,
+      "logps/rejected": -457.94586181640625,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.29108190536499023,
+      "rewards/margins": 0.15347807109355927,
+      "rewards/rejected": 0.13760384917259216,
+      "step": 240
+    },
+    {
+      "epoch": 0.40916530278232405,
+      "grad_norm": 10.825005531311035,
+      "learning_rate": 4.836264402668284e-05,
+      "logits/chosen": -1.3271738290786743,
+      "logits/rejected": -1.030890703201294,
+      "logps/chosen": -421.89349365234375,
+      "logps/rejected": -400.1674499511719,
+      "loss": 0.6536,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.42532652616500854,
+      "rewards/margins": 0.34351587295532227,
+      "rewards/rejected": 0.08181065320968628,
+      "step": 250
+    },
+    {
+      "epoch": 0.425531914893617,
+      "grad_norm": 9.450189590454102,
+      "learning_rate": 4.8059429957550034e-05,
+      "logits/chosen": -1.0212231874465942,
+      "logits/rejected": -1.0058345794677734,
+      "logps/chosen": -422.1316833496094,
+      "logps/rejected": -390.0425109863281,
+      "loss": 0.6384,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.490867555141449,
+      "rewards/margins": 0.35965341329574585,
+      "rewards/rejected": 0.13121409714221954,
+      "step": 260
+    },
+    {
+      "epoch": 0.44189852700491,
+      "grad_norm": 8.231124877929688,
+      "learning_rate": 4.7756215888417225e-05,
+      "logits/chosen": -0.9735445976257324,
+      "logits/rejected": -0.937743067741394,
+      "logps/chosen": -552.4168701171875,
+      "logps/rejected": -501.0535583496094,
+      "loss": 0.6694,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.4536367952823639,
+      "rewards/margins": 0.19916944205760956,
+      "rewards/rejected": 0.2544674277305603,
+      "step": 270
+    },
+    {
+      "epoch": 0.4582651391162029,
+      "grad_norm": 9.207930564880371,
+      "learning_rate": 4.7453001819284416e-05,
+      "logits/chosen": -0.9774928092956543,
+      "logits/rejected": -0.8485246896743774,
+      "logps/chosen": -444.54827880859375,
+      "logps/rejected": -419.5541076660156,
+      "loss": 0.6607,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.3690536618232727,
+      "rewards/margins": 0.22144463658332825,
+      "rewards/rejected": 0.14760902523994446,
+      "step": 280
+    },
+    {
+      "epoch": 0.4746317512274959,
+      "grad_norm": 6.876308441162109,
+      "learning_rate": 4.714978775015161e-05,
+      "logits/chosen": -1.2095867395401,
+      "logits/rejected": -0.8773800730705261,
+      "logps/chosen": -437.869384765625,
+      "logps/rejected": -419.9366149902344,
+      "loss": 0.5867,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.5094749331474304,
+      "rewards/margins": 0.5250319242477417,
+      "rewards/rejected": -0.015556998550891876,
+      "step": 290
+    },
+    {
+      "epoch": 0.4909983633387889,
+      "grad_norm": 12.918950080871582,
+      "learning_rate": 4.68465736810188e-05,
+      "logits/chosen": -0.9076827764511108,
+      "logits/rejected": -0.7181829214096069,
+      "logps/chosen": -469.4864196777344,
+      "logps/rejected": -420.38885498046875,
+      "loss": 0.6833,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.5444287657737732,
+      "rewards/margins": 0.2743019461631775,
+      "rewards/rejected": 0.2701267600059509,
+      "step": 300
+    },
+    {
+      "epoch": 0.5073649754500819,
+      "grad_norm": 4.9246110916137695,
+      "learning_rate": 4.6543359611885996e-05,
+      "logits/chosen": -1.214247226715088,
+      "logits/rejected": -0.9910456538200378,
+      "logps/chosen": -385.4250183105469,
+      "logps/rejected": -390.26043701171875,
+      "loss": 0.6419,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.4199371337890625,
+      "rewards/margins": 0.34462255239486694,
+      "rewards/rejected": 0.07531457394361496,
+      "step": 310
+    },
+    {
+      "epoch": 0.5237315875613748,
+      "grad_norm": 8.829249382019043,
+      "learning_rate": 4.624014554275319e-05,
+      "logits/chosen": -1.0396336317062378,
+      "logits/rejected": -0.9439867734909058,
+      "logps/chosen": -452.1678161621094,
+      "logps/rejected": -379.70745849609375,
+      "loss": 0.6396,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.472791850566864,
+      "rewards/margins": 0.23509594798088074,
+      "rewards/rejected": 0.23769588768482208,
+      "step": 320
+    },
+    {
+      "epoch": 0.5400981996726678,
+      "grad_norm": 13.249287605285645,
+      "learning_rate": 4.593693147362038e-05,
+      "logits/chosen": -0.927478015422821,
+      "logits/rejected": -0.7090943455696106,
+      "logps/chosen": -488.009765625,
+      "logps/rejected": -473.47833251953125,
+      "loss": 0.6833,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.4911763668060303,
+      "rewards/margins": 0.25351986289024353,
+      "rewards/rejected": 0.23765654861927032,
+      "step": 330
+    },
+    {
+      "epoch": 0.5564648117839607,
+      "grad_norm": 9.580172538757324,
+      "learning_rate": 4.566403881140085e-05,
+      "logits/chosen": -0.9790302515029907,
+      "logits/rejected": -0.8282285928726196,
+      "logps/chosen": -481.6971740722656,
+      "logps/rejected": -441.3619689941406,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.4195849299430847,
+      "rewards/margins": 0.210377499461174,
+      "rewards/rejected": 0.20920750498771667,
+      "step": 340
+    },
+    {
+      "epoch": 0.5728314238952537,
+      "grad_norm": 6.872241973876953,
+      "learning_rate": 4.536082474226804e-05,
+      "logits/chosen": -0.9942235946655273,
+      "logits/rejected": -0.824662983417511,
+      "logps/chosen": -437.9869079589844,
+      "logps/rejected": -417.3473205566406,
+      "loss": 0.6071,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.49881643056869507,
+      "rewards/margins": 0.41778793931007385,
+      "rewards/rejected": 0.08102855831384659,
+      "step": 350
+    },
+    {
+      "epoch": 0.5891980360065466,
+      "grad_norm": 7.748345375061035,
+      "learning_rate": 4.5057610673135234e-05,
+      "logits/chosen": -1.0704625844955444,
+      "logits/rejected": -0.7879603505134583,
+      "logps/chosen": -428.53021240234375,
+      "logps/rejected": -435.9026794433594,
+      "loss": 0.6084,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.4484317898750305,
+      "rewards/margins": 0.4125775396823883,
+      "rewards/rejected": 0.035854239016771317,
+      "step": 360
+    },
+    {
+      "epoch": 0.6055646481178396,
+      "grad_norm": 7.428864479064941,
+      "learning_rate": 4.475439660400243e-05,
+      "logits/chosen": -1.098026990890503,
+      "logits/rejected": -0.9232645034790039,
+      "logps/chosen": -441.7071228027344,
+      "logps/rejected": -402.95770263671875,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.37514036893844604,
+      "rewards/margins": 0.2525361478328705,
+      "rewards/rejected": 0.12260414659976959,
+      "step": 370
+    },
+    {
+      "epoch": 0.6219312602291326,
+      "grad_norm": 9.591557502746582,
+      "learning_rate": 4.445118253486962e-05,
+      "logits/chosen": -1.2409312725067139,
+      "logits/rejected": -0.8442489504814148,
+      "logps/chosen": -466.13311767578125,
+      "logps/rejected": -420.9102478027344,
+      "loss": 0.7684,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 0.2286566197872162,
+      "rewards/margins": 0.11457403004169464,
+      "rewards/rejected": 0.11408257484436035,
+      "step": 380
+    },
+    {
+      "epoch": 0.6382978723404256,
+      "grad_norm": 7.025293350219727,
+      "learning_rate": 4.414796846573681e-05,
+      "logits/chosen": -1.1844158172607422,
+      "logits/rejected": -1.1069843769073486,
+      "logps/chosen": -428.5450134277344,
+      "logps/rejected": -371.85614013671875,
+      "loss": 0.6921,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.2637328803539276,
+      "rewards/margins": 0.10889965295791626,
+      "rewards/rejected": 0.15483322739601135,
+      "step": 390
+    },
+    {
+      "epoch": 0.6546644844517185,
+      "grad_norm": 19.2535343170166,
+      "learning_rate": 4.3844754396604006e-05,
+      "logits/chosen": -1.1141570806503296,
+      "logits/rejected": -0.8798454999923706,
+      "logps/chosen": -493.08197021484375,
+      "logps/rejected": -465.0943298339844,
+      "loss": 0.6545,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.4174181818962097,
+      "rewards/margins": 0.25259822607040405,
+      "rewards/rejected": 0.16481998562812805,
+      "step": 400
+    },
+    {
+      "epoch": 0.6710310965630114,
+      "grad_norm": 8.808697700500488,
+      "learning_rate": 4.35415403274712e-05,
+      "logits/chosen": -0.9878439903259277,
+      "logits/rejected": -0.7299278974533081,
+      "logps/chosen": -523.9139404296875,
+      "logps/rejected": -482.0597229003906,
+      "loss": 0.552,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.4945390820503235,
+      "rewards/margins": 0.6175243854522705,
+      "rewards/rejected": -0.1229853481054306,
+      "step": 410
+    },
+    {
+      "epoch": 0.6873977086743044,
+      "grad_norm": 9.99456787109375,
+      "learning_rate": 4.323832625833839e-05,
+      "logits/chosen": -1.0902565717697144,
+      "logits/rejected": -0.9221500158309937,
+      "logps/chosen": -443.0076599121094,
+      "logps/rejected": -375.8013610839844,
+      "loss": 0.5684,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.46177297830581665,
+      "rewards/margins": 0.509793221950531,
+      "rewards/rejected": -0.048020295798778534,
+      "step": 420
+    },
+    {
+      "epoch": 0.7037643207855974,
+      "grad_norm": 5.767345428466797,
+      "learning_rate": 4.293511218920558e-05,
+      "logits/chosen": -1.0450439453125,
+      "logits/rejected": -0.7902520298957825,
+      "logps/chosen": -458.2330017089844,
+      "logps/rejected": -382.6787109375,
+      "loss": 0.6126,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.5793684124946594,
+      "rewards/margins": 0.35584205389022827,
+      "rewards/rejected": 0.22352632880210876,
+      "step": 430
+    },
+    {
+      "epoch": 0.7201309328968903,
+      "grad_norm": 11.51710033416748,
+      "learning_rate": 4.263189812007277e-05,
+      "logits/chosen": -0.8710979223251343,
+      "logits/rejected": -0.783151388168335,
+      "logps/chosen": -451.32122802734375,
+      "logps/rejected": -416.0005798339844,
+      "loss": 0.6751,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.5284687280654907,
+      "rewards/margins": 0.2338237315416336,
+      "rewards/rejected": 0.2946449816226959,
+      "step": 440
+    },
+    {
+      "epoch": 0.7364975450081833,
+      "grad_norm": 8.408599853515625,
+      "learning_rate": 4.232868405093997e-05,
+      "logits/chosen": -0.9611997604370117,
+      "logits/rejected": -0.8005222082138062,
+      "logps/chosen": -390.06817626953125,
+      "logps/rejected": -376.57183837890625,
+      "loss": 0.723,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.35626164078712463,
+      "rewards/margins": 0.07882154732942581,
+      "rewards/rejected": 0.27744007110595703,
+      "step": 450
+    },
+    {
+      "epoch": 0.7528641571194763,
+      "grad_norm": 7.085683822631836,
+      "learning_rate": 4.202546998180716e-05,
+      "logits/chosen": -1.181065559387207,
+      "logits/rejected": -0.9188621640205383,
+      "logps/chosen": -446.1695861816406,
+      "logps/rejected": -461.0537109375,
+      "loss": 0.6376,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.6099992990493774,
+      "rewards/margins": 0.40477481484413147,
+      "rewards/rejected": 0.20522448420524597,
+      "step": 460
+    },
+    {
+      "epoch": 0.7692307692307693,
+      "grad_norm": 7.778029918670654,
+      "learning_rate": 4.172225591267435e-05,
+      "logits/chosen": -1.1612136363983154,
+      "logits/rejected": -0.7909379601478577,
+      "logps/chosen": -415.356201171875,
+      "logps/rejected": -405.0163879394531,
+      "loss": 0.639,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.5554541349411011,
+      "rewards/margins": 0.49387890100479126,
+      "rewards/rejected": 0.06157524511218071,
+      "step": 470
+    },
+    {
+      "epoch": 0.7855973813420621,
+      "grad_norm": 9.144447326660156,
+      "learning_rate": 4.141904184354154e-05,
+      "logits/chosen": -1.1227318048477173,
+      "logits/rejected": -0.8980533480644226,
+      "logps/chosen": -470.1067810058594,
+      "logps/rejected": -385.7353210449219,
+      "loss": 0.6301,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.5011923909187317,
+      "rewards/margins": 0.36913150548934937,
+      "rewards/rejected": 0.1320609152317047,
+      "step": 480
+    },
+    {
+      "epoch": 0.8019639934533551,
+      "grad_norm": 9.528294563293457,
+      "learning_rate": 4.111582777440873e-05,
+      "logits/chosen": -1.4198505878448486,
+      "logits/rejected": -1.1528406143188477,
+      "logps/chosen": -430.28668212890625,
+      "logps/rejected": -385.2845764160156,
+      "loss": 0.634,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.6369161605834961,
+      "rewards/margins": 0.4485607147216797,
+      "rewards/rejected": 0.18835540115833282,
+      "step": 490
+    },
+    {
+      "epoch": 0.8183306055646481,
+      "grad_norm": 6.964690208435059,
+      "learning_rate": 4.0812613705275924e-05,
+      "logits/chosen": -1.1329933404922485,
+      "logits/rejected": -1.066542625427246,
+      "logps/chosen": -423.4967346191406,
+      "logps/rejected": -376.92803955078125,
+      "loss": 0.5904,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.6795145273208618,
+      "rewards/margins": 0.3617789149284363,
+      "rewards/rejected": 0.3177356719970703,
+      "step": 500
+    },
+    {
+      "epoch": 0.8346972176759411,
+      "grad_norm": 11.78022289276123,
+      "learning_rate": 4.050939963614312e-05,
+      "logits/chosen": -1.1565020084381104,
+      "logits/rejected": -1.064300298690796,
+      "logps/chosen": -465.0497131347656,
+      "logps/rejected": -438.4883728027344,
+      "loss": 0.6384,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.6123272180557251,
+      "rewards/margins": 0.3706541657447815,
+      "rewards/rejected": 0.241673082113266,
+      "step": 510
+    },
+    {
+      "epoch": 0.851063829787234,
+      "grad_norm": 6.378419876098633,
+      "learning_rate": 4.020618556701031e-05,
+      "logits/chosen": -1.0936065912246704,
+      "logits/rejected": -1.157414436340332,
+      "logps/chosen": -431.05657958984375,
+      "logps/rejected": -380.1803283691406,
+      "loss": 0.6866,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.514751672744751,
+      "rewards/margins": 0.2915990948677063,
+      "rewards/rejected": 0.22315259277820587,
+      "step": 520
+    },
+    {
+      "epoch": 0.867430441898527,
+      "grad_norm": 6.811694145202637,
+      "learning_rate": 3.9902971497877504e-05,
+      "logits/chosen": -1.0564637184143066,
+      "logits/rejected": -0.8537761569023132,
+      "logps/chosen": -400.8801574707031,
+      "logps/rejected": -400.9678039550781,
+      "loss": 0.6036,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.5905786156654358,
+      "rewards/margins": 0.49240899085998535,
+      "rewards/rejected": 0.09816964715719223,
+      "step": 530
+    },
+    {
+      "epoch": 0.88379705400982,
+      "grad_norm": 8.516433715820312,
+      "learning_rate": 3.9599757428744695e-05,
+      "logits/chosen": -1.0425868034362793,
+      "logits/rejected": -1.0106391906738281,
+      "logps/chosen": -494.23681640625,
+      "logps/rejected": -451.3269958496094,
+      "loss": 0.7295,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.480445921421051,
+      "rewards/margins": 0.12438355386257172,
+      "rewards/rejected": 0.3560623824596405,
+      "step": 540
+    },
+    {
+      "epoch": 0.900163666121113,
+      "grad_norm": 9.584715843200684,
+      "learning_rate": 3.9296543359611886e-05,
+      "logits/chosen": -1.1160447597503662,
+      "logits/rejected": -0.898239016532898,
+      "logps/chosen": -471.17718505859375,
+      "logps/rejected": -404.9838562011719,
+      "loss": 0.5973,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.4906229078769684,
+      "rewards/margins": 0.3674588203430176,
+      "rewards/rejected": 0.1231641173362732,
+      "step": 550
+    },
+    {
+      "epoch": 0.9165302782324058,
+      "grad_norm": 9.050358772277832,
+      "learning_rate": 3.899332929047908e-05,
+      "logits/chosen": -1.0188591480255127,
+      "logits/rejected": NaN,
+      "logps/chosen": -505.81146240234375,
+      "logps/rejected": -455.074462890625,
+      "loss": 0.672,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.5001901388168335,
+      "rewards/margins": 0.3212779462337494,
+      "rewards/rejected": 0.1789121776819229,
+      "step": 560
+    },
+    {
+      "epoch": 0.9328968903436988,
+      "grad_norm": 10.636481285095215,
+      "learning_rate": 3.8690115221346276e-05,
+      "logits/chosen": -1.0234591960906982,
+      "logits/rejected": -0.9390643835067749,
+      "logps/chosen": -423.03594970703125,
+      "logps/rejected": -383.97369384765625,
+      "loss": 0.6515,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.40482932329177856,
+      "rewards/margins": 0.20538429915905,
+      "rewards/rejected": 0.19944503903388977,
+      "step": 570
+    },
+    {
+      "epoch": 0.9492635024549918,
+      "grad_norm": 12.001391410827637,
+      "learning_rate": 3.838690115221347e-05,
+      "logits/chosen": -1.254690170288086,
+      "logits/rejected": -1.098749041557312,
+      "logps/chosen": -400.72021484375,
+      "logps/rejected": -348.39678955078125,
+      "loss": 0.6332,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.5642321705818176,
+      "rewards/margins": 0.34682947397232056,
+      "rewards/rejected": 0.21740269660949707,
+      "step": 580
+    },
+    {
+      "epoch": 0.9656301145662848,
+      "grad_norm": 9.584880828857422,
+      "learning_rate": 3.808368708308066e-05,
+      "logits/chosen": -0.9567238092422485,
+      "logits/rejected": -0.8445194959640503,
+      "logps/chosen": -475.3368225097656,
+      "logps/rejected": -418.3975524902344,
+      "loss": 0.609,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.7818068861961365,
+      "rewards/margins": 0.4438362717628479,
+      "rewards/rejected": 0.3379705548286438,
+      "step": 590
+    },
+    {
+      "epoch": 0.9819967266775778,
+      "grad_norm": 8.225208282470703,
+      "learning_rate": 3.778047301394785e-05,
+      "logits/chosen": -0.8194648623466492,
+      "logits/rejected": -0.6516283750534058,
+      "logps/chosen": -464.95648193359375,
+      "logps/rejected": -454.74896240234375,
+      "loss": 0.6792,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.673040509223938,
+      "rewards/margins": 0.3073888421058655,
+      "rewards/rejected": 0.3656516671180725,
+      "step": 600
+    },
+    {
+      "epoch": 0.9983633387888707,
+      "grad_norm": 7.938261032104492,
+      "learning_rate": 3.747725894481504e-05,
+      "logits/chosen": -0.9632278680801392,
+      "logits/rejected": -0.7192381620407104,
+      "logps/chosen": -428.2535095214844,
+      "logps/rejected": -426.05780029296875,
+      "loss": 0.5784,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.778376042842865,
+      "rewards/margins": 0.44757041335105896,
+      "rewards/rejected": 0.33080559968948364,
+      "step": 610
+    },
+    {
+      "epoch": 1.0147299509001637,
+      "grad_norm": 6.6163482666015625,
+      "learning_rate": 3.717404487568223e-05,
+      "logits/chosen": -0.8798831105232239,
+      "logits/rejected": -0.716325581073761,
+      "logps/chosen": -401.0724182128906,
+      "logps/rejected": -393.04925537109375,
+      "loss": 0.5397,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.6812083721160889,
+      "rewards/margins": 0.6590946912765503,
+      "rewards/rejected": 0.022113658487796783,
+      "step": 620
+    },
+    {
+      "epoch": 1.0310965630114566,
+      "grad_norm": 4.5942792892456055,
+      "learning_rate": 3.687083080654943e-05,
+      "logits/chosen": -0.595669150352478,
+      "logits/rejected": -0.6483780741691589,
+      "logps/chosen": -519.7013549804688,
+      "logps/rejected": -459.6736755371094,
+      "loss": 0.3798,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.9690455198287964,
+      "rewards/margins": 1.2637821435928345,
+      "rewards/rejected": -0.2947366237640381,
+      "step": 630
+    },
+    {
+      "epoch": 1.0474631751227497,
+      "grad_norm": 5.822299003601074,
+      "learning_rate": 3.6567616737416614e-05,
+      "logits/chosen": -1.1080423593521118,
+      "logits/rejected": -0.8690654635429382,
+      "logps/chosen": -430.8399963378906,
+      "logps/rejected": -382.16131591796875,
+      "loss": 0.3725,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 0.926641583442688,
+      "rewards/margins": 1.1296237707138062,
+      "rewards/rejected": -0.20298215746879578,
+      "step": 640
+    },
+    {
+      "epoch": 1.0638297872340425,
+      "grad_norm": 5.2786078453063965,
+      "learning_rate": 3.626440266828381e-05,
+      "logits/chosen": -0.986005961894989,
+      "logits/rejected": -0.8763575553894043,
+      "logps/chosen": -407.85260009765625,
+      "logps/rejected": -399.2723693847656,
+      "loss": 0.4381,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.8758468627929688,
+      "rewards/margins": 0.9750221371650696,
+      "rewards/rejected": -0.09917531907558441,
+      "step": 650
+    },
+    {
+      "epoch": 1.0801963993453354,
+      "grad_norm": 11.273237228393555,
+      "learning_rate": 3.5961188599151e-05,
+      "logits/chosen": -1.3430869579315186,
+      "logits/rejected": -1.056379795074463,
+      "logps/chosen": -410.79791259765625,
+      "logps/rejected": -434.85845947265625,
+      "loss": 0.4005,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.9130380749702454,
+      "rewards/margins": 0.9950882196426392,
+      "rewards/rejected": -0.08205002546310425,
+      "step": 660
+    },
+    {
+      "epoch": 1.0965630114566285,
+      "grad_norm": 7.220339775085449,
+      "learning_rate": 3.5657974530018194e-05,
+      "logits/chosen": -1.0506983995437622,
+      "logits/rejected": -1.0361990928649902,
+      "logps/chosen": -408.18035888671875,
+      "logps/rejected": -391.70562744140625,
+      "loss": 0.4471,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.7731181979179382,
+      "rewards/margins": 0.9366775751113892,
+      "rewards/rejected": -0.16355939209461212,
+      "step": 670
+    },
+    {
+      "epoch": 1.1129296235679214,
+      "grad_norm": 5.74124813079834,
+      "learning_rate": 3.535476046088539e-05,
+      "logits/chosen": -1.206479787826538,
+      "logits/rejected": -1.1451863050460815,
+      "logps/chosen": -444.80438232421875,
+      "logps/rejected": -439.0628356933594,
+      "loss": 0.4043,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.8405340313911438,
+      "rewards/margins": 1.1449936628341675,
+      "rewards/rejected": -0.30445969104766846,
+      "step": 680
+    },
+    {
+      "epoch": 1.1292962356792144,
+      "grad_norm": 7.32386589050293,
+      "learning_rate": 3.5051546391752576e-05,
+      "logits/chosen": -1.0227863788604736,
+      "logits/rejected": -0.9067742228507996,
+      "logps/chosen": -476.1012268066406,
+      "logps/rejected": -466.14971923828125,
+      "loss": 0.4261,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.8802071809768677,
+      "rewards/margins": 1.1421091556549072,
+      "rewards/rejected": -0.2619021236896515,
+      "step": 690
+    },
+    {
+      "epoch": 1.1456628477905073,
+      "grad_norm": 4.43737268447876,
+      "learning_rate": 3.474833232261977e-05,
+      "logits/chosen": -1.135181188583374,
+      "logits/rejected": -1.0297483205795288,
+      "logps/chosen": -506.5615234375,
+      "logps/rejected": -483.1039123535156,
+      "loss": 0.3875,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.5547564029693604,
+      "rewards/margins": 1.281551480293274,
+      "rewards/rejected": -0.7267951369285583,
+      "step": 700
+    },
+    {
+      "epoch": 1.1620294599018004,
+      "grad_norm": 2.298978328704834,
+      "learning_rate": 3.4445118253486965e-05,
+      "logits/chosen": -1.426138162612915,
+      "logits/rejected": -1.091489553451538,
+      "logps/chosen": -400.107177734375,
+      "logps/rejected": -401.62109375,
+      "loss": 0.4072,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.6946335434913635,
+      "rewards/margins": 1.441784143447876,
+      "rewards/rejected": -0.7471505999565125,
+      "step": 710
+    },
+    {
+      "epoch": 1.1783960720130933,
+      "grad_norm": 4.7108154296875,
+      "learning_rate": 3.4141904184354156e-05,
+      "logits/chosen": -1.1755582094192505,
+      "logits/rejected": -0.901213526725769,
+      "logps/chosen": -425.28759765625,
+      "logps/rejected": -402.042236328125,
+      "loss": 0.4615,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.6500431299209595,
+      "rewards/margins": 0.9347268342971802,
+      "rewards/rejected": -0.2846837639808655,
+      "step": 720
+    },
+    {
+      "epoch": 1.1947626841243864,
+      "grad_norm": 12.316296577453613,
+      "learning_rate": 3.383869011522135e-05,
+      "logits/chosen": -1.284589409828186,
+      "logits/rejected": -1.0795433521270752,
+      "logps/chosen": -413.8037109375,
+      "logps/rejected": -491.56146240234375,
+      "loss": 0.4385,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.6515102386474609,
+      "rewards/margins": 1.2281266450881958,
+      "rewards/rejected": -0.5766164660453796,
+      "step": 730
+    },
+    {
+      "epoch": 1.2111292962356792,
+      "grad_norm": 9.386224746704102,
+      "learning_rate": 3.353547604608854e-05,
+      "logits/chosen": -1.257037878036499,
+      "logits/rejected": -1.0705680847167969,
+      "logps/chosen": -505.6416931152344,
+      "logps/rejected": -459.56805419921875,
+      "loss": 0.3726,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.7937537431716919,
+      "rewards/margins": 1.4233770370483398,
+      "rewards/rejected": -0.629623293876648,
+      "step": 740
+    },
+    {
+      "epoch": 1.227495908346972,
+      "grad_norm": 6.265888214111328,
+      "learning_rate": 3.323226197695573e-05,
+      "logits/chosen": -1.332099199295044,
+      "logits/rejected": -1.0086009502410889,
+      "logps/chosen": -457.0177307128906,
+      "logps/rejected": -449.7435607910156,
+      "loss": 0.4034,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.790605366230011,
+      "rewards/margins": 1.2190719842910767,
+      "rewards/rejected": -0.4284667372703552,
+      "step": 750
+    },
+    {
+      "epoch": 1.2438625204582652,
+      "grad_norm": 6.070497035980225,
+      "learning_rate": 3.292904790782292e-05,
+      "logits/chosen": -1.2865736484527588,
+      "logits/rejected": -1.072819471359253,
+      "logps/chosen": -432.710693359375,
+      "logps/rejected": -467.6375427246094,
+      "loss": 0.3773,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.7443059682846069,
+      "rewards/margins": 1.3002400398254395,
+      "rewards/rejected": -0.5559341907501221,
+      "step": 760
+    },
+    {
+      "epoch": 1.260229132569558,
+      "grad_norm": 5.7988128662109375,
+      "learning_rate": 3.262583383869012e-05,
+      "logits/chosen": -1.0803556442260742,
+      "logits/rejected": -0.8561310768127441,
+      "logps/chosen": -514.7871704101562,
+      "logps/rejected": -503.19500732421875,
+      "loss": 0.3216,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 0.7713959217071533,
+      "rewards/margins": 1.6244882345199585,
+      "rewards/rejected": -0.8530920743942261,
+      "step": 770
+    },
+    {
+      "epoch": 1.2765957446808511,
+      "grad_norm": 4.323042869567871,
+      "learning_rate": 3.232261976955731e-05,
+      "logits/chosen": -1.408276915550232,
+      "logits/rejected": -0.9835014343261719,
+      "logps/chosen": -456.21142578125,
+      "logps/rejected": -429.10467529296875,
+      "loss": 0.3832,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.5816126465797424,
+      "rewards/margins": 1.3987071514129639,
+      "rewards/rejected": -0.8170945048332214,
+      "step": 780
+    },
+    {
+      "epoch": 1.292962356792144,
+      "grad_norm": 7.249789237976074,
+      "learning_rate": 3.20194057004245e-05,
+      "logits/chosen": -1.1671850681304932,
+      "logits/rejected": -0.653769314289093,
+      "logps/chosen": -481.74859619140625,
+      "logps/rejected": -441.5457458496094,
+      "loss": 0.3293,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 0.5319666862487793,
+      "rewards/margins": 1.7422034740447998,
+      "rewards/rejected": -1.21023690700531,
+      "step": 790
+    },
+    {
+      "epoch": 1.3093289689034369,
+      "grad_norm": 9.989459037780762,
+      "learning_rate": 3.171619163129169e-05,
+      "logits/chosen": -1.2330873012542725,
+      "logits/rejected": -1.1575677394866943,
+      "logps/chosen": -462.0821228027344,
+      "logps/rejected": -419.34149169921875,
+      "loss": 0.4737,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.7070258855819702,
+      "rewards/margins": 1.2338263988494873,
+      "rewards/rejected": -0.5268004536628723,
+      "step": 800
+    },
+    {
+      "epoch": 1.32569558101473,
+      "grad_norm": 5.914211750030518,
+      "learning_rate": 3.141297756215888e-05,
+      "logits/chosen": -1.257925271987915,
+      "logits/rejected": -1.1667311191558838,
+      "logps/chosen": -492.0726623535156,
+      "logps/rejected": -463.2273864746094,
+      "loss": 0.3597,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.7776979207992554,
+      "rewards/margins": 1.3328384160995483,
+      "rewards/rejected": -0.555140495300293,
+      "step": 810
+    },
+    {
+      "epoch": 1.342062193126023,
+      "grad_norm": 3.587632179260254,
+      "learning_rate": 3.110976349302608e-05,
+      "logits/chosen": -1.2870782613754272,
+      "logits/rejected": -0.9442241787910461,
+      "logps/chosen": -493.3888244628906,
+      "logps/rejected": -464.4835510253906,
+      "loss": 0.3151,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.9334325790405273,
+      "rewards/margins": 1.9022817611694336,
+      "rewards/rejected": -0.9688493013381958,
+      "step": 820
+    },
+    {
+      "epoch": 1.358428805237316,
+      "grad_norm": 6.462862968444824,
+      "learning_rate": 3.080654942389327e-05,
+      "logits/chosen": -1.4893635511398315,
+      "logits/rejected": -1.0626758337020874,
+      "logps/chosen": -373.52288818359375,
+      "logps/rejected": -411.831298828125,
+      "loss": 0.438,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.5565814971923828,
+      "rewards/margins": 1.1154134273529053,
+      "rewards/rejected": -0.5588318705558777,
+      "step": 830
+    },
+    {
+      "epoch": 1.3747954173486088,
+      "grad_norm": 6.482163429260254,
+      "learning_rate": 3.0503335354760464e-05,
+      "logits/chosen": -1.2426273822784424,
+      "logits/rejected": -1.0962947607040405,
+      "logps/chosen": -418.48577880859375,
+      "logps/rejected": -414.7897033691406,
+      "loss": 0.4516,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.7719891667366028,
+      "rewards/margins": 1.0017683506011963,
+      "rewards/rejected": -0.22977924346923828,
+      "step": 840
+    },
+    {
+      "epoch": 1.3911620294599019,
+      "grad_norm": 8.918207168579102,
+      "learning_rate": 3.020012128562765e-05,
+      "logits/chosen": -1.3620984554290771,
+      "logits/rejected": -1.1984447240829468,
+      "logps/chosen": -465.46270751953125,
+      "logps/rejected": -446.5343322753906,
+      "loss": 0.4053,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.6932386159896851,
+      "rewards/margins": 1.3631744384765625,
+      "rewards/rejected": -0.6699355840682983,
+      "step": 850
+    },
+    {
+      "epoch": 1.4075286415711947,
+      "grad_norm": 19.940839767456055,
+      "learning_rate": 2.9896907216494846e-05,
+      "logits/chosen": -1.2039759159088135,
+      "logits/rejected": -0.9377444386482239,
+      "logps/chosen": -574.588623046875,
+      "logps/rejected": -502.51214599609375,
+      "loss": 0.4461,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.8642657399177551,
+      "rewards/margins": 1.3982216119766235,
+      "rewards/rejected": -0.5339558124542236,
+      "step": 860
+    },
+    {
+      "epoch": 1.4238952536824878,
+      "grad_norm": 6.069481372833252,
+      "learning_rate": 2.959369314736204e-05,
+      "logits/chosen": -1.3913848400115967,
+      "logits/rejected": -1.2518432140350342,
+      "logps/chosen": -458.3941345214844,
+      "logps/rejected": -452.67352294921875,
+      "loss": 0.3604,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 1.043166160583496,
+      "rewards/margins": 1.7257499694824219,
+      "rewards/rejected": -0.6825838685035706,
+      "step": 870
+    },
+    {
+      "epoch": 1.4402618657937807,
+      "grad_norm": 3.334609031677246,
+      "learning_rate": 2.929047907822923e-05,
+      "logits/chosen": -1.3935604095458984,
+      "logits/rejected": NaN,
+      "logps/chosen": -490.9991149902344,
+      "logps/rejected": -452.0387268066406,
+      "loss": 0.3173,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 1.0780329704284668,
+      "rewards/margins": 1.919669508934021,
+      "rewards/rejected": -0.8416364789009094,
+      "step": 880
+    },
+    {
+      "epoch": 1.4566284779050735,
+      "grad_norm": 5.730930805206299,
+      "learning_rate": 2.8987265009096426e-05,
+      "logits/chosen": -1.3444249629974365,
+      "logits/rejected": -1.124793291091919,
+      "logps/chosen": -409.4859619140625,
+      "logps/rejected": -429.79150390625,
+      "loss": 0.4307,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.8416284322738647,
+      "rewards/margins": 1.227608561515808,
+      "rewards/rejected": -0.3859800696372986,
+      "step": 890
+    },
+    {
+      "epoch": 1.4729950900163666,
+      "grad_norm": 4.7208333015441895,
+      "learning_rate": 2.8684050939963614e-05,
+      "logits/chosen": -1.2483845949172974,
+      "logits/rejected": -1.162945032119751,
+      "logps/chosen": -438.02978515625,
+      "logps/rejected": -483.64776611328125,
+      "loss": 0.4359,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 0.6855508089065552,
+      "rewards/margins": 1.3711297512054443,
+      "rewards/rejected": -0.6855790019035339,
+      "step": 900
+    },
+    {
+      "epoch": 1.4893617021276595,
+      "grad_norm": 4.603919982910156,
+      "learning_rate": 2.838083687083081e-05,
+      "logits/chosen": -1.3359647989273071,
+      "logits/rejected": -1.2829469442367554,
+      "logps/chosen": -478.17022705078125,
+      "logps/rejected": -408.0987243652344,
+      "loss": 0.431,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.9056881666183472,
+      "rewards/margins": 1.124969244003296,
+      "rewards/rejected": -0.21928110718727112,
+      "step": 910
+    },
+    {
+      "epoch": 1.5057283142389526,
+      "grad_norm": 7.60992431640625,
+      "learning_rate": 2.8077622801698e-05,
+      "logits/chosen": -1.4223133325576782,
+      "logits/rejected": -1.2066584825515747,
+      "logps/chosen": -428.1543884277344,
+      "logps/rejected": -370.4458923339844,
+      "loss": 0.4003,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.8528293371200562,
+      "rewards/margins": 1.363084077835083,
+      "rewards/rejected": -0.5102548599243164,
+      "step": 920
+    },
+    {
+      "epoch": 1.5220949263502455,
+      "grad_norm": 4.768836498260498,
+      "learning_rate": 2.7774408732565194e-05,
+      "logits/chosen": -1.2342239618301392,
+      "logits/rejected": -1.1602811813354492,
+      "logps/chosen": -440.5987854003906,
+      "logps/rejected": -405.0908508300781,
+      "loss": 0.3879,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 0.9851611256599426,
+      "rewards/margins": 1.1822245121002197,
+      "rewards/rejected": -0.19706343114376068,
+      "step": 930
+    },
+    {
+      "epoch": 1.5384615384615383,
+      "grad_norm": 8.149114608764648,
+      "learning_rate": 2.7471194663432385e-05,
+      "logits/chosen": -1.1173126697540283,
+      "logits/rejected": -1.014575481414795,
+      "logps/chosen": -402.3595275878906,
+      "logps/rejected": -410.60003662109375,
+      "loss": 0.4529,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.7943775653839111,
+      "rewards/margins": 1.2599968910217285,
+      "rewards/rejected": -0.4656193256378174,
+      "step": 940
+    },
+    {
+      "epoch": 1.5548281505728314,
+      "grad_norm": 7.4743852615356445,
+      "learning_rate": 2.7167980594299573e-05,
+      "logits/chosen": NaN,
+      "logits/rejected": -1.143424391746521,
+      "logps/chosen": -446.26629638671875,
+      "logps/rejected": -469.7017517089844,
+      "loss": 0.491,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.797395646572113,
+      "rewards/margins": 1.1872708797454834,
+      "rewards/rejected": -0.38987529277801514,
+      "step": 950
+    },
+    {
+      "epoch": 1.5711947626841245,
+      "grad_norm": 8.159062385559082,
+      "learning_rate": 2.6864766525166768e-05,
+      "logits/chosen": -1.14289128780365,
+      "logits/rejected": NaN,
+      "logps/chosen": -450.77117919921875,
+      "logps/rejected": -438.8096618652344,
+      "loss": 0.4282,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.7298113107681274,
+      "rewards/margins": 1.092790126800537,
+      "rewards/rejected": -0.3629787862300873,
+      "step": 960
+    },
+    {
+      "epoch": 1.5875613747954174,
+      "grad_norm": 9.274547576904297,
+      "learning_rate": 2.6561552456033962e-05,
+      "logits/chosen": -0.9688614010810852,
+      "logits/rejected": -0.897729754447937,
+      "logps/chosen": -515.4688720703125,
+      "logps/rejected": -436.51275634765625,
+      "loss": 0.4169,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.7370742559432983,
+      "rewards/margins": 1.1543539762496948,
+      "rewards/rejected": -0.4172796607017517,
+      "step": 970
+    },
+    {
+      "epoch": 1.6039279869067102,
+      "grad_norm": 5.051360130310059,
+      "learning_rate": 2.6258338386901153e-05,
+      "logits/chosen": -1.0737905502319336,
+      "logits/rejected": -0.9251714944839478,
+      "logps/chosen": -471.31146240234375,
+      "logps/rejected": -432.572509765625,
+      "loss": 0.3841,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.8669958114624023,
+      "rewards/margins": 1.4004991054534912,
+      "rewards/rejected": -0.5335032939910889,
+      "step": 980
+    },
+    {
+      "epoch": 1.6202945990180033,
+      "grad_norm": 6.768540859222412,
+      "learning_rate": 2.5955124317768348e-05,
+      "logits/chosen": -1.1147942543029785,
+      "logits/rejected": -1.1377754211425781,
+      "logps/chosen": -462.5465393066406,
+      "logps/rejected": -390.2607727050781,
+      "loss": 0.3919,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.8462246060371399,
+      "rewards/margins": 1.2740905284881592,
+      "rewards/rejected": -0.4278659224510193,
+      "step": 990
+    },
+    {
+      "epoch": 1.6366612111292962,
+      "grad_norm": 8.313865661621094,
+      "learning_rate": 2.5651910248635535e-05,
+      "logits/chosen": -1.0041131973266602,
+      "logits/rejected": -0.8046473264694214,
+      "logps/chosen": -461.7525939941406,
+      "logps/rejected": -443.0941467285156,
+      "loss": 0.4222,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.8126258850097656,
+      "rewards/margins": 1.1657203435897827,
+      "rewards/rejected": -0.3530944585800171,
+      "step": 1000
+    },
+    {
+      "epoch": 1.6530278232405893,
+      "grad_norm": 11.539511680603027,
+      "learning_rate": 2.534869617950273e-05,
+      "logits/chosen": -1.2518326044082642,
+      "logits/rejected": -0.9322371482849121,
+      "logps/chosen": -477.0799255371094,
+      "logps/rejected": -450.06976318359375,
+      "loss": 0.3355,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.9717556834220886,
+      "rewards/margins": 1.7565772533416748,
+      "rewards/rejected": -0.784821629524231,
+      "step": 1010
+    },
+    {
+      "epoch": 1.6693944353518821,
+      "grad_norm": 8.115518569946289,
+      "learning_rate": 2.504548211036992e-05,
+      "logits/chosen": -1.4984604120254517,
+      "logits/rejected": -1.33262038230896,
+      "logps/chosen": -469.9319763183594,
+      "logps/rejected": -408.9674072265625,
+      "loss": 0.3737,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.8488012552261353,
+      "rewards/margins": 1.4102153778076172,
+      "rewards/rejected": -0.5614141225814819,
+      "step": 1020
+    },
+    {
+      "epoch": 1.685761047463175,
+      "grad_norm": 7.375257968902588,
+      "learning_rate": 2.4742268041237116e-05,
+      "logits/chosen": -1.078918695449829,
+      "logits/rejected": -0.8934460878372192,
+      "logps/chosen": -492.44769287109375,
+      "logps/rejected": -451.58795166015625,
+      "loss": 0.3332,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 1.0642842054367065,
+      "rewards/margins": 1.744214415550232,
+      "rewards/rejected": -0.6799300909042358,
+      "step": 1030
+    },
+    {
+      "epoch": 1.702127659574468,
+      "grad_norm": 3.4032139778137207,
+      "learning_rate": 2.4439053972104307e-05,
+      "logits/chosen": -1.1205917596817017,
+      "logits/rejected": -0.8298323750495911,
+      "logps/chosen": -420.125,
+      "logps/rejected": -380.03729248046875,
+      "loss": 0.4017,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.7596099376678467,
+      "rewards/margins": 1.2399921417236328,
+      "rewards/rejected": -0.48038214445114136,
+      "step": 1040
+    },
+    {
+      "epoch": 1.7184942716857612,
+      "grad_norm": 3.1656646728515625,
+      "learning_rate": 2.4135839902971498e-05,
+      "logits/chosen": -1.0980288982391357,
+      "logits/rejected": -0.9042747616767883,
+      "logps/chosen": -448.61029052734375,
+      "logps/rejected": -430.3683166503906,
+      "loss": 0.3857,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.9228833317756653,
+      "rewards/margins": 1.3001716136932373,
+      "rewards/rejected": -0.37728843092918396,
+      "step": 1050
+    },
+    {
+      "epoch": 1.734860883797054,
+      "grad_norm": 6.291409015655518,
+      "learning_rate": 2.3832625833838693e-05,
+      "logits/chosen": -1.2553077936172485,
+      "logits/rejected": -0.9304355382919312,
+      "logps/chosen": -443.744140625,
+      "logps/rejected": -375.67840576171875,
+      "loss": 0.4043,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.7896615862846375,
+      "rewards/margins": 1.1822974681854248,
+      "rewards/rejected": -0.39263591170310974,
+      "step": 1060
+    },
+    {
+      "epoch": 1.751227495908347,
+      "grad_norm": 4.611414909362793,
+      "learning_rate": 2.3529411764705884e-05,
+      "logits/chosen": -1.3921210765838623,
+      "logits/rejected": -1.2024405002593994,
+      "logps/chosen": -438.8999938964844,
+      "logps/rejected": -379.8556213378906,
+      "loss": 0.3666,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.7746474742889404,
+      "rewards/margins": 1.3105146884918213,
+      "rewards/rejected": -0.5358672142028809,
+      "step": 1070
+    },
+    {
+      "epoch": 1.7675941080196398,
+      "grad_norm": 2.6192657947540283,
+      "learning_rate": 2.3226197695573075e-05,
+      "logits/chosen": -1.2743886709213257,
+      "logits/rejected": -1.193303108215332,
+      "logps/chosen": -469.51220703125,
+      "logps/rejected": -446.6626892089844,
+      "loss": 0.4615,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.9081875085830688,
+      "rewards/margins": 1.3380632400512695,
+      "rewards/rejected": -0.4298757016658783,
+      "step": 1080
+    },
+    {
+      "epoch": 1.7839607201309329,
+      "grad_norm": 5.901863098144531,
+      "learning_rate": 2.2922983626440266e-05,
+      "logits/chosen": -1.249284267425537,
+      "logits/rejected": -1.0896720886230469,
+      "logps/chosen": -388.6412048339844,
+      "logps/rejected": -381.84747314453125,
+      "loss": 0.4718,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.62062668800354,
+      "rewards/margins": 1.1487479209899902,
+      "rewards/rejected": -0.5281211733818054,
+      "step": 1090
+    },
+    {
+      "epoch": 1.800327332242226,
+      "grad_norm": 4.81393575668335,
+      "learning_rate": 2.261976955730746e-05,
+      "logits/chosen": -1.2179973125457764,
+      "logits/rejected": -1.1146996021270752,
+      "logps/chosen": -432.94970703125,
+      "logps/rejected": -404.9814453125,
+      "loss": 0.4061,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.49649685621261597,
+      "rewards/margins": 1.0791733264923096,
+      "rewards/rejected": -0.5826765298843384,
+      "step": 1100
+    },
+    {
+      "epoch": 1.8166939443535188,
+      "grad_norm": 7.300076484680176,
+      "learning_rate": 2.2316555488174655e-05,
+      "logits/chosen": -1.4200313091278076,
+      "logits/rejected": -1.3520318269729614,
+      "logps/chosen": -398.1583557128906,
+      "logps/rejected": -411.41864013671875,
+      "loss": 0.4397,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.8094931840896606,
+      "rewards/margins": 1.2237316370010376,
+      "rewards/rejected": -0.41423845291137695,
+      "step": 1110
+    },
+    {
+      "epoch": 1.8330605564648117,
+      "grad_norm": 5.185066223144531,
+      "learning_rate": 2.2013341419041843e-05,
+      "logits/chosen": -1.282606840133667,
+      "logits/rejected": -1.3600891828536987,
+      "logps/chosen": -397.7345275878906,
+      "logps/rejected": -368.4806823730469,
+      "loss": 0.4605,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.7671560049057007,
+      "rewards/margins": 0.9713778495788574,
+      "rewards/rejected": -0.20422179996967316,
+      "step": 1120
+    },
+    {
+      "epoch": 1.8494271685761048,
+      "grad_norm": 6.64644718170166,
+      "learning_rate": 2.1710127349909037e-05,
+      "logits/chosen": -1.309072732925415,
+      "logits/rejected": -1.237240195274353,
+      "logps/chosen": -508.67864990234375,
+      "logps/rejected": -486.6451110839844,
+      "loss": 0.3327,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 1.003493070602417,
+      "rewards/margins": 1.6196180582046509,
+      "rewards/rejected": -0.6161248683929443,
+      "step": 1130
+    },
+    {
+      "epoch": 1.8657937806873979,
+      "grad_norm": 8.295884132385254,
+      "learning_rate": 2.140691328077623e-05,
+      "logits/chosen": -1.2569806575775146,
+      "logits/rejected": -1.0191644430160522,
+      "logps/chosen": -450.56884765625,
+      "logps/rejected": -396.7924499511719,
+      "loss": 0.4093,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.6329465508460999,
+      "rewards/margins": 1.3668501377105713,
+      "rewards/rejected": -0.7339035868644714,
+      "step": 1140
+    },
+    {
+      "epoch": 1.8821603927986907,
+      "grad_norm": 5.191346168518066,
+      "learning_rate": 2.110369921164342e-05,
+      "logits/chosen": -1.2032971382141113,
+      "logits/rejected": -0.9198576211929321,
+      "logps/chosen": -401.7949523925781,
+      "logps/rejected": -387.2808532714844,
+      "loss": 0.3817,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.5356651544570923,
+      "rewards/margins": 1.356689214706421,
+      "rewards/rejected": -0.8210241198539734,
+      "step": 1150
+    },
+    {
+      "epoch": 1.8985270049099836,
+      "grad_norm": 4.5822367668151855,
+      "learning_rate": 2.0800485142510614e-05,
+      "logits/chosen": -1.16325843334198,
+      "logits/rejected": -0.8339241147041321,
+      "logps/chosen": -385.2162170410156,
+      "logps/rejected": -399.42401123046875,
+      "loss": 0.4329,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.5546852946281433,
+      "rewards/margins": 1.410249948501587,
+      "rewards/rejected": -0.8555648922920227,
+      "step": 1160
+    },
+    {
+      "epoch": 1.9148936170212765,
+      "grad_norm": 7.598694324493408,
+      "learning_rate": 2.0497271073377805e-05,
+      "logits/chosen": -1.2841596603393555,
+      "logits/rejected": -1.0715458393096924,
+      "logps/chosen": -426.3335876464844,
+      "logps/rejected": -392.77947998046875,
+      "loss": 0.3999,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.6999267339706421,
+      "rewards/margins": 1.2335584163665771,
+      "rewards/rejected": -0.5336317420005798,
+      "step": 1170
+    },
+    {
+      "epoch": 1.9312602291325696,
+      "grad_norm": 12.754505157470703,
+      "learning_rate": 2.0194057004245e-05,
+      "logits/chosen": -1.2506614923477173,
+      "logits/rejected": -1.1044342517852783,
+      "logps/chosen": -464.2216796875,
+      "logps/rejected": -420.13336181640625,
+      "loss": 0.4258,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.7730525135993958,
+      "rewards/margins": 1.1193318367004395,
+      "rewards/rejected": -0.3462793827056885,
+      "step": 1180
+    },
+    {
+      "epoch": 1.9476268412438626,
+      "grad_norm": 5.040411949157715,
+      "learning_rate": 1.9890842935112188e-05,
+      "logits/chosen": -1.5200366973876953,
+      "logits/rejected": -1.453028917312622,
+      "logps/chosen": -471.79180908203125,
+      "logps/rejected": -421.40899658203125,
+      "loss": 0.3997,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.7299527525901794,
+      "rewards/margins": 1.2998378276824951,
+      "rewards/rejected": -0.56988525390625,
+      "step": 1190
+    },
+    {
+      "epoch": 1.9639934533551555,
+      "grad_norm": 12.064972877502441,
+      "learning_rate": 1.9587628865979382e-05,
+      "logits/chosen": -1.2272264957427979,
+      "logits/rejected": -0.9471932649612427,
+      "logps/chosen": -501.19842529296875,
+      "logps/rejected": -451.31658935546875,
+      "loss": 0.4069,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.8413068056106567,
+      "rewards/margins": 1.3548190593719482,
+      "rewards/rejected": -0.5135123133659363,
+      "step": 1200
+    },
+    {
+      "epoch": 1.9803600654664484,
+      "grad_norm": 7.827969551086426,
+      "learning_rate": 1.9284414796846577e-05,
+      "logits/chosen": -1.1897993087768555,
+      "logits/rejected": -1.085766077041626,
+      "logps/chosen": -467.1263122558594,
+      "logps/rejected": -420.46417236328125,
+      "loss": 0.4029,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.8640726208686829,
+      "rewards/margins": 1.2383184432983398,
+      "rewards/rejected": -0.3742457330226898,
+      "step": 1210
+    },
+    {
+      "epoch": 1.9967266775777412,
+      "grad_norm": 4.033712863922119,
+      "learning_rate": 1.8981200727713764e-05,
+      "logits/chosen": -1.1938722133636475,
+      "logits/rejected": -0.7925646305084229,
+      "logps/chosen": -434.82916259765625,
+      "logps/rejected": -415.64178466796875,
+      "loss": 0.4062,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 0.6897398233413696,
+      "rewards/margins": 1.3555059432983398,
+      "rewards/rejected": -0.6657661199569702,
+      "step": 1220
+    },
+    {
+      "epoch": 2.0130932896890346,
+      "grad_norm": 2.060443162918091,
+      "learning_rate": 1.867798665858096e-05,
+      "logits/chosen": NaN,
+      "logits/rejected": -1.3225266933441162,
+      "logps/chosen": -402.45654296875,
+      "logps/rejected": -432.16448974609375,
+      "loss": 0.3632,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.8448646664619446,
+      "rewards/margins": 1.4718005657196045,
+      "rewards/rejected": -0.6269359588623047,
+      "step": 1230
+    },
+    {
+      "epoch": 2.0294599018003274,
+      "grad_norm": 2.7568705081939697,
+      "learning_rate": 1.8374772589448153e-05,
+      "logits/chosen": -1.589988112449646,
+      "logits/rejected": -1.4496564865112305,
+      "logps/chosen": -415.6414489746094,
+      "logps/rejected": -382.269287109375,
+      "loss": 0.3286,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 1.015729308128357,
+      "rewards/margins": 1.6888082027435303,
+      "rewards/rejected": -0.6730789542198181,
+      "step": 1240
+    },
+    {
+      "epoch": 2.0458265139116203,
+      "grad_norm": 7.693350315093994,
+      "learning_rate": 1.8071558520315345e-05,
+      "logits/chosen": -1.2396166324615479,
+      "logits/rejected": -1.0914833545684814,
+      "logps/chosen": -428.75146484375,
+      "logps/rejected": -439.94537353515625,
+      "loss": 0.3436,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.9149911999702454,
+      "rewards/margins": 1.669695496559143,
+      "rewards/rejected": -0.7547044157981873,
+      "step": 1250
+    },
+    {
+      "epoch": 2.062193126022913,
+      "grad_norm": 6.800998210906982,
+      "learning_rate": 1.7768344451182536e-05,
+      "logits/chosen": -1.3096355199813843,
+      "logits/rejected": -1.2225430011749268,
+      "logps/chosen": -544.235595703125,
+      "logps/rejected": -471.3172302246094,
+      "loss": 0.2387,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 1.1924231052398682,
+      "rewards/margins": 1.9907211065292358,
+      "rewards/rejected": -0.7982980608940125,
+      "step": 1260
+    },
+    {
+      "epoch": 2.078559738134206,
+      "grad_norm": 2.9473204612731934,
+      "learning_rate": 1.7465130382049727e-05,
+      "logits/chosen": -1.5475558042526245,
+      "logits/rejected": -1.2119872570037842,
+      "logps/chosen": -425.4368591308594,
+      "logps/rejected": -431.3377990722656,
+      "loss": 0.1952,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 1.1395524740219116,
+      "rewards/margins": 2.476022243499756,
+      "rewards/rejected": -1.3364696502685547,
+      "step": 1270
+    },
+    {
+      "epoch": 2.0949263502454993,
+      "grad_norm": 5.591909885406494,
+      "learning_rate": 1.716191631291692e-05,
+      "logits/chosen": -1.5607810020446777,
+      "logits/rejected": -1.4012185335159302,
+      "logps/chosen": -413.8076171875,
+      "logps/rejected": -411.9990234375,
+      "loss": 0.2864,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 1.065088152885437,
+      "rewards/margins": 1.9613841772079468,
+      "rewards/rejected": -0.8962962031364441,
+      "step": 1280
+    },
+    {
+      "epoch": 2.111292962356792,
+      "grad_norm": 4.462930679321289,
+      "learning_rate": 1.6858702243784113e-05,
+      "logits/chosen": -1.4985883235931396,
+      "logits/rejected": -1.2055768966674805,
+      "logps/chosen": -402.1160583496094,
+      "logps/rejected": -397.81402587890625,
+      "loss": 0.2957,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.8168094754219055,
+      "rewards/margins": 1.9579219818115234,
+      "rewards/rejected": -1.1411124467849731,
+      "step": 1290
+    },
+    {
+      "epoch": 2.127659574468085,
+      "grad_norm": 3.3461086750030518,
+      "learning_rate": 1.6555488174651304e-05,
+      "logits/chosen": -1.5269520282745361,
+      "logits/rejected": -1.0539462566375732,
+      "logps/chosen": -507.36834716796875,
+      "logps/rejected": -461.356689453125,
+      "loss": 0.2565,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 0.6355259418487549,
+      "rewards/margins": 2.068328380584717,
+      "rewards/rejected": -1.432802438735962,
+      "step": 1300
+    },
+    {
+      "epoch": 2.144026186579378,
+      "grad_norm": 4.208719253540039,
+      "learning_rate": 1.6252274105518498e-05,
+      "logits/chosen": -1.403276801109314,
+      "logits/rejected": -1.199331283569336,
+      "logps/chosen": -427.9095153808594,
+      "logps/rejected": -373.31756591796875,
+      "loss": 0.2796,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 0.8879494667053223,
+      "rewards/margins": 1.912683129310608,
+      "rewards/rejected": -1.0247336626052856,
+      "step": 1310
+    },
+    {
+      "epoch": 2.160392798690671,
+      "grad_norm": 3.667640447616577,
+      "learning_rate": 1.594906003638569e-05,
+      "logits/chosen": -1.4091194868087769,
+      "logits/rejected": -1.123011827468872,
+      "logps/chosen": -457.79461669921875,
+      "logps/rejected": -444.686279296875,
+      "loss": 0.2564,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 0.8721135258674622,
+      "rewards/margins": 2.0980000495910645,
+      "rewards/rejected": -1.2258864641189575,
+      "step": 1320
+    },
+    {
+      "epoch": 2.176759410801964,
+      "grad_norm": 2.9427664279937744,
+      "learning_rate": 1.564584596725288e-05,
+      "logits/chosen": -1.4273817539215088,
+      "logits/rejected": -1.2326085567474365,
+      "logps/chosen": -431.9471130371094,
+      "logps/rejected": -421.22747802734375,
+      "loss": 0.3017,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 0.7032436728477478,
+      "rewards/margins": 1.7488292455673218,
+      "rewards/rejected": -1.0455853939056396,
+      "step": 1330
+    },
+    {
+      "epoch": 2.193126022913257,
+      "grad_norm": 3.592177629470825,
+      "learning_rate": 1.5342631898120075e-05,
+      "logits/chosen": -1.3208458423614502,
+      "logits/rejected": -1.1550036668777466,
+      "logps/chosen": -442.4117736816406,
+      "logps/rejected": -370.07708740234375,
+      "loss": 0.3644,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.9467415809631348,
+      "rewards/margins": 1.7952324151992798,
+      "rewards/rejected": -0.8484910726547241,
+      "step": 1340
+    },
+    {
+      "epoch": 2.20949263502455,
+      "grad_norm": 2.6388635635375977,
+      "learning_rate": 1.5039417828987265e-05,
+      "logits/chosen": -1.305037021636963,
+      "logits/rejected": -1.2544772624969482,
+      "logps/chosen": -438.3041076660156,
+      "logps/rejected": -412.14923095703125,
+      "loss": 0.2553,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 1.2883623838424683,
+      "rewards/margins": 1.9844329357147217,
+      "rewards/rejected": -0.6960704326629639,
+      "step": 1350
+    },
+    {
+      "epoch": 2.2258592471358427,
+      "grad_norm": 5.4221954345703125,
+      "learning_rate": 1.4736203759854459e-05,
+      "logits/chosen": -1.3623788356781006,
+      "logits/rejected": -1.030826210975647,
+      "logps/chosen": -437.98150634765625,
+      "logps/rejected": -413.3927307128906,
+      "loss": 0.3294,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.8824920654296875,
+      "rewards/margins": 1.7985508441925049,
+      "rewards/rejected": -0.9160588383674622,
+      "step": 1360
+    },
+    {
+      "epoch": 2.242225859247136,
+      "grad_norm": 1.4540634155273438,
+      "learning_rate": 1.4432989690721649e-05,
+      "logits/chosen": -1.4040526151657104,
+      "logits/rejected": -1.2125266790390015,
+      "logps/chosen": -444.2418518066406,
+      "logps/rejected": -445.39154052734375,
+      "loss": 0.2202,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 1.2369893789291382,
+      "rewards/margins": 2.244361162185669,
+      "rewards/rejected": -1.0073716640472412,
+      "step": 1370
+    },
+    {
+      "epoch": 2.258592471358429,
+      "grad_norm": 5.328054904937744,
+      "learning_rate": 1.4129775621588841e-05,
+      "logits/chosen": -1.1481372117996216,
+      "logits/rejected": -1.1810128688812256,
+      "logps/chosen": -483.92901611328125,
+      "logps/rejected": -435.12408447265625,
+      "loss": 0.2465,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 1.3297772407531738,
+      "rewards/margins": 2.0740952491760254,
+      "rewards/rejected": -0.7443181872367859,
+      "step": 1380
+    },
+    {
+      "epoch": 2.2749590834697218,
+      "grad_norm": 5.371053695678711,
+      "learning_rate": 1.3826561552456036e-05,
+      "logits/chosen": -1.341086506843567,
+      "logits/rejected": -1.2301025390625,
+      "logps/chosen": -441.9823303222656,
+      "logps/rejected": -371.228271484375,
+      "loss": 0.2796,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 1.0860683917999268,
+      "rewards/margins": 1.9399855136871338,
+      "rewards/rejected": -0.853917121887207,
+      "step": 1390
+    },
+    {
+      "epoch": 2.2913256955810146,
+      "grad_norm": 4.199862480163574,
+      "learning_rate": 1.3523347483323225e-05,
+      "logits/chosen": -1.2742938995361328,
+      "logits/rejected": -1.0367982387542725,
+      "logps/chosen": -441.23193359375,
+      "logps/rejected": -432.3246154785156,
+      "loss": 0.2161,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 1.0744221210479736,
+      "rewards/margins": 2.1862921714782715,
+      "rewards/rejected": -1.1118695735931396,
+      "step": 1400
+    },
+    {
+      "epoch": 2.3076923076923075,
+      "grad_norm": 4.229262351989746,
+      "learning_rate": 1.322013341419042e-05,
+      "logits/chosen": -1.4364268779754639,
+      "logits/rejected": -1.0992625951766968,
+      "logps/chosen": -425.8427734375,
+      "logps/rejected": -459.83642578125,
+      "loss": 0.2952,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.9090694189071655,
+      "rewards/margins": 1.8446950912475586,
+      "rewards/rejected": -0.9356255531311035,
+      "step": 1410
+    },
+    {
+      "epoch": 2.324058919803601,
+      "grad_norm": 2.794767379760742,
+      "learning_rate": 1.2916919345057613e-05,
+      "logits/chosen": -1.354282021522522,
+      "logits/rejected": -1.2035588026046753,
+      "logps/chosen": -420.01800537109375,
+      "logps/rejected": -375.9340515136719,
+      "loss": 0.2299,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 1.1438004970550537,
+      "rewards/margins": 2.1549389362335205,
+      "rewards/rejected": -1.0111383199691772,
+      "step": 1420
+    },
+    {
+      "epoch": 2.3404255319148937,
+      "grad_norm": 1.970015525817871,
+      "learning_rate": 1.2613705275924804e-05,
+      "logits/chosen": -1.3893083333969116,
+      "logits/rejected": -1.3427255153656006,
+      "logps/chosen": -428.79400634765625,
+      "logps/rejected": -409.81500244140625,
+      "loss": 0.2522,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 1.1792007684707642,
+      "rewards/margins": 2.0457987785339355,
+      "rewards/rejected": -0.8665979504585266,
+      "step": 1430
+    },
+    {
+      "epoch": 2.3567921440261865,
+      "grad_norm": 3.0225942134857178,
+      "learning_rate": 1.2310491206791997e-05,
+      "logits/chosen": -1.5218456983566284,
+      "logits/rejected": -1.236580491065979,
+      "logps/chosen": -474.56829833984375,
+      "logps/rejected": -513.43408203125,
+      "loss": 0.2505,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 1.0795310735702515,
+      "rewards/margins": 2.1252081394195557,
+      "rewards/rejected": -1.0456770658493042,
+      "step": 1440
+    },
+    {
+      "epoch": 2.3731587561374794,
+      "grad_norm": 4.234305381774902,
+      "learning_rate": 1.2007277137659188e-05,
+      "logits/chosen": -1.3646624088287354,
+      "logits/rejected": -1.2389724254608154,
+      "logps/chosen": -431.1107482910156,
+      "logps/rejected": -442.53997802734375,
+      "loss": 0.3115,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.8404763340950012,
+      "rewards/margins": 2.168391466140747,
+      "rewards/rejected": -1.3279149532318115,
+      "step": 1450
+    },
+    {
+      "epoch": 2.3895253682487727,
+      "grad_norm": 9.905987739562988,
+      "learning_rate": 1.170406306852638e-05,
+      "logits/chosen": -1.4754550457000732,
+      "logits/rejected": -1.4108117818832397,
+      "logps/chosen": -453.54364013671875,
+      "logps/rejected": -432.04693603515625,
+      "loss": 0.3368,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.6598278284072876,
+      "rewards/margins": 1.6955798864364624,
+      "rewards/rejected": -1.0357519388198853,
+      "step": 1460
+    },
+    {
+      "epoch": 2.4058919803600656,
+      "grad_norm": 4.8183746337890625,
+      "learning_rate": 1.1400848999393572e-05,
+      "logits/chosen": -1.3298962116241455,
+      "logits/rejected": -1.2613223791122437,
+      "logps/chosen": -480.92205810546875,
+      "logps/rejected": -446.716796875,
+      "loss": 0.3334,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 0.8991876840591431,
+      "rewards/margins": 1.7357298135757446,
+      "rewards/rejected": -0.8365424275398254,
+      "step": 1470
+    },
+    {
+      "epoch": 2.4222585924713584,
+      "grad_norm": 6.064842700958252,
+      "learning_rate": 1.1097634930260765e-05,
+      "logits/chosen": -1.4571373462677002,
+      "logits/rejected": -1.1927803754806519,
+      "logps/chosen": -447.56787109375,
+      "logps/rejected": -440.72064208984375,
+      "loss": 0.2964,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 1.0991294384002686,
+      "rewards/margins": 2.0829381942749023,
+      "rewards/rejected": -0.983808696269989,
+      "step": 1480
+    },
+    {
+      "epoch": 2.4386252045826513,
+      "grad_norm": 2.8285984992980957,
+      "learning_rate": 1.0794420861127958e-05,
+      "logits/chosen": -1.4223463535308838,
+      "logits/rejected": -1.2678343057632446,
+      "logps/chosen": -438.255126953125,
+      "logps/rejected": -454.03948974609375,
+      "loss": 0.2041,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 1.2381608486175537,
+      "rewards/margins": 2.366389036178589,
+      "rewards/rejected": -1.1282285451889038,
+      "step": 1490
+    },
+    {
+      "epoch": 2.454991816693944,
+      "grad_norm": 3.3027286529541016,
+      "learning_rate": 1.0491206791995149e-05,
+      "logits/chosen": -1.3270039558410645,
+      "logits/rejected": -1.2212953567504883,
+      "logps/chosen": -447.6775817871094,
+      "logps/rejected": -454.34918212890625,
+      "loss": 0.246,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 0.9780976176261902,
+      "rewards/margins": 1.9729305505752563,
+      "rewards/rejected": -0.9948328137397766,
+      "step": 1500
+    },
+    {
+      "epoch": 2.4713584288052375,
+      "grad_norm": 5.344712734222412,
+      "learning_rate": 1.0187992722862342e-05,
+      "logits/chosen": -1.2262275218963623,
+      "logits/rejected": -1.0121392011642456,
+      "logps/chosen": -464.63897705078125,
+      "logps/rejected": -446.21954345703125,
+      "loss": 0.2936,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 1.0501153469085693,
+      "rewards/margins": 2.157829999923706,
+      "rewards/rejected": -1.1077148914337158,
+      "step": 1510
+    },
+    {
+      "epoch": 2.4877250409165304,
+      "grad_norm": 3.925086259841919,
+      "learning_rate": 9.884778653729533e-06,
+      "logits/chosen": -1.2609909772872925,
+      "logits/rejected": -1.3114054203033447,
+      "logps/chosen": -483.9059143066406,
+      "logps/rejected": -450.318115234375,
+      "loss": 0.3007,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 1.0718286037445068,
+      "rewards/margins": 1.886443853378296,
+      "rewards/rejected": -0.8146154284477234,
+      "step": 1520
+    },
+    {
+      "epoch": 2.504091653027823,
+      "grad_norm": 5.031212329864502,
+      "learning_rate": 9.581564584596727e-06,
+      "logits/chosen": -1.3080980777740479,
+      "logits/rejected": -1.1388887166976929,
+      "logps/chosen": -433.4994201660156,
+      "logps/rejected": -423.66961669921875,
+      "loss": 0.2731,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 0.9743747711181641,
+      "rewards/margins": 1.9618078470230103,
+      "rewards/rejected": -0.9874328374862671,
+      "step": 1530
+    },
+    {
+      "epoch": 2.520458265139116,
+      "grad_norm": 4.041630744934082,
+      "learning_rate": 9.278350515463918e-06,
+      "logits/chosen": -1.4856176376342773,
+      "logits/rejected": -1.3291361331939697,
+      "logps/chosen": -454.8204040527344,
+      "logps/rejected": -457.5218811035156,
+      "loss": 0.217,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 0.9250513911247253,
+      "rewards/margins": 2.4876701831817627,
+      "rewards/rejected": -1.562618613243103,
+      "step": 1540
+    },
+    {
+      "epoch": 2.5368248772504094,
+      "grad_norm": 9.800390243530273,
+      "learning_rate": 8.97513644633111e-06,
+      "logits/chosen": -1.6522566080093384,
+      "logits/rejected": -1.5452460050582886,
+      "logps/chosen": -444.83001708984375,
+      "logps/rejected": -452.7578125,
+      "loss": 0.2926,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.9433294534683228,
+      "rewards/margins": 1.8863716125488281,
+      "rewards/rejected": -0.943041980266571,
+      "step": 1550
+    },
+    {
+      "epoch": 2.5531914893617023,
+      "grad_norm": 4.188653945922852,
+      "learning_rate": 8.671922377198302e-06,
+      "logits/chosen": -1.1361147165298462,
+      "logits/rejected": -1.007997751235962,
+      "logps/chosen": -455.88995361328125,
+      "logps/rejected": -436.4647521972656,
+      "loss": 0.3054,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.9836293458938599,
+      "rewards/margins": 2.064480781555176,
+      "rewards/rejected": -1.0808513164520264,
+      "step": 1560
+    },
+    {
+      "epoch": 2.569558101472995,
+      "grad_norm": 3.1928329467773438,
+      "learning_rate": 8.368708308065494e-06,
+      "logits/chosen": -1.5517833232879639,
+      "logits/rejected": -1.3304557800292969,
+      "logps/chosen": -412.3135681152344,
+      "logps/rejected": -410.6241149902344,
+      "loss": 0.292,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.6604477167129517,
+      "rewards/margins": 1.9019458293914795,
+      "rewards/rejected": -1.2414982318878174,
+      "step": 1570
+    },
+    {
+      "epoch": 2.585924713584288,
+      "grad_norm": 4.983669757843018,
+      "learning_rate": 8.065494238932688e-06,
+      "logits/chosen": -1.5554888248443604,
+      "logits/rejected": -1.4289400577545166,
+      "logps/chosen": -402.4801025390625,
+      "logps/rejected": -390.77130126953125,
+      "loss": 0.3494,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.9371005296707153,
+      "rewards/margins": 1.8578503131866455,
+      "rewards/rejected": -0.9207497835159302,
+      "step": 1580
+    },
+    {
+      "epoch": 2.602291325695581,
+      "grad_norm": 2.0281553268432617,
+      "learning_rate": 7.76228016979988e-06,
+      "logits/chosen": -1.1428009271621704,
+      "logits/rejected": -0.8446558713912964,
+      "logps/chosen": -462.48895263671875,
+      "logps/rejected": -451.9178161621094,
+      "loss": 0.2745,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.9884368777275085,
+      "rewards/margins": 2.026540756225586,
+      "rewards/rejected": -1.0381040573120117,
+      "step": 1590
+    },
+    {
+      "epoch": 2.6186579378068737,
+      "grad_norm": 2.423633337020874,
+      "learning_rate": 7.459066100667071e-06,
+      "logits/chosen": -1.3738397359848022,
+      "logits/rejected": -1.1381474733352661,
+      "logps/chosen": -450.98870849609375,
+      "logps/rejected": -426.8583068847656,
+      "loss": 0.2262,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 1.0699903964996338,
+      "rewards/margins": 2.1258344650268555,
+      "rewards/rejected": -1.0558441877365112,
+      "step": 1600
+    },
+    {
+      "epoch": 2.635024549918167,
+      "grad_norm": 4.554977893829346,
+      "learning_rate": 7.155852031534263e-06,
+      "logits/chosen": -1.4370437860488892,
+      "logits/rejected": -1.2265866994857788,
+      "logps/chosen": -460.3763732910156,
+      "logps/rejected": -427.2916564941406,
+      "loss": 0.2898,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 0.8689700961112976,
+      "rewards/margins": 2.047645092010498,
+      "rewards/rejected": -1.1786749362945557,
+      "step": 1610
+    },
+    {
+      "epoch": 2.65139116202946,
+      "grad_norm": 5.240372180938721,
+      "learning_rate": 6.852637962401456e-06,
+      "logits/chosen": -1.2221819162368774,
+      "logits/rejected": -0.9715522527694702,
+      "logps/chosen": -443.93487548828125,
+      "logps/rejected": -417.72918701171875,
+      "loss": 0.2262,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 1.062182903289795,
+      "rewards/margins": 2.2453255653381348,
+      "rewards/rejected": -1.1831427812576294,
+      "step": 1620
+    },
+    {
+      "epoch": 2.6677577741407528,
+      "grad_norm": 2.34958553314209,
+      "learning_rate": 6.549423893268648e-06,
+      "logits/chosen": -1.2060081958770752,
+      "logits/rejected": -1.2046623229980469,
+      "logps/chosen": -474.5213317871094,
+      "logps/rejected": -523.4539794921875,
+      "loss": 0.2484,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 0.9350897669792175,
+      "rewards/margins": 2.201655149459839,
+      "rewards/rejected": -1.2665654420852661,
+      "step": 1630
+    },
+    {
+      "epoch": 2.684124386252046,
+      "grad_norm": 1.095371961593628,
+      "learning_rate": 6.24620982413584e-06,
+      "logits/chosen": -1.0618741512298584,
+      "logits/rejected": -0.699921727180481,
+      "logps/chosen": -463.51556396484375,
+      "logps/rejected": -431.7710876464844,
+      "loss": 0.2054,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 1.197824239730835,
+      "rewards/margins": 2.400318145751953,
+      "rewards/rejected": -1.2024939060211182,
+      "step": 1640
+    },
+    {
+      "epoch": 2.700490998363339,
+      "grad_norm": 2.7052650451660156,
+      "learning_rate": 5.942995755003032e-06,
+      "logits/chosen": -1.5151501893997192,
+      "logits/rejected": -1.2694567441940308,
+      "logps/chosen": -461.31048583984375,
+      "logps/rejected": -468.01806640625,
+      "loss": 0.2232,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 1.1455790996551514,
+      "rewards/margins": 2.2321972846984863,
+      "rewards/rejected": -1.086618185043335,
+      "step": 1650
+    },
+    {
+      "epoch": 2.716857610474632,
+      "grad_norm": 6.372529029846191,
+      "learning_rate": 5.639781685870225e-06,
+      "logits/chosen": -1.3566817045211792,
+      "logits/rejected": -1.074830412864685,
+      "logps/chosen": -443.8597717285156,
+      "logps/rejected": -400.56842041015625,
+      "loss": 0.367,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.7199681997299194,
+      "rewards/margins": 1.740003228187561,
+      "rewards/rejected": -1.020034909248352,
+      "step": 1660
+    },
+    {
+      "epoch": 2.7332242225859247,
+      "grad_norm": 8.768135070800781,
+      "learning_rate": 5.336567616737417e-06,
+      "logits/chosen": -1.5840198993682861,
+      "logits/rejected": -1.4000660181045532,
+      "logps/chosen": -443.77056884765625,
+      "logps/rejected": -431.5873107910156,
+      "loss": 0.2731,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 1.0286372900009155,
+      "rewards/margins": 2.149087429046631,
+      "rewards/rejected": -1.1204503774642944,
+      "step": 1670
+    },
+    {
+      "epoch": 2.7495908346972175,
+      "grad_norm": 2.6619386672973633,
+      "learning_rate": 5.033353547604609e-06,
+      "logits/chosen": -1.4543578624725342,
+      "logits/rejected": -1.2476505041122437,
+      "logps/chosen": -423.39764404296875,
+      "logps/rejected": -421.17462158203125,
+      "loss": 0.2493,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 0.9351073503494263,
+      "rewards/margins": 2.071972370147705,
+      "rewards/rejected": -1.1368649005889893,
+      "step": 1680
+    },
+    {
+      "epoch": 2.7659574468085104,
+      "grad_norm": 4.837337493896484,
+      "learning_rate": 4.730139478471801e-06,
+      "logits/chosen": -1.6075165271759033,
+      "logits/rejected": NaN,
+      "logps/chosen": -437.904052734375,
+      "logps/rejected": -409.699951171875,
+      "loss": 0.2651,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 0.9201074838638306,
+      "rewards/margins": 2.1968226432800293,
+      "rewards/rejected": -1.2767151594161987,
+      "step": 1690
+    },
+    {
+      "epoch": 2.7823240589198037,
+      "grad_norm": 7.250920295715332,
+      "learning_rate": 4.426925409338994e-06,
+      "logits/chosen": -1.3620741367340088,
+      "logits/rejected": -1.1615064144134521,
+      "logps/chosen": -470.387451171875,
+      "logps/rejected": -418.77716064453125,
+      "loss": 0.3151,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 1.0041719675064087,
+      "rewards/margins": 1.8728843927383423,
+      "rewards/rejected": -0.868712306022644,
+      "step": 1700
+    },
+    {
+      "epoch": 2.7986906710310966,
+      "grad_norm": 5.2374982833862305,
+      "learning_rate": 4.123711340206186e-06,
+      "logits/chosen": -1.193421721458435,
+      "logits/rejected": -1.2663942575454712,
+      "logps/chosen": -432.0567932128906,
+      "logps/rejected": -416.0810546875,
+      "loss": 0.3065,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.8853948712348938,
+      "rewards/margins": 1.7219581604003906,
+      "rewards/rejected": -0.8365634679794312,
+      "step": 1710
+    },
+    {
+      "epoch": 2.8150572831423895,
+      "grad_norm": 6.9263224601745605,
+      "learning_rate": 3.8204972710733785e-06,
+      "logits/chosen": -1.42405366897583,
+      "logits/rejected": -1.272728681564331,
+      "logps/chosen": -438.894287109375,
+      "logps/rejected": -394.73284912109375,
+      "loss": 0.2415,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 1.0647566318511963,
+      "rewards/margins": 2.0775082111358643,
+      "rewards/rejected": -1.0127513408660889,
+      "step": 1720
+    },
+    {
+      "epoch": 2.8314238952536823,
+      "grad_norm": 2.6749045848846436,
+      "learning_rate": 3.51728320194057e-06,
+      "logits/chosen": -1.2201988697052002,
+      "logits/rejected": -0.9426723718643188,
+      "logps/chosen": -474.87530517578125,
+      "logps/rejected": -410.86798095703125,
+      "loss": 0.2664,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 1.1347922086715698,
+      "rewards/margins": 2.3549981117248535,
+      "rewards/rejected": -1.2202056646347046,
+      "step": 1730
+    },
+    {
+      "epoch": 2.8477905073649756,
+      "grad_norm": 4.696692943572998,
+      "learning_rate": 3.214069132807762e-06,
+      "logits/chosen": -1.124267339706421,
+      "logits/rejected": -0.9841214418411255,
+      "logps/chosen": -530.7108154296875,
+      "logps/rejected": -499.8089904785156,
+      "loss": 0.2433,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 1.319337010383606,
+      "rewards/margins": 2.231715202331543,
+      "rewards/rejected": -0.9123780131340027,
+      "step": 1740
+    },
+    {
+      "epoch": 2.8641571194762685,
+      "grad_norm": 2.817408323287964,
+      "learning_rate": 2.910855063674955e-06,
+      "logits/chosen": -1.3371939659118652,
+      "logits/rejected": -1.0595276355743408,
+      "logps/chosen": -457.59234619140625,
+      "logps/rejected": -451.3644104003906,
+      "loss": 0.2328,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 1.0093997716903687,
+      "rewards/margins": 2.203674554824829,
+      "rewards/rejected": -1.194274663925171,
+      "step": 1750
+    },
+    {
+      "epoch": 2.8805237315875614,
+      "grad_norm": 1.328909993171692,
+      "learning_rate": 2.607640994542147e-06,
+      "logits/chosen": -1.3779757022857666,
+      "logits/rejected": -1.2147364616394043,
+      "logps/chosen": -431.8528747558594,
+      "logps/rejected": -456.26727294921875,
+      "loss": 0.2552,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 0.9310500025749207,
+      "rewards/margins": 2.0603978633880615,
+      "rewards/rejected": -1.1293480396270752,
+      "step": 1760
+    },
+    {
+      "epoch": 2.8968903436988542,
+      "grad_norm": 2.1010754108428955,
+      "learning_rate": 2.3044269254093393e-06,
+      "logits/chosen": -1.59529447555542,
+      "logits/rejected": -1.4413836002349854,
+      "logps/chosen": -441.6121520996094,
+      "logps/rejected": -462.1275939941406,
+      "loss": 0.2184,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 1.1032123565673828,
+      "rewards/margins": 2.270881175994873,
+      "rewards/rejected": -1.1676688194274902,
+      "step": 1770
+    },
+    {
+      "epoch": 2.913256955810147,
+      "grad_norm": 2.0799882411956787,
+      "learning_rate": 2.0012128562765313e-06,
+      "logits/chosen": -1.1777410507202148,
+      "logits/rejected": -1.0771524906158447,
+      "logps/chosen": -493.805419921875,
+      "logps/rejected": -447.61468505859375,
+      "loss": 0.2117,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 1.1594716310501099,
+      "rewards/margins": 2.220271587371826,
+      "rewards/rejected": -1.0607998371124268,
+      "step": 1780
+    },
+    {
+      "epoch": 2.9296235679214404,
+      "grad_norm": 5.403112888336182,
+      "learning_rate": 1.6979987871437233e-06,
+      "logits/chosen": -1.5264140367507935,
+      "logits/rejected": -1.4546772241592407,
+      "logps/chosen": -421.8680725097656,
+      "logps/rejected": -406.50579833984375,
+      "loss": 0.2935,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.9573639035224915,
+      "rewards/margins": 1.8112157583236694,
+      "rewards/rejected": -0.8538519144058228,
+      "step": 1790
+    },
+    {
+      "epoch": 2.9459901800327333,
+      "grad_norm": 2.95246958732605,
+      "learning_rate": 1.394784718010916e-06,
+      "logits/chosen": -1.1958624124526978,
+      "logits/rejected": -0.8615037798881531,
+      "logps/chosen": -402.82305908203125,
+      "logps/rejected": -379.9725341796875,
+      "loss": 0.2734,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.86333829164505,
+      "rewards/margins": 2.0156140327453613,
+      "rewards/rejected": -1.152275800704956,
+      "step": 1800
+    },
+    {
+      "epoch": 2.962356792144026,
+      "grad_norm": 2.8880884647369385,
+      "learning_rate": 1.091570648878108e-06,
+      "logits/chosen": -1.4118956327438354,
+      "logits/rejected": -1.0699918270111084,
+      "logps/chosen": -404.4219665527344,
+      "logps/rejected": -381.11639404296875,
+      "loss": 0.2644,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 0.8657559156417847,
+      "rewards/margins": 2.258451223373413,
+      "rewards/rejected": -1.392695665359497,
+      "step": 1810
+    },
+    {
+      "epoch": 2.978723404255319,
+      "grad_norm": 1.423218011856079,
+      "learning_rate": 7.883565797453002e-07,
+      "logits/chosen": -1.5209050178527832,
+      "logits/rejected": -1.1870254278182983,
+      "logps/chosen": -393.4710998535156,
+      "logps/rejected": -393.87353515625,
+      "loss": 0.2505,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 0.913676917552948,
+      "rewards/margins": 2.0497212409973145,
+      "rewards/rejected": -1.1360442638397217,
+      "step": 1820
+    },
+    {
+      "epoch": 2.9950900163666123,
+      "grad_norm": 9.121944427490234,
+      "learning_rate": 4.851425106124925e-07,
+      "logits/chosen": -1.419418215751648,
+      "logits/rejected": -1.2076327800750732,
+      "logps/chosen": -436.1336364746094,
+      "logps/rejected": -481.68548583984375,
+      "loss": 0.2557,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 1.0021239519119263,
+      "rewards/margins": 2.3086962699890137,
+      "rewards/rejected": -1.3065721988677979,
+      "step": 1830
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 1833,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}