{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 5.6,
  "eval_steps": 500,
  "global_step": 7000,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.008,
      "grad_norm": 2.3092777729034424,
      "learning_rate": 3.6000000000000005e-08,
      "logits/chosen": 0.7764996886253357,
      "logits/rejected": 0.8174192309379578,
      "logps/chosen": -195.11270141601562,
      "logps/rejected": -207.083251953125,
      "loss": 0.6965,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.0012020017020404339,
      "rewards/margins": -0.006086898501962423,
      "rewards/rejected": 0.004884896334260702,
      "step": 10
    },
    {
      "epoch": 0.016,
      "grad_norm": 3.044440507888794,
      "learning_rate": 7.6e-08,
      "logits/chosen": 0.8046936988830566,
      "logits/rejected": 0.7519802451133728,
      "logps/chosen": -196.82322692871094,
      "logps/rejected": -199.39141845703125,
      "loss": 0.6877,
      "rewards/accuracies": 0.5250000357627869,
      "rewards/chosen": 0.0069522010162472725,
      "rewards/margins": 0.01239713840186596,
      "rewards/rejected": -0.005444936919957399,
      "step": 20
    },
    {
      "epoch": 0.024,
      "grad_norm": 2.9533703327178955,
      "learning_rate": 1.16e-07,
      "logits/chosen": 0.7309688925743103,
      "logits/rejected": 0.6841751933097839,
      "logps/chosen": -201.09934997558594,
      "logps/rejected": -231.3838653564453,
      "loss": 0.6962,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.0034579948987811804,
      "rewards/margins": -0.00475282734259963,
      "rewards/rejected": 0.0012948326766490936,
      "step": 30
    },
    {
      "epoch": 0.032,
      "grad_norm": 3.043637990951538,
      "learning_rate": 1.56e-07,
      "logits/chosen": 0.707830548286438,
      "logits/rejected": 0.6798078417778015,
      "logps/chosen": -217.506103515625,
      "logps/rejected": -214.7266845703125,
      "loss": 0.695,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.006588793825358152,
      "rewards/margins": -0.0025365306064486504,
      "rewards/rejected": -0.004052260424941778,
      "step": 40
    },
    {
      "epoch": 0.04,
      "grad_norm": 2.9874885082244873,
      "learning_rate": 1.96e-07,
      "logits/chosen": 0.6864386796951294,
      "logits/rejected": 0.6313192248344421,
      "logps/chosen": -215.640869140625,
      "logps/rejected": -208.9573211669922,
      "loss": 0.688,
      "rewards/accuracies": 0.5875000357627869,
      "rewards/chosen": -0.0006361968698911369,
      "rewards/margins": 0.011526194401085377,
      "rewards/rejected": -0.012162390165030956,
      "step": 50
    },
    {
      "epoch": 0.048,
      "grad_norm": 2.3293204307556152,
      "learning_rate": 2.3600000000000002e-07,
      "logits/chosen": 0.7226251363754272,
      "logits/rejected": 0.826255738735199,
      "logps/chosen": -202.46560668945312,
      "logps/rejected": -201.2319793701172,
      "loss": 0.6887,
      "rewards/accuracies": 0.5875000357627869,
      "rewards/chosen": 0.007742859423160553,
      "rewards/margins": 0.010210123844444752,
      "rewards/rejected": -0.0024672651197761297,
      "step": 60
    },
    {
      "epoch": 0.056,
      "grad_norm": 2.408390522003174,
      "learning_rate": 2.7600000000000004e-07,
      "logits/chosen": 0.6049055457115173,
      "logits/rejected": 0.6685499548912048,
      "logps/chosen": -193.7244873046875,
      "logps/rejected": -202.5239715576172,
      "loss": 0.6949,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.004926090594381094,
      "rewards/margins": -0.0017412189627066255,
      "rewards/rejected": -0.0031848729122430086,
      "step": 70
    },
    {
      "epoch": 0.064,
      "grad_norm": 3.6175403594970703,
      "learning_rate": 3.160000000000001e-07,
      "logits/chosen": 0.5864802598953247,
      "logits/rejected": 0.6271843314170837,
      "logps/chosen": -202.54612731933594,
      "logps/rejected": -233.43150329589844,
      "loss": 0.6904,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": 0.00966743566095829,
      "rewards/margins": 0.006970310118049383,
      "rewards/rejected": 0.002697124844416976,
      "step": 80
    },
    {
      "epoch": 0.072,
      "grad_norm": 2.3893001079559326,
      "learning_rate": 3.56e-07,
      "logits/chosen": 0.730164647102356,
      "logits/rejected": 0.713482677936554,
      "logps/chosen": -195.13555908203125,
      "logps/rejected": -219.647216796875,
      "loss": 0.6963,
      "rewards/accuracies": 0.5875000357627869,
      "rewards/chosen": 0.011823897249996662,
      "rewards/margins": -0.005143971648067236,
      "rewards/rejected": 0.01696786843240261,
      "step": 90
    },
    {
      "epoch": 0.08,
      "grad_norm": 2.7904205322265625,
      "learning_rate": 3.9600000000000005e-07,
      "logits/chosen": 0.5790210962295532,
      "logits/rejected": 0.6835765838623047,
      "logps/chosen": -202.36647033691406,
      "logps/rejected": -213.2338104248047,
      "loss": 0.696,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.004648447968065739,
      "rewards/margins": -0.004634796176105738,
      "rewards/rejected": -1.3651699191541411e-05,
      "step": 100
    },
    {
      "epoch": 0.088,
      "grad_norm": 2.350283145904541,
      "learning_rate": 4.3600000000000004e-07,
      "logits/chosen": 0.8016487956047058,
      "logits/rejected": 0.8312139511108398,
      "logps/chosen": -202.82815551757812,
      "logps/rejected": -218.19920349121094,
      "loss": 0.6913,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.01035931147634983,
      "rewards/margins": 0.005032673478126526,
      "rewards/rejected": 0.005326639395207167,
      "step": 110
    },
    {
      "epoch": 0.096,
      "grad_norm": 2.2766993045806885,
      "learning_rate": 4.760000000000001e-07,
      "logits/chosen": 0.8250360488891602,
      "logits/rejected": 0.835590660572052,
      "logps/chosen": -196.98216247558594,
      "logps/rejected": -210.0322723388672,
      "loss": 0.6919,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.00589400390163064,
      "rewards/margins": 0.003922100644558668,
      "rewards/rejected": 0.0019719023257493973,
      "step": 120
    },
    {
      "epoch": 0.104,
      "grad_norm": 2.0690665245056152,
      "learning_rate": 5.16e-07,
      "logits/chosen": 0.7067065238952637,
      "logits/rejected": 0.7281522154808044,
      "logps/chosen": -195.405029296875,
      "logps/rejected": -206.66079711914062,
      "loss": 0.6954,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": 0.006292525213211775,
      "rewards/margins": -0.0033251051791012287,
      "rewards/rejected": 0.009617629460990429,
      "step": 130
    },
    {
      "epoch": 0.112,
      "grad_norm": 2.388901948928833,
      "learning_rate": 5.560000000000001e-07,
      "logits/chosen": 0.6768548488616943,
      "logits/rejected": 0.6542965173721313,
      "logps/chosen": -214.23342895507812,
      "logps/rejected": -216.52366638183594,
      "loss": 0.6939,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.012295923195779324,
      "rewards/margins": 8.453801456198562e-06,
      "rewards/rejected": -0.012304377742111683,
      "step": 140
    },
    {
      "epoch": 0.12,
      "grad_norm": 2.4146411418914795,
      "learning_rate": 5.960000000000001e-07,
      "logits/chosen": 0.8362228274345398,
      "logits/rejected": 0.780723512172699,
      "logps/chosen": -192.99110412597656,
      "logps/rejected": -182.7938232421875,
      "loss": 0.6965,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.008289004676043987,
      "rewards/margins": -0.005621676333248615,
      "rewards/rejected": 0.013910681009292603,
      "step": 150
    },
    {
      "epoch": 0.128,
      "grad_norm": 2.408268690109253,
      "learning_rate": 6.360000000000001e-07,
      "logits/chosen": 0.6485953330993652,
      "logits/rejected": 0.7263766527175903,
      "logps/chosen": -191.4886016845703,
      "logps/rejected": -218.2458953857422,
      "loss": 0.7014,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.007325764745473862,
      "rewards/margins": -0.014910398982465267,
      "rewards/rejected": 0.022236162796616554,
      "step": 160
    },
    {
      "epoch": 0.136,
      "grad_norm": 2.3368568420410156,
      "learning_rate": 6.76e-07,
      "logits/chosen": 0.6823139190673828,
      "logits/rejected": 0.7047578692436218,
      "logps/chosen": -188.77145385742188,
      "logps/rejected": -230.02903747558594,
      "loss": 0.6928,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.009600992314517498,
      "rewards/margins": 0.0018820532131940126,
      "rewards/rejected": 0.007718939334154129,
      "step": 170
    },
    {
      "epoch": 0.144,
      "grad_norm": 2.562350273132324,
      "learning_rate": 7.16e-07,
      "logits/chosen": 0.7193002700805664,
      "logits/rejected": 0.6238381266593933,
      "logps/chosen": -199.7821807861328,
      "logps/rejected": -189.610595703125,
      "loss": 0.6955,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.00945024099200964,
      "rewards/margins": -0.003362303366884589,
      "rewards/rejected": -0.00608793692663312,
      "step": 180
    },
    {
      "epoch": 0.152,
      "grad_norm": 2.5308778285980225,
      "learning_rate": 7.56e-07,
      "logits/chosen": 0.7583810091018677,
      "logits/rejected": 0.7683423161506653,
      "logps/chosen": -179.4371795654297,
      "logps/rejected": -193.5501708984375,
      "loss": 0.6947,
      "rewards/accuracies": 0.45000001788139343,
      "rewards/chosen": 0.024644900113344193,
      "rewards/margins": -0.0013052498688921332,
      "rewards/rejected": 0.025950148701667786,
      "step": 190
    },
    {
      "epoch": 0.16,
      "grad_norm": 2.271794080734253,
      "learning_rate": 7.960000000000001e-07,
      "logits/chosen": 0.8137730956077576,
      "logits/rejected": 0.6786491274833679,
      "logps/chosen": -202.08787536621094,
      "logps/rejected": -198.05364990234375,
      "loss": 0.6975,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.01316139753907919,
      "rewards/margins": -0.007475724909454584,
      "rewards/rejected": -0.005685672629624605,
      "step": 200
    },
    {
      "epoch": 0.168,
      "grad_norm": 2.6240570545196533,
      "learning_rate": 8.36e-07,
      "logits/chosen": 0.6418821215629578,
      "logits/rejected": 0.7391870617866516,
      "logps/chosen": -202.360107421875,
      "logps/rejected": -229.84375,
      "loss": 0.6896,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.006038342602550983,
      "rewards/margins": 0.00873213168233633,
      "rewards/rejected": -0.002693791640922427,
      "step": 210
    },
    {
      "epoch": 0.176,
      "grad_norm": 2.454711675643921,
      "learning_rate": 8.760000000000001e-07,
      "logits/chosen": 0.6627649664878845,
      "logits/rejected": 0.5812577605247498,
      "logps/chosen": -231.45840454101562,
      "logps/rejected": -224.667724609375,
      "loss": 0.6921,
      "rewards/accuracies": 0.5250000357627869,
      "rewards/chosen": -0.004710569512099028,
      "rewards/margins": 0.0032497793436050415,
      "rewards/rejected": -0.007960348390042782,
      "step": 220
    },
    {
      "epoch": 0.184,
      "grad_norm": 2.9100327491760254,
      "learning_rate": 9.160000000000001e-07,
      "logits/chosen": 0.7058368921279907,
      "logits/rejected": 0.6532104015350342,
      "logps/chosen": -209.2056884765625,
      "logps/rejected": -206.2039031982422,
      "loss": 0.6916,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.00045182276517152786,
      "rewards/margins": 0.004544996656477451,
      "rewards/rejected": -0.004093174822628498,
      "step": 230
    },
    {
      "epoch": 0.192,
      "grad_norm": 2.845057487487793,
      "learning_rate": 9.56e-07,
      "logits/chosen": 0.5265730023384094,
      "logits/rejected": 0.6017157435417175,
      "logps/chosen": -202.66600036621094,
      "logps/rejected": -229.9880828857422,
      "loss": 0.697,
      "rewards/accuracies": 0.5250000357627869,
      "rewards/chosen": 0.0027109149377793074,
      "rewards/margins": -0.00603170320391655,
      "rewards/rejected": 0.008742619305849075,
      "step": 240
    },
    {
      "epoch": 0.2,
      "grad_norm": 2.409635543823242,
      "learning_rate": 9.96e-07,
      "logits/chosen": 0.7155398726463318,
      "logits/rejected": 0.7485236525535583,
      "logps/chosen": -208.7382354736328,
      "logps/rejected": -224.94078063964844,
      "loss": 0.689,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.021835366263985634,
      "rewards/margins": 0.009591616690158844,
      "rewards/rejected": 0.01224374771118164,
      "step": 250
    },
    {
      "epoch": 0.208,
      "grad_norm": 2.2228920459747314,
      "learning_rate": 1.0360000000000001e-06,
      "logits/chosen": 0.7711376547813416,
      "logits/rejected": 0.6718112230300903,
      "logps/chosen": -204.5934600830078,
      "logps/rejected": -199.9540252685547,
      "loss": 0.6978,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 0.006919704377651215,
      "rewards/margins": -0.008324065245687962,
      "rewards/rejected": 0.015243768692016602,
      "step": 260
    },
    {
      "epoch": 0.216,
      "grad_norm": 2.7563939094543457,
      "learning_rate": 1.0760000000000002e-06,
      "logits/chosen": 0.7490439414978027,
      "logits/rejected": 0.8420026898384094,
      "logps/chosen": -194.5001678466797,
      "logps/rejected": -194.9221954345703,
      "loss": 0.7006,
      "rewards/accuracies": 0.4124999940395355,
      "rewards/chosen": 0.0004544306721072644,
      "rewards/margins": -0.01385478675365448,
      "rewards/rejected": 0.014309215359389782,
      "step": 270
    },
    {
      "epoch": 0.224,
      "grad_norm": 2.703477621078491,
      "learning_rate": 1.1160000000000002e-06,
      "logits/chosen": 0.654878556728363,
      "logits/rejected": 0.6851706504821777,
      "logps/chosen": -191.12637329101562,
      "logps/rejected": -179.5616455078125,
      "loss": 0.6963,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.002727155340835452,
      "rewards/margins": -0.005241455975919962,
      "rewards/rejected": 0.002514300402253866,
      "step": 280
    },
    {
      "epoch": 0.232,
      "grad_norm": 2.8370096683502197,
      "learning_rate": 1.156e-06,
      "logits/chosen": 0.7035982012748718,
      "logits/rejected": 0.6883268356323242,
      "logps/chosen": -198.1017608642578,
      "logps/rejected": -205.367431640625,
      "loss": 0.6936,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.0034764811862260103,
      "rewards/margins": 0.0005613662651740015,
      "rewards/rejected": 0.0029151157941669226,
      "step": 290
    },
    {
      "epoch": 0.24,
      "grad_norm": 2.892529249191284,
      "learning_rate": 1.196e-06,
      "logits/chosen": 0.661008358001709,
      "logits/rejected": 0.6034583449363708,
      "logps/chosen": -197.7530975341797,
      "logps/rejected": -212.4963836669922,
      "loss": 0.6944,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.005977009888738394,
      "rewards/margins": -0.0009391207131557167,
      "rewards/rejected": 0.006916132755577564,
      "step": 300
    },
    {
      "epoch": 0.248,
      "grad_norm": 2.4562253952026367,
      "learning_rate": 1.2360000000000001e-06,
      "logits/chosen": 0.7135326266288757,
      "logits/rejected": 0.7250908017158508,
      "logps/chosen": -203.17738342285156,
      "logps/rejected": -208.8437042236328,
      "loss": 0.6925,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 0.0018625364173203707,
      "rewards/margins": 0.002679466502740979,
      "rewards/rejected": -0.0008169323555193841,
      "step": 310
    },
    {
      "epoch": 0.256,
      "grad_norm": 2.6124107837677,
      "learning_rate": 1.276e-06,
      "logits/chosen": 0.8584432601928711,
      "logits/rejected": 0.8116118311882019,
      "logps/chosen": -195.6661376953125,
      "logps/rejected": -196.38145446777344,
      "loss": 0.6932,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.005818118806928396,
      "rewards/margins": 0.0013451100094243884,
      "rewards/rejected": 0.004473009612411261,
      "step": 320
    },
    {
      "epoch": 0.264,
      "grad_norm": 2.860166311264038,
      "learning_rate": 1.316e-06,
      "logits/chosen": 0.6980428099632263,
      "logits/rejected": 0.8046857118606567,
      "logps/chosen": -189.9589080810547,
      "logps/rejected": -203.92202758789062,
      "loss": 0.694,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 0.018240585923194885,
      "rewards/margins": -0.0006985944928601384,
      "rewards/rejected": 0.018939180299639702,
      "step": 330
    },
    {
      "epoch": 0.272,
      "grad_norm": 2.8200535774230957,
      "learning_rate": 1.356e-06,
      "logits/chosen": 0.8003985285758972,
      "logits/rejected": 0.77861088514328,
      "logps/chosen": -201.60928344726562,
      "logps/rejected": -203.56936645507812,
      "loss": 0.693,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.002375365002080798,
      "rewards/margins": 0.001602139906026423,
      "rewards/rejected": 0.0007732249796390533,
      "step": 340
    },
    {
      "epoch": 0.28,
      "grad_norm": 3.1359260082244873,
      "learning_rate": 1.396e-06,
      "logits/chosen": 0.7161771655082703,
      "logits/rejected": 0.6535243988037109,
      "logps/chosen": -196.94798278808594,
      "logps/rejected": -207.852783203125,
      "loss": 0.6921,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.0128513528034091,
      "rewards/margins": 0.003678938141092658,
      "rewards/rejected": 0.009172416292130947,
      "step": 350
    },
    {
      "epoch": 0.288,
      "grad_norm": 3.2514476776123047,
      "learning_rate": 1.436e-06,
      "logits/chosen": 0.6619038581848145,
      "logits/rejected": 0.656987726688385,
      "logps/chosen": -200.4764862060547,
      "logps/rejected": -231.81895446777344,
      "loss": 0.6878,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.022911271080374718,
      "rewards/margins": 0.012182674370706081,
      "rewards/rejected": 0.010728596709668636,
      "step": 360
    },
    {
      "epoch": 0.296,
      "grad_norm": 2.784947156906128,
      "learning_rate": 1.4760000000000001e-06,
      "logits/chosen": 0.7618936896324158,
      "logits/rejected": 0.7150487899780273,
      "logps/chosen": -196.36228942871094,
      "logps/rejected": -211.08935546875,
      "loss": 0.6905,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 0.017282454296946526,
      "rewards/margins": 0.006800856441259384,
      "rewards/rejected": 0.010481595061719418,
      "step": 370
    },
    {
      "epoch": 0.304,
      "grad_norm": 2.753596067428589,
      "learning_rate": 1.5160000000000002e-06,
      "logits/chosen": 0.8928642272949219,
      "logits/rejected": 0.8450748324394226,
      "logps/chosen": -204.9441680908203,
      "logps/rejected": -198.3724822998047,
      "loss": 0.6932,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.029540909454226494,
      "rewards/margins": 0.0015256500337272882,
      "rewards/rejected": 0.028015261515975,
      "step": 380
    },
    {
      "epoch": 0.312,
      "grad_norm": 3.1084704399108887,
      "learning_rate": 1.556e-06,
      "logits/chosen": 0.8117288947105408,
      "logits/rejected": 0.766746461391449,
      "logps/chosen": -195.1800079345703,
      "logps/rejected": -199.2845001220703,
      "loss": 0.6961,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.01765977405011654,
      "rewards/margins": -0.004569740500301123,
      "rewards/rejected": 0.0222295131534338,
      "step": 390
    },
    {
      "epoch": 0.32,
      "grad_norm": 3.7862966060638428,
      "learning_rate": 1.596e-06,
      "logits/chosen": 0.8835588693618774,
      "logits/rejected": 0.8266305327415466,
      "logps/chosen": -208.58726501464844,
      "logps/rejected": -217.5359649658203,
      "loss": 0.6976,
      "rewards/accuracies": 0.38750001788139343,
      "rewards/chosen": 0.020054074004292488,
      "rewards/margins": -0.0072321416810154915,
      "rewards/rejected": 0.027286216616630554,
      "step": 400
    },
    {
      "epoch": 0.328,
      "grad_norm": 2.675278663635254,
      "learning_rate": 1.636e-06,
      "logits/chosen": 0.7499464750289917,
      "logits/rejected": 0.7868902087211609,
      "logps/chosen": -176.598876953125,
      "logps/rejected": -185.85235595703125,
      "loss": 0.697,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": 0.015415973030030727,
      "rewards/margins": -0.006510419305413961,
      "rewards/rejected": 0.02192639373242855,
      "step": 410
    },
    {
      "epoch": 0.336,
      "grad_norm": 2.8932230472564697,
      "learning_rate": 1.6760000000000001e-06,
      "logits/chosen": 0.8424084782600403,
      "logits/rejected": 0.8695581555366516,
      "logps/chosen": -196.9973602294922,
      "logps/rejected": -203.99620056152344,
      "loss": 0.6901,
      "rewards/accuracies": 0.5250000357627869,
      "rewards/chosen": 0.015818921849131584,
      "rewards/margins": 0.007251453585922718,
      "rewards/rejected": 0.008567466400563717,
      "step": 420
    },
    {
      "epoch": 0.344,
      "grad_norm": 2.983330488204956,
      "learning_rate": 1.7160000000000002e-06,
      "logits/chosen": 0.5948252081871033,
      "logits/rejected": 0.6821457743644714,
      "logps/chosen": -192.26760864257812,
      "logps/rejected": -200.97569274902344,
      "loss": 0.6913,
      "rewards/accuracies": 0.5250000357627869,
      "rewards/chosen": 0.010373975150287151,
      "rewards/margins": 0.004705019760876894,
      "rewards/rejected": 0.005668954458087683,
      "step": 430
    },
    {
      "epoch": 0.352,
      "grad_norm": 3.401388168334961,
      "learning_rate": 1.7560000000000002e-06,
      "logits/chosen": 0.7682614922523499,
      "logits/rejected": 0.8079833984375,
      "logps/chosen": -201.40171813964844,
      "logps/rejected": -215.61692810058594,
      "loss": 0.6936,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.018279600888490677,
      "rewards/margins": 0.0005021900869905949,
      "rewards/rejected": 0.01777741126716137,
      "step": 440
    },
    {
      "epoch": 0.36,
      "grad_norm": 3.9692232608795166,
      "learning_rate": 1.7960000000000003e-06,
      "logits/chosen": 0.6712386012077332,
      "logits/rejected": 0.7559861540794373,
      "logps/chosen": -198.07620239257812,
      "logps/rejected": -226.9971466064453,
      "loss": 0.6865,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02878693677484989,
      "rewards/margins": 0.014408250339329243,
      "rewards/rejected": 0.014378686435520649,
      "step": 450
    },
    {
      "epoch": 0.368,
      "grad_norm": 2.5181920528411865,
      "learning_rate": 1.8360000000000003e-06,
      "logits/chosen": 0.6611719131469727,
      "logits/rejected": 0.7472667098045349,
      "logps/chosen": -186.1030731201172,
      "logps/rejected": -198.84902954101562,
      "loss": 0.6893,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.022867394611239433,
      "rewards/margins": 0.009127254597842693,
      "rewards/rejected": 0.01374014001339674,
      "step": 460
    },
    {
      "epoch": 0.376,
      "grad_norm": 3.491785764694214,
      "learning_rate": 1.8760000000000001e-06,
      "logits/chosen": 0.6375329494476318,
      "logits/rejected": 0.7117382287979126,
      "logps/chosen": -191.62840270996094,
      "logps/rejected": -192.08445739746094,
      "loss": 0.6936,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 0.013536167331039906,
      "rewards/margins": 0.0005845252308063209,
      "rewards/rejected": 0.012951642274856567,
      "step": 470
    },
    {
      "epoch": 0.384,
      "grad_norm": 3.381208896636963,
      "learning_rate": 1.916e-06,
      "logits/chosen": 0.5306805968284607,
      "logits/rejected": 0.6659603714942932,
      "logps/chosen": -200.27557373046875,
      "logps/rejected": -223.94517517089844,
      "loss": 0.6912,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.020672379061579704,
      "rewards/margins": 0.0055915359407663345,
      "rewards/rejected": 0.015080844052135944,
      "step": 480
    },
    {
      "epoch": 0.392,
      "grad_norm": 2.9275639057159424,
      "learning_rate": 1.956e-06,
      "logits/chosen": 0.7187590599060059,
      "logits/rejected": 0.7936305403709412,
      "logps/chosen": -198.9750213623047,
      "logps/rejected": -198.08738708496094,
      "loss": 0.6898,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.023504601791501045,
      "rewards/margins": 0.007863587699830532,
      "rewards/rejected": 0.01564101316034794,
      "step": 490
    },
    {
      "epoch": 0.4,
      "grad_norm": 2.6241567134857178,
      "learning_rate": 1.996e-06,
      "logits/chosen": 0.7280531525611877,
      "logits/rejected": 0.8506690859794617,
      "logps/chosen": -188.4355010986328,
      "logps/rejected": -195.9119110107422,
      "loss": 0.69,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.03479978069663048,
      "rewards/margins": 0.007221006788313389,
      "rewards/rejected": 0.027578774839639664,
      "step": 500
    },
    {
      "epoch": 0.408,
      "grad_norm": 2.9502575397491455,
      "learning_rate": 2.036e-06,
      "logits/chosen": 0.7209169268608093,
      "logits/rejected": 0.6922793984413147,
      "logps/chosen": -197.63975524902344,
      "logps/rejected": -215.0830535888672,
      "loss": 0.689,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.02393524721264839,
      "rewards/margins": 0.009743581525981426,
      "rewards/rejected": 0.01419166661798954,
      "step": 510
    },
    {
      "epoch": 0.416,
      "grad_norm": 2.9870102405548096,
      "learning_rate": 2.076e-06,
      "logits/chosen": 0.8047041296958923,
      "logits/rejected": 0.764294445514679,
      "logps/chosen": -202.94752502441406,
      "logps/rejected": -195.111572265625,
      "loss": 0.7018,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.029683226719498634,
      "rewards/margins": -0.015081966295838356,
      "rewards/rejected": 0.04476520046591759,
      "step": 520
    },
    {
      "epoch": 0.424,
      "grad_norm": 2.907334804534912,
      "learning_rate": 2.116e-06,
      "logits/chosen": 0.7759226560592651,
      "logits/rejected": 0.7076265215873718,
      "logps/chosen": -193.26431274414062,
      "logps/rejected": -193.8086700439453,
      "loss": 0.6929,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.04167484864592552,
      "rewards/margins": 0.0013663482386618853,
      "rewards/rejected": 0.040308497846126556,
      "step": 530
    },
    {
      "epoch": 0.432,
      "grad_norm": 2.9025113582611084,
      "learning_rate": 2.156e-06,
      "logits/chosen": 0.674287736415863,
      "logits/rejected": 0.6436707377433777,
      "logps/chosen": -195.1566619873047,
      "logps/rejected": -226.0045166015625,
      "loss": 0.6892,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.03686271235346794,
      "rewards/margins": 0.009263001382350922,
      "rewards/rejected": 0.02759971097111702,
      "step": 540
    },
    {
      "epoch": 0.44,
      "grad_norm": 3.0069284439086914,
      "learning_rate": 2.1960000000000002e-06,
      "logits/chosen": 0.5614029169082642,
      "logits/rejected": 0.618638277053833,
      "logps/chosen": -204.8896026611328,
      "logps/rejected": -218.7178497314453,
      "loss": 0.683,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.037945158779621124,
      "rewards/margins": 0.021600285544991493,
      "rewards/rejected": 0.01634487323462963,
      "step": 550
    },
    {
      "epoch": 0.448,
      "grad_norm": 3.801163673400879,
      "learning_rate": 2.2360000000000003e-06,
      "logits/chosen": 0.7861131429672241,
      "logits/rejected": 0.6601011753082275,
      "logps/chosen": -201.13636779785156,
      "logps/rejected": -198.4203643798828,
      "loss": 0.6883,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.056006718426942825,
      "rewards/margins": 0.011804056353867054,
      "rewards/rejected": 0.044202663004398346,
      "step": 560
    },
    {
      "epoch": 0.456,
      "grad_norm": 3.0970733165740967,
      "learning_rate": 2.2760000000000003e-06,
      "logits/chosen": 0.8429245352745056,
      "logits/rejected": 0.8626702427864075,
      "logps/chosen": -195.6461639404297,
      "logps/rejected": -192.6861572265625,
      "loss": 0.684,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.06717582792043686,
      "rewards/margins": 0.020122915506362915,
      "rewards/rejected": 0.047052908688783646,
      "step": 570
    },
    {
      "epoch": 0.464,
      "grad_norm": 3.105346918106079,
      "learning_rate": 2.3160000000000004e-06,
      "logits/chosen": 0.6571624875068665,
      "logits/rejected": 0.7619710564613342,
      "logps/chosen": -191.7879180908203,
      "logps/rejected": -222.39219665527344,
      "loss": 0.6883,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.056688953191041946,
      "rewards/margins": 0.010812098160386086,
      "rewards/rejected": 0.04587685689330101,
      "step": 580
    },
    {
      "epoch": 0.472,
      "grad_norm": 3.110258102416992,
      "learning_rate": 2.3560000000000004e-06,
      "logits/chosen": 0.6822569966316223,
      "logits/rejected": 0.6781023144721985,
      "logps/chosen": -215.3318328857422,
      "logps/rejected": -217.4102020263672,
      "loss": 0.689,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.06999214738607407,
      "rewards/margins": 0.01019731443375349,
      "rewards/rejected": 0.0597948394715786,
      "step": 590
    },
    {
      "epoch": 0.48,
      "grad_norm": 3.3028745651245117,
      "learning_rate": 2.3960000000000004e-06,
      "logits/chosen": 0.6857186555862427,
      "logits/rejected": 0.8173799514770508,
      "logps/chosen": -199.44915771484375,
      "logps/rejected": -205.1499481201172,
      "loss": 0.6833,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.07196470350027084,
      "rewards/margins": 0.021447917446494102,
      "rewards/rejected": 0.05051679164171219,
      "step": 600
    },
    {
      "epoch": 0.488,
      "grad_norm": 3.5192556381225586,
      "learning_rate": 2.4360000000000005e-06,
      "logits/chosen": 0.664252758026123,
      "logits/rejected": 0.7970002889633179,
      "logps/chosen": -183.03111267089844,
      "logps/rejected": -200.3971405029297,
      "loss": 0.6889,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.09819995611906052,
      "rewards/margins": 0.01058993861079216,
      "rewards/rejected": 0.08761002123355865,
      "step": 610
    },
    {
      "epoch": 0.496,
      "grad_norm": 3.326347827911377,
      "learning_rate": 2.476e-06,
      "logits/chosen": 0.7640261054039001,
      "logits/rejected": 0.7449796795845032,
      "logps/chosen": -186.8088836669922,
      "logps/rejected": -198.9549560546875,
      "loss": 0.685,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.10616890341043472,
      "rewards/margins": 0.01862834393978119,
      "rewards/rejected": 0.08754055947065353,
      "step": 620
    },
    {
      "epoch": 0.504,
      "grad_norm": 3.3991470336914062,
      "learning_rate": 2.516e-06,
      "logits/chosen": 0.6187211275100708,
      "logits/rejected": 0.5761955380439758,
      "logps/chosen": -191.6369171142578,
      "logps/rejected": -208.64756774902344,
      "loss": 0.6814,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.10338740795850754,
      "rewards/margins": 0.028011484071612358,
      "rewards/rejected": 0.07537592202425003,
      "step": 630
    },
    {
      "epoch": 0.512,
      "grad_norm": 3.6961252689361572,
      "learning_rate": 2.556e-06,
      "logits/chosen": 0.6848690509796143,
      "logits/rejected": 0.679492175579071,
      "logps/chosen": -211.4331512451172,
      "logps/rejected": -206.33084106445312,
      "loss": 0.6836,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.09747160971164703,
      "rewards/margins": 0.0221050176769495,
      "rewards/rejected": 0.07536659389734268,
      "step": 640
    },
    {
      "epoch": 0.52,
      "grad_norm": 3.4172682762145996,
      "learning_rate": 2.5960000000000002e-06,
      "logits/chosen": 0.6366490721702576,
      "logits/rejected": 0.5817402005195618,
      "logps/chosen": -209.3129425048828,
      "logps/rejected": -210.4381866455078,
      "loss": 0.6758,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.12278693169355392,
      "rewards/margins": 0.03931554779410362,
      "rewards/rejected": 0.08347138017416,
      "step": 650
    },
    {
      "epoch": 0.528,
      "grad_norm": 3.1855289936065674,
      "learning_rate": 2.6360000000000003e-06,
      "logits/chosen": 0.6927405595779419,
      "logits/rejected": 0.6666523218154907,
      "logps/chosen": -204.4365234375,
      "logps/rejected": -215.52566528320312,
      "loss": 0.6857,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.11687055975198746,
      "rewards/margins": 0.01899743638932705,
      "rewards/rejected": 0.09787313640117645,
      "step": 660
    },
    {
      "epoch": 0.536,
      "grad_norm": 3.221082925796509,
      "learning_rate": 2.6760000000000003e-06,
      "logits/chosen": 0.675836980342865,
      "logits/rejected": 0.6653029322624207,
      "logps/chosen": -190.93809509277344,
      "logps/rejected": -192.1734161376953,
      "loss": 0.6701,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.16091938316822052,
      "rewards/margins": 0.05060316249728203,
      "rewards/rejected": 0.1103162169456482,
      "step": 670
    },
    {
      "epoch": 0.544,
      "grad_norm": 3.2758514881134033,
      "learning_rate": 2.7160000000000003e-06,
      "logits/chosen": 0.6717751622200012,
      "logits/rejected": 0.5811682343482971,
      "logps/chosen": -208.1886749267578,
      "logps/rejected": -225.54039001464844,
      "loss": 0.6732,
      "rewards/accuracies": 0.6500000357627869,
      "rewards/chosen": 0.17058324813842773,
      "rewards/margins": 0.046290840953588486,
      "rewards/rejected": 0.12429242581129074,
      "step": 680
    },
    {
      "epoch": 0.552,
      "grad_norm": 3.140164375305176,
      "learning_rate": 2.7560000000000004e-06,
      "logits/chosen": 0.7419080138206482,
      "logits/rejected": 0.6190251111984253,
      "logps/chosen": -193.7239227294922,
      "logps/rejected": -195.77371215820312,
      "loss": 0.6682,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.2087523490190506,
      "rewards/margins": 0.05501072481274605,
      "rewards/rejected": 0.15374161303043365,
      "step": 690
    },
    {
      "epoch": 0.56,
      "grad_norm": 3.4456276893615723,
      "learning_rate": 2.7960000000000004e-06,
      "logits/chosen": 0.7370930910110474,
      "logits/rejected": 0.8321747779846191,
      "logps/chosen": -186.71826171875,
      "logps/rejected": -214.2365264892578,
      "loss": 0.6777,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.22456181049346924,
      "rewards/margins": 0.03610905632376671,
      "rewards/rejected": 0.18845276534557343,
      "step": 700
    },
    {
      "epoch": 0.568,
      "grad_norm": 3.7341623306274414,
      "learning_rate": 2.8360000000000005e-06,
      "logits/chosen": 0.7567041516304016,
      "logits/rejected": 0.7720674872398376,
      "logps/chosen": -190.39320373535156,
      "logps/rejected": -222.18826293945312,
      "loss": 0.6632,
      "rewards/accuracies": 0.6500000357627869,
      "rewards/chosen": 0.22317533195018768,
      "rewards/margins": 0.06986651569604874,
      "rewards/rejected": 0.15330885350704193,
      "step": 710
    },
    {
      "epoch": 0.576,
      "grad_norm": 2.9456615447998047,
      "learning_rate": 2.8760000000000005e-06,
      "logits/chosen": 0.6696327328681946,
      "logits/rejected": 0.697002649307251,
      "logps/chosen": -203.71766662597656,
      "logps/rejected": -222.6426239013672,
      "loss": 0.6658,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.22747135162353516,
      "rewards/margins": 0.06573095172643661,
      "rewards/rejected": 0.16174040734767914,
      "step": 720
    },
    {
      "epoch": 0.584,
      "grad_norm": 3.985300302505493,
      "learning_rate": 2.9160000000000005e-06,
      "logits/chosen": 0.6535480618476868,
      "logits/rejected": 0.7352523803710938,
      "logps/chosen": -202.01422119140625,
      "logps/rejected": -223.960693359375,
      "loss": 0.6824,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.1612798571586609,
      "rewards/margins": 0.03076176717877388,
      "rewards/rejected": 0.1305180937051773,
      "step": 730
    },
    {
      "epoch": 0.592,
      "grad_norm": 3.8837013244628906,
      "learning_rate": 2.956e-06,
      "logits/chosen": 0.6671428084373474,
      "logits/rejected": 0.6649322509765625,
      "logps/chosen": -191.92437744140625,
      "logps/rejected": -226.7194366455078,
      "loss": 0.6725,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.16540491580963135,
      "rewards/margins": 0.051003750413656235,
      "rewards/rejected": 0.11440115422010422,
      "step": 740
    },
    {
      "epoch": 0.6,
      "grad_norm": 3.762125015258789,
      "learning_rate": 2.996e-06,
      "logits/chosen": 0.7455517649650574,
      "logits/rejected": 0.6791686415672302,
      "logps/chosen": -201.34228515625,
      "logps/rejected": -205.9029998779297,
      "loss": 0.6731,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.15725918114185333,
      "rewards/margins": 0.05198857933282852,
      "rewards/rejected": 0.10527060180902481,
      "step": 750
    },
    {
      "epoch": 0.608,
      "grad_norm": 4.017007827758789,
      "learning_rate": 3.0360000000000002e-06,
      "logits/chosen": 0.5845433473587036,
      "logits/rejected": 0.6173285841941833,
      "logps/chosen": -211.0860595703125,
      "logps/rejected": -239.35105895996094,
      "loss": 0.6691,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.1531037837266922,
      "rewards/margins": 0.06398037821054459,
      "rewards/rejected": 0.08912339061498642,
      "step": 760
    },
    {
      "epoch": 0.616,
      "grad_norm": 3.3737785816192627,
      "learning_rate": 3.0760000000000003e-06,
      "logits/chosen": 0.6799761056900024,
      "logits/rejected": 0.7338641285896301,
      "logps/chosen": -197.92454528808594,
      "logps/rejected": -196.76071166992188,
      "loss": 0.6861,
      "rewards/accuracies": 0.5250000357627869,
      "rewards/chosen": 0.14501558244228363,
      "rewards/margins": 0.024691270664334297,
      "rewards/rejected": 0.12032430619001389,
      "step": 770
    },
    {
      "epoch": 0.624,
      "grad_norm": 3.9343101978302,
      "learning_rate": 3.1160000000000003e-06,
      "logits/chosen": 0.6844798922538757,
      "logits/rejected": 0.6486098766326904,
      "logps/chosen": -209.14988708496094,
      "logps/rejected": -208.64990234375,
      "loss": 0.684,
      "rewards/accuracies": 0.5250000357627869,
      "rewards/chosen": 0.12247097492218018,
      "rewards/margins": 0.03311007842421532,
      "rewards/rejected": 0.08936089277267456,
      "step": 780
    },
    {
      "epoch": 0.632,
      "grad_norm": 3.3528807163238525,
      "learning_rate": 3.1560000000000004e-06,
      "logits/chosen": 0.5940313339233398,
      "logits/rejected": 0.5495096445083618,
      "logps/chosen": -200.02276611328125,
      "logps/rejected": -203.39085388183594,
      "loss": 0.6646,
      "rewards/accuracies": 0.5875000357627869,
      "rewards/chosen": 0.21792371571063995,
      "rewards/margins": 0.07114797830581665,
      "rewards/rejected": 0.1467757374048233,
      "step": 790
    },
    {
      "epoch": 0.64,
      "grad_norm": 3.5861222743988037,
      "learning_rate": 3.1960000000000004e-06,
      "logits/chosen": 0.7208414077758789,
      "logits/rejected": 0.7849133610725403,
      "logps/chosen": -168.2899932861328,
      "logps/rejected": -183.3815155029297,
      "loss": 0.6719,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.22198240458965302,
      "rewards/margins": 0.059726305305957794,
      "rewards/rejected": 0.162256121635437,
      "step": 800
    },
    {
      "epoch": 0.648,
      "grad_norm": 3.695352077484131,
      "learning_rate": 3.2360000000000004e-06,
      "logits/chosen": 0.7138350009918213,
      "logits/rejected": 0.7761127352714539,
      "logps/chosen": -191.26907348632812,
      "logps/rejected": -199.15538024902344,
      "loss": 0.6862,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.17254018783569336,
      "rewards/margins": 0.027448756620287895,
      "rewards/rejected": 0.14509142935276031,
      "step": 810
    },
    {
      "epoch": 0.656,
      "grad_norm": 4.936346054077148,
      "learning_rate": 3.2760000000000005e-06,
      "logits/chosen": 0.651190459728241,
      "logits/rejected": 0.6843032240867615,
      "logps/chosen": -185.53834533691406,
      "logps/rejected": -210.05166625976562,
      "loss": 0.6538,
      "rewards/accuracies": 0.6500000357627869,
      "rewards/chosen": 0.17089328169822693,
      "rewards/margins": 0.09923329949378967,
      "rewards/rejected": 0.07165997475385666,
      "step": 820
    },
    {
      "epoch": 0.664,
      "grad_norm": 3.7326865196228027,
      "learning_rate": 3.3160000000000005e-06,
      "logits/chosen": 0.8238712549209595,
      "logits/rejected": 0.849129855632782,
      "logps/chosen": -199.64431762695312,
      "logps/rejected": -206.5725555419922,
      "loss": 0.6559,
      "rewards/accuracies": 0.6500000357627869,
      "rewards/chosen": 0.14575400948524475,
      "rewards/margins": 0.09553883224725723,
      "rewards/rejected": 0.05021516606211662,
      "step": 830
    },
    {
      "epoch": 0.672,
      "grad_norm": 3.8525242805480957,
      "learning_rate": 3.3560000000000006e-06,
      "logits/chosen": 0.68717360496521,
      "logits/rejected": 0.775698184967041,
      "logps/chosen": -198.4635009765625,
      "logps/rejected": -218.93565368652344,
      "loss": 0.6672,
      "rewards/accuracies": 0.5875000357627869,
      "rewards/chosen": 0.03423814848065376,
      "rewards/margins": 0.07274122536182404,
      "rewards/rejected": -0.03850306198000908,
      "step": 840
    },
    {
      "epoch": 0.68,
      "grad_norm": 3.930669069290161,
      "learning_rate": 3.3960000000000006e-06,
      "logits/chosen": 0.7518499493598938,
      "logits/rejected": 0.8216703534126282,
      "logps/chosen": -205.54248046875,
      "logps/rejected": -207.52676391601562,
      "loss": 0.6571,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.12743444740772247,
      "rewards/margins": 0.0990920141339302,
      "rewards/rejected": 0.028342435136437416,
      "step": 850
    },
    {
      "epoch": 0.688,
      "grad_norm": 3.8377163410186768,
      "learning_rate": 3.4360000000000006e-06,
      "logits/chosen": 0.7670461535453796,
      "logits/rejected": 0.7580811977386475,
      "logps/chosen": -182.31930541992188,
      "logps/rejected": -194.57054138183594,
      "loss": 0.6654,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.18013069033622742,
      "rewards/margins": 0.07547135651111603,
      "rewards/rejected": 0.10465934127569199,
      "step": 860
    },
    {
      "epoch": 0.696,
      "grad_norm": 4.037013053894043,
      "learning_rate": 3.4760000000000007e-06,
      "logits/chosen": 0.6302778720855713,
      "logits/rejected": 0.5533519983291626,
      "logps/chosen": -201.7500457763672,
      "logps/rejected": -217.78018188476562,
      "loss": 0.6598,
      "rewards/accuracies": 0.5875000357627869,
      "rewards/chosen": 0.1704377979040146,
      "rewards/margins": 0.10010436922311783,
      "rewards/rejected": 0.07033341377973557,
      "step": 870
    },
    {
      "epoch": 0.704,
      "grad_norm": 3.9927608966827393,
      "learning_rate": 3.5160000000000007e-06,
      "logits/chosen": 0.6597375273704529,
      "logits/rejected": 0.6159626245498657,
      "logps/chosen": -193.73165893554688,
      "logps/rejected": -198.38595581054688,
      "loss": 0.6935,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": 0.19550852477550507,
      "rewards/margins": 0.04272466525435448,
      "rewards/rejected": 0.15278387069702148,
      "step": 880
    },
    {
      "epoch": 0.712,
      "grad_norm": 3.5315475463867188,
      "learning_rate": 3.5560000000000008e-06,
      "logits/chosen": 0.685095489025116,
      "logits/rejected": 0.6823846697807312,
      "logps/chosen": -201.48406982421875,
      "logps/rejected": -212.81094360351562,
      "loss": 0.6487,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.20819902420043945,
      "rewards/margins": 0.10897611826658249,
      "rewards/rejected": 0.09922291338443756,
      "step": 890
    },
    {
      "epoch": 0.72,
      "grad_norm": 3.673504114151001,
      "learning_rate": 3.596e-06,
      "logits/chosen": 0.7472502589225769,
      "logits/rejected": 0.623907208442688,
      "logps/chosen": -183.38999938964844,
      "logps/rejected": -181.52911376953125,
      "loss": 0.6561,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.28343966603279114,
      "rewards/margins": 0.09876996278762817,
      "rewards/rejected": 0.18466970324516296,
      "step": 900
    },
    {
      "epoch": 0.728,
      "grad_norm": 4.183164596557617,
      "learning_rate": 3.636e-06,
      "logits/chosen": 0.6977134346961975,
      "logits/rejected": 0.6741080284118652,
      "logps/chosen": -192.44366455078125,
      "logps/rejected": -186.1464080810547,
      "loss": 0.6923,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.25711265206336975,
      "rewards/margins": 0.020020050927996635,
      "rewards/rejected": 0.23709259927272797,
      "step": 910
    },
    {
      "epoch": 0.736,
      "grad_norm": 3.74733304977417,
      "learning_rate": 3.676e-06,
      "logits/chosen": 0.7447720170021057,
      "logits/rejected": 0.6725283265113831,
      "logps/chosen": -186.6201934814453,
      "logps/rejected": -204.659423828125,
      "loss": 0.6199,
      "rewards/accuracies": 0.7125000357627869,
      "rewards/chosen": 0.3455497920513153,
      "rewards/margins": 0.18741247057914734,
      "rewards/rejected": 0.15813732147216797,
      "step": 920
    },
    {
      "epoch": 0.744,
      "grad_norm": 4.2909255027771,
      "learning_rate": 3.716e-06,
      "logits/chosen": 0.656113862991333,
      "logits/rejected": 0.6516739130020142,
      "logps/chosen": -197.02088928222656,
      "logps/rejected": -213.64414978027344,
      "loss": 0.6621,
      "rewards/accuracies": 0.6500000357627869,
      "rewards/chosen": 0.20855574309825897,
      "rewards/margins": 0.08712134510278702,
      "rewards/rejected": 0.12143440544605255,
      "step": 930
    },
    {
      "epoch": 0.752,
      "grad_norm": 4.346622943878174,
      "learning_rate": 3.756e-06,
      "logits/chosen": 0.7425007224082947,
      "logits/rejected": 0.7672116160392761,
      "logps/chosen": -164.4781494140625,
      "logps/rejected": -189.1973876953125,
      "loss": 0.6322,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.2733791768550873,
      "rewards/margins": 0.15534010529518127,
      "rewards/rejected": 0.1180390939116478,
      "step": 940
    },
    {
      "epoch": 0.76,
      "grad_norm": 3.8516108989715576,
      "learning_rate": 3.796e-06,
      "logits/chosen": 0.6322047710418701,
      "logits/rejected": 0.6729938387870789,
      "logps/chosen": -179.63258361816406,
      "logps/rejected": -213.58462524414062,
      "loss": 0.6348,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.10944360494613647,
      "rewards/margins": 0.161842480301857,
      "rewards/rejected": -0.05239887163043022,
      "step": 950
    },
    {
      "epoch": 0.768,
      "grad_norm": 3.8706507682800293,
      "learning_rate": 3.836e-06,
      "logits/chosen": 0.7383615374565125,
      "logits/rejected": 0.645232617855072,
      "logps/chosen": -209.4774932861328,
      "logps/rejected": -218.1513214111328,
      "loss": 0.6402,
      "rewards/accuracies": 0.6500000357627869,
      "rewards/chosen": 0.016749557107686996,
      "rewards/margins": 0.16539375483989716,
      "rewards/rejected": -0.14864420890808105,
      "step": 960
    },
    {
      "epoch": 0.776,
      "grad_norm": 3.860457420349121,
      "learning_rate": 3.876000000000001e-06,
      "logits/chosen": 0.7996999025344849,
      "logits/rejected": 0.7975447773933411,
      "logps/chosen": -197.4671630859375,
      "logps/rejected": -207.5522003173828,
      "loss": 0.6625,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.1389116793870926,
      "rewards/margins": 0.09984000772237778,
      "rewards/rejected": 0.03907167166471481,
      "step": 970
    },
    {
      "epoch": 0.784,
      "grad_norm": 4.529206275939941,
      "learning_rate": 3.916e-06,
      "logits/chosen": 0.7569703459739685,
      "logits/rejected": 0.7382944226264954,
      "logps/chosen": -191.17686462402344,
      "logps/rejected": -207.4125518798828,
      "loss": 0.625,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.0675772950053215,
      "rewards/margins": 0.2260824292898178,
      "rewards/rejected": -0.15850511193275452,
      "step": 980
    },
    {
      "epoch": 0.792,
      "grad_norm": 3.819338083267212,
      "learning_rate": 3.956000000000001e-06,
      "logits/chosen": 0.7830851674079895,
      "logits/rejected": 0.7947285771369934,
      "logps/chosen": -205.89517211914062,
      "logps/rejected": -224.50967407226562,
      "loss": 0.6524,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.003182247979566455,
      "rewards/margins": 0.1461525410413742,
      "rewards/rejected": -0.14933478832244873,
      "step": 990
    },
    {
      "epoch": 0.8,
      "grad_norm": 3.4207472801208496,
      "learning_rate": 3.996e-06,
      "logits/chosen": 0.6669479012489319,
      "logits/rejected": 0.7112082839012146,
      "logps/chosen": -205.70693969726562,
      "logps/rejected": -223.3166046142578,
      "loss": 0.6352,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.08382979780435562,
      "rewards/margins": 0.19078943133354187,
      "rewards/rejected": -0.10695965588092804,
      "step": 1000
    },
    {
      "epoch": 0.808,
      "grad_norm": 4.2891526222229,
      "learning_rate": 4.036000000000001e-06,
      "logits/chosen": 0.6510803699493408,
      "logits/rejected": 0.6577145457267761,
      "logps/chosen": -191.43975830078125,
      "logps/rejected": -217.14639282226562,
      "loss": 0.6366,
      "rewards/accuracies": 0.6500000357627869,
      "rewards/chosen": 0.09330642223358154,
      "rewards/margins": 0.17022596299648285,
      "rewards/rejected": -0.07691951841115952,
      "step": 1010
    },
    {
      "epoch": 0.816,
      "grad_norm": 4.56123685836792,
      "learning_rate": 4.0760000000000004e-06,
      "logits/chosen": 0.6556877493858337,
      "logits/rejected": 0.6388117671012878,
      "logps/chosen": -197.0552520751953,
      "logps/rejected": -203.9344482421875,
      "loss": 0.6354,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.07625667005777359,
      "rewards/margins": 0.21561256051063538,
      "rewards/rejected": -0.13935589790344238,
      "step": 1020
    },
    {
      "epoch": 0.824,
      "grad_norm": 3.7361807823181152,
      "learning_rate": 4.116000000000001e-06,
      "logits/chosen": 0.7679457068443298,
      "logits/rejected": 0.7817143797874451,
      "logps/chosen": -205.3223419189453,
      "logps/rejected": -211.694580078125,
      "loss": 0.654,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.11138266324996948,
      "rewards/margins": 0.13815438747406006,
      "rewards/rejected": -0.026771722361445427,
      "step": 1030
    },
    {
      "epoch": 0.832,
      "grad_norm": 5.420875072479248,
      "learning_rate": 4.1560000000000005e-06,
      "logits/chosen": 0.6636489629745483,
      "logits/rejected": 0.692511260509491,
      "logps/chosen": -207.5504150390625,
      "logps/rejected": -225.18753051757812,
      "loss": 0.6344,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.1605958491563797,
      "rewards/margins": 0.1674598604440689,
      "rewards/rejected": -0.00686401454731822,
      "step": 1040
    },
    {
      "epoch": 0.84,
      "grad_norm": 3.272894859313965,
      "learning_rate": 4.196e-06,
      "logits/chosen": 0.7486108541488647,
      "logits/rejected": 0.7712265253067017,
      "logps/chosen": -194.90943908691406,
      "logps/rejected": -205.329345703125,
      "loss": 0.6322,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.22325454652309418,
      "rewards/margins": 0.22328029572963715,
      "rewards/rejected": -2.5737286705407314e-05,
      "step": 1050
    },
    {
      "epoch": 0.848,
      "grad_norm": 5.14847993850708,
      "learning_rate": 4.236e-06,
      "logits/chosen": 0.7043625116348267,
      "logits/rejected": 0.7297138571739197,
      "logps/chosen": -214.93301391601562,
      "logps/rejected": -221.634765625,
      "loss": 0.6797,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.19927440583705902,
      "rewards/margins": 0.10101515054702759,
      "rewards/rejected": 0.09825924783945084,
      "step": 1060
    },
    {
      "epoch": 0.856,
      "grad_norm": 4.329235076904297,
      "learning_rate": 4.276e-06,
      "logits/chosen": 0.7435252070426941,
      "logits/rejected": 0.7416861653327942,
      "logps/chosen": -206.2967987060547,
      "logps/rejected": -221.4429168701172,
      "loss": 0.6532,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.21223540604114532,
      "rewards/margins": 0.14147967100143433,
      "rewards/rejected": 0.070755735039711,
      "step": 1070
    },
    {
      "epoch": 0.864,
      "grad_norm": 3.7461328506469727,
      "learning_rate": 4.316e-06,
      "logits/chosen": 0.8374633193016052,
      "logits/rejected": 0.8583774566650391,
      "logps/chosen": -203.61318969726562,
      "logps/rejected": -218.10350036621094,
      "loss": 0.6082,
      "rewards/accuracies": 0.6500000357627869,
      "rewards/chosen": 0.2129761427640915,
      "rewards/margins": 0.223725363612175,
      "rewards/rejected": -0.010749227367341518,
      "step": 1080
    },
    {
      "epoch": 0.872,
      "grad_norm": 3.664881467819214,
      "learning_rate": 4.356e-06,
      "logits/chosen": 0.7724778652191162,
      "logits/rejected": 0.7607793807983398,
      "logps/chosen": -194.4547576904297,
      "logps/rejected": -221.48941040039062,
      "loss": 0.6174,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.14989307522773743,
      "rewards/margins": 0.24988269805908203,
      "rewards/rejected": -0.0999896302819252,
      "step": 1090
    },
    {
      "epoch": 0.88,
      "grad_norm": 3.962773323059082,
      "learning_rate": 4.396e-06,
      "logits/chosen": 0.8602146506309509,
      "logits/rejected": 0.7610459327697754,
      "logps/chosen": -186.56153869628906,
      "logps/rejected": -191.79127502441406,
      "loss": 0.609,
      "rewards/accuracies": 0.6500000357627869,
      "rewards/chosen": 0.13783621788024902,
      "rewards/margins": 0.24294762313365936,
      "rewards/rejected": -0.10511143505573273,
      "step": 1100
    },
    {
      "epoch": 0.888,
      "grad_norm": 3.6281254291534424,
      "learning_rate": 4.436e-06,
      "logits/chosen": 0.7969589233398438,
      "logits/rejected": 0.7449702620506287,
      "logps/chosen": -199.1891326904297,
      "logps/rejected": -212.9011688232422,
      "loss": 0.6054,
      "rewards/accuracies": 0.6500000357627869,
      "rewards/chosen": -0.007032311055809259,
      "rewards/margins": 0.28387296199798584,
      "rewards/rejected": -0.29090526700019836,
      "step": 1110
    },
    {
      "epoch": 0.896,
      "grad_norm": 4.473705768585205,
      "learning_rate": 4.476e-06,
      "logits/chosen": 0.8331708312034607,
      "logits/rejected": 0.8655223846435547,
      "logps/chosen": -190.3954620361328,
      "logps/rejected": -204.8839874267578,
      "loss": 0.6281,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.1991288810968399,
      "rewards/margins": 0.20112191140651703,
      "rewards/rejected": -0.0019930123817175627,
      "step": 1120
    },
    {
      "epoch": 0.904,
      "grad_norm": 6.322517395019531,
      "learning_rate": 4.5160000000000005e-06,
      "logits/chosen": 0.8980779051780701,
      "logits/rejected": 0.8646361231803894,
      "logps/chosen": -207.82090759277344,
      "logps/rejected": -202.1289520263672,
      "loss": 0.6373,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.1084756851196289,
      "rewards/margins": 0.20956145226955414,
      "rewards/rejected": -0.10108575969934464,
      "step": 1130
    },
    {
      "epoch": 0.912,
      "grad_norm": 6.67050838470459,
      "learning_rate": 4.556e-06,
      "logits/chosen": 0.6762229800224304,
      "logits/rejected": 0.843460738658905,
      "logps/chosen": -205.12686157226562,
      "logps/rejected": -238.93972778320312,
      "loss": 0.6326,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.20181012153625488,
      "rewards/margins": 0.23140645027160645,
      "rewards/rejected": -0.43321657180786133,
      "step": 1140
    },
    {
      "epoch": 0.92,
      "grad_norm": 3.84826922416687,
      "learning_rate": 4.5960000000000006e-06,
      "logits/chosen": 0.8595611453056335,
      "logits/rejected": 0.7575063705444336,
      "logps/chosen": -191.56690979003906,
      "logps/rejected": -211.3734588623047,
      "loss": 0.7016,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.017901450395584106,
      "rewards/margins": 0.08475067466497421,
      "rewards/rejected": -0.10265214741230011,
      "step": 1150
    },
    {
      "epoch": 0.928,
      "grad_norm": 5.697420120239258,
      "learning_rate": 4.636e-06,
      "logits/chosen": 0.7715900540351868,
      "logits/rejected": 0.6638416051864624,
      "logps/chosen": -197.2618865966797,
      "logps/rejected": -178.20579528808594,
      "loss": 0.6362,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.10496443510055542,
      "rewards/margins": 0.22472666203975677,
      "rewards/rejected": -0.11976220458745956,
      "step": 1160
    },
    {
      "epoch": 0.936,
      "grad_norm": 4.553537845611572,
      "learning_rate": 4.676000000000001e-06,
      "logits/chosen": 0.7215653657913208,
      "logits/rejected": 0.7404984831809998,
      "logps/chosen": -198.07424926757812,
      "logps/rejected": -231.4817657470703,
      "loss": 0.5759,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.14309090375900269,
      "rewards/margins": 0.4066457450389862,
      "rewards/rejected": -0.2635548412799835,
      "step": 1170
    },
    {
      "epoch": 0.944,
      "grad_norm": 6.022956848144531,
      "learning_rate": 4.716e-06,
      "logits/chosen": 0.8578891754150391,
      "logits/rejected": 0.8036954998970032,
      "logps/chosen": -202.0140380859375,
      "logps/rejected": -212.0448760986328,
      "loss": 0.668,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.03294707089662552,
      "rewards/margins": 0.15671278536319733,
      "rewards/rejected": -0.18965983390808105,
      "step": 1180
    },
    {
      "epoch": 0.952,
      "grad_norm": 4.317718029022217,
      "learning_rate": 4.756000000000001e-06,
      "logits/chosen": 0.8726083636283875,
      "logits/rejected": 0.9713365435600281,
      "logps/chosen": -185.42198181152344,
      "logps/rejected": -206.3362274169922,
      "loss": 0.6041,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.05594133213162422,
      "rewards/margins": 0.28803297877311707,
      "rewards/rejected": -0.3439743220806122,
      "step": 1190
    },
    {
      "epoch": 0.96,
      "grad_norm": 4.923951625823975,
      "learning_rate": 4.796e-06,
      "logits/chosen": 0.8178095817565918,
      "logits/rejected": 0.8852421641349792,
      "logps/chosen": -177.03765869140625,
      "logps/rejected": -189.92286682128906,
      "loss": 0.6351,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.016861964017152786,
      "rewards/margins": 0.20337799191474915,
      "rewards/rejected": -0.18651603162288666,
      "step": 1200
    },
    {
      "epoch": 0.968,
      "grad_norm": 4.542464256286621,
      "learning_rate": 4.836e-06,
      "logits/chosen": 0.8311338424682617,
      "logits/rejected": 0.7399374842643738,
      "logps/chosen": -202.60267639160156,
      "logps/rejected": -203.94236755371094,
      "loss": 0.6779,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.19154860079288483,
      "rewards/margins": 0.115830197930336,
      "rewards/rejected": -0.3073787987232208,
      "step": 1210
    },
    {
      "epoch": 0.976,
      "grad_norm": 6.436461925506592,
      "learning_rate": 4.876e-06,
      "logits/chosen": 0.794135570526123,
      "logits/rejected": 0.7919009327888489,
      "logps/chosen": -207.1306610107422,
      "logps/rejected": -205.37637329101562,
      "loss": 0.6314,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07707437127828598,
      "rewards/margins": 0.25921908020973206,
      "rewards/rejected": -0.33629345893859863,
      "step": 1220
    },
    {
      "epoch": 0.984,
      "grad_norm": 5.068552494049072,
      "learning_rate": 4.916e-06,
      "logits/chosen": 0.9020944833755493,
      "logits/rejected": 0.8565078973770142,
      "logps/chosen": -191.53636169433594,
      "logps/rejected": -192.14581298828125,
      "loss": 0.6288,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.21075907349586487,
      "rewards/margins": 0.26774677634239197,
      "rewards/rejected": -0.056987714022397995,
      "step": 1230
    },
    {
      "epoch": 0.992,
      "grad_norm": 3.66788649559021,
      "learning_rate": 4.9560000000000005e-06,
      "logits/chosen": 0.759829044342041,
      "logits/rejected": 0.7424188852310181,
      "logps/chosen": -196.9650115966797,
      "logps/rejected": -197.923583984375,
      "loss": 0.5986,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.22332988679409027,
      "rewards/margins": 0.3243139684200287,
      "rewards/rejected": -0.1009841114282608,
      "step": 1240
    },
    {
      "epoch": 1.0,
      "grad_norm": 5.5345940589904785,
      "learning_rate": 4.996e-06,
      "logits/chosen": 0.8046256899833679,
      "logits/rejected": 0.7486367225646973,
      "logps/chosen": -196.2307586669922,
      "logps/rejected": -213.2323455810547,
      "loss": 0.6443,
      "rewards/accuracies": 0.6500000357627869,
      "rewards/chosen": -0.04985840991139412,
      "rewards/margins": 0.21199627220630646,
      "rewards/rejected": -0.2618546485900879,
      "step": 1250
    },
    {
      "epoch": 1.008,
      "grad_norm": 3.8329803943634033,
      "learning_rate": 4.9999921043206356e-06,
      "logits/chosen": 0.8027931451797485,
      "logits/rejected": 0.7976822853088379,
      "logps/chosen": -198.8918914794922,
      "logps/rejected": -188.731201171875,
      "loss": 0.5909,
      "rewards/accuracies": 0.7125000357627869,
      "rewards/chosen": 0.2840830385684967,
      "rewards/margins": 0.3135008215904236,
      "rewards/rejected": -0.029417768120765686,
      "step": 1260
    },
    {
      "epoch": 1.016,
      "grad_norm": 5.155350685119629,
      "learning_rate": 4.99996481067822e-06,
      "logits/chosen": 0.8220119476318359,
      "logits/rejected": 0.8561745882034302,
      "logps/chosen": -206.8731231689453,
      "logps/rejected": -216.76406860351562,
      "loss": 0.5927,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.008052355609834194,
      "rewards/margins": 0.3377354145050049,
      "rewards/rejected": -0.3296830654144287,
      "step": 1270
    },
    {
      "epoch": 1.024,
      "grad_norm": 4.907769203186035,
      "learning_rate": 4.99991802180802e-06,
      "logits/chosen": 0.9498602151870728,
      "logits/rejected": 0.8992852568626404,
      "logps/chosen": -205.61570739746094,
      "logps/rejected": -227.8877716064453,
      "loss": 0.6093,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.015175617299973965,
      "rewards/margins": 0.2621200382709503,
      "rewards/rejected": -0.24694442749023438,
      "step": 1280
    },
    {
      "epoch": 1.032,
      "grad_norm": 5.946913242340088,
      "learning_rate": 4.999851738074904e-06,
      "logits/chosen": 0.9917473196983337,
      "logits/rejected": 0.9724864363670349,
      "logps/chosen": -179.44322204589844,
      "logps/rejected": -202.18516540527344,
      "loss": 0.6529,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.23127515614032745,
      "rewards/margins": 0.1929376870393753,
      "rewards/rejected": 0.038337476551532745,
      "step": 1290
    },
    {
      "epoch": 1.04,
      "grad_norm": 3.936194896697998,
      "learning_rate": 4.999765959995769e-06,
      "logits/chosen": 0.9820284247398376,
      "logits/rejected": 0.8659685254096985,
      "logps/chosen": -204.54019165039062,
      "logps/rejected": -193.43544006347656,
      "loss": 0.5873,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.2293062061071396,
      "rewards/margins": 0.34357786178588867,
      "rewards/rejected": -0.11427167803049088,
      "step": 1300
    },
    {
      "epoch": 1.048,
      "grad_norm": 4.1008076667785645,
      "learning_rate": 4.999660688239527e-06,
      "logits/chosen": 0.835074245929718,
      "logits/rejected": 0.7787433862686157,
      "logps/chosen": -188.2495880126953,
      "logps/rejected": -197.9439697265625,
      "loss": 0.6111,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.2561519742012024,
      "rewards/margins": 0.31733056902885437,
      "rewards/rejected": -0.06117859110236168,
      "step": 1310
    },
    {
      "epoch": 1.056,
      "grad_norm": 5.592878341674805,
      "learning_rate": 4.9995359236271094e-06,
      "logits/chosen": 0.883532702922821,
      "logits/rejected": 0.8534946441650391,
      "logps/chosen": -199.9098358154297,
      "logps/rejected": -197.54641723632812,
      "loss": 0.6772,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.2197556495666504,
      "rewards/margins": 0.17633479833602905,
      "rewards/rejected": 0.04342082887887955,
      "step": 1320
    },
    {
      "epoch": 1.064,
      "grad_norm": 4.941606521606445,
      "learning_rate": 4.999391667131456e-06,
      "logits/chosen": 0.860753059387207,
      "logits/rejected": 0.8954046368598938,
      "logps/chosen": -178.4503936767578,
      "logps/rejected": -196.09449768066406,
      "loss": 0.6786,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.48869457840919495,
      "rewards/margins": 0.12707066535949707,
      "rewards/rejected": 0.36162394285202026,
      "step": 1330
    },
    {
      "epoch": 1.072,
      "grad_norm": 5.233992576599121,
      "learning_rate": 4.999227919877506e-06,
      "logits/chosen": 0.8799529075622559,
      "logits/rejected": 0.8783776164054871,
      "logps/chosen": -195.3287811279297,
      "logps/rejected": -195.21665954589844,
      "loss": 0.6231,
      "rewards/accuracies": 0.6500000357627869,
      "rewards/chosen": 0.4896652400493622,
      "rewards/margins": 0.27517566084861755,
      "rewards/rejected": 0.2144896239042282,
      "step": 1340
    },
    {
      "epoch": 1.08,
      "grad_norm": 8.488966941833496,
      "learning_rate": 4.999044683142196e-06,
      "logits/chosen": 1.042872667312622,
      "logits/rejected": 0.8863340616226196,
      "logps/chosen": -197.02415466308594,
      "logps/rejected": -202.64767456054688,
      "loss": 0.6329,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.4562924802303314,
      "rewards/margins": 0.19963227212429047,
      "rewards/rejected": 0.25666019320487976,
      "step": 1350
    },
    {
      "epoch": 1.088,
      "grad_norm": 4.995717525482178,
      "learning_rate": 4.99884195835444e-06,
      "logits/chosen": 0.8799748420715332,
      "logits/rejected": 0.8477336764335632,
      "logps/chosen": -191.94471740722656,
      "logps/rejected": -206.7391815185547,
      "loss": 0.6352,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.18572449684143066,
      "rewards/margins": 0.2937488555908203,
      "rewards/rejected": -0.10802433639764786,
      "step": 1360
    },
    {
      "epoch": 1.096,
      "grad_norm": 4.870668888092041,
      "learning_rate": 4.998619747095129e-06,
      "logits/chosen": 0.7523741722106934,
      "logits/rejected": 0.7566614747047424,
      "logps/chosen": -200.09573364257812,
      "logps/rejected": -209.72647094726562,
      "loss": 0.6096,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.1902216672897339,
      "rewards/margins": 0.32739758491516113,
      "rewards/rejected": -0.5176193118095398,
      "step": 1370
    },
    {
      "epoch": 1.104,
      "grad_norm": 4.8878889083862305,
      "learning_rate": 4.998378051097111e-06,
      "logits/chosen": 0.6805158257484436,
      "logits/rejected": 0.6997960805892944,
      "logps/chosen": -211.3839874267578,
      "logps/rejected": -243.62759399414062,
      "loss": 0.6292,
      "rewards/accuracies": 0.6500000357627869,
      "rewards/chosen": -0.17687633633613586,
      "rewards/margins": 0.31230995059013367,
      "rewards/rejected": -0.48918625712394714,
      "step": 1380
    },
    {
      "epoch": 1.112,
      "grad_norm": 4.575040817260742,
      "learning_rate": 4.998116872245178e-06,
      "logits/chosen": 0.9538719058036804,
      "logits/rejected": 0.8567468523979187,
      "logps/chosen": -197.8600311279297,
      "logps/rejected": -194.09893798828125,
      "loss": 0.6508,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0491158552467823,
      "rewards/margins": 0.23099613189697266,
      "rewards/rejected": -0.28011199831962585,
      "step": 1390
    },
    {
      "epoch": 1.12,
      "grad_norm": 4.09750509262085,
      "learning_rate": 4.997836212576057e-06,
      "logits/chosen": 0.7558371424674988,
      "logits/rejected": 0.7620294690132141,
      "logps/chosen": -210.34483337402344,
      "logps/rejected": -220.9149627685547,
      "loss": 0.583,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07163530588150024,
      "rewards/margins": 0.40291815996170044,
      "rewards/rejected": -0.4745534360408783,
      "step": 1400
    },
    {
      "epoch": 1.1280000000000001,
      "grad_norm": 4.7588887214660645,
      "learning_rate": 4.997536074278388e-06,
      "logits/chosen": 0.7905246019363403,
      "logits/rejected": 0.8206660151481628,
      "logps/chosen": -211.0833740234375,
      "logps/rejected": -218.993896484375,
      "loss": 0.6144,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.14193372428417206,
      "rewards/margins": 0.2602233588695526,
      "rewards/rejected": -0.4021570682525635,
      "step": 1410
    },
    {
      "epoch": 1.1360000000000001,
      "grad_norm": 6.200011730194092,
      "learning_rate": 4.9972164596927085e-06,
      "logits/chosen": 0.9080677032470703,
      "logits/rejected": 0.8735291361808777,
      "logps/chosen": -194.74559020996094,
      "logps/rejected": -200.72265625,
      "loss": 0.6563,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.202849343419075,
      "rewards/margins": 0.19263452291488647,
      "rewards/rejected": -0.3954838514328003,
      "step": 1420
    },
    {
      "epoch": 1.144,
      "grad_norm": 7.389719486236572,
      "learning_rate": 4.996877371311439e-06,
      "logits/chosen": 0.7800935506820679,
      "logits/rejected": 0.8093517422676086,
      "logps/chosen": -211.73667907714844,
      "logps/rejected": -222.60000610351562,
      "loss": 0.5757,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2545677125453949,
      "rewards/margins": 0.49100762605667114,
      "rewards/rejected": -0.7455753087997437,
      "step": 1430
    },
    {
      "epoch": 1.152,
      "grad_norm": 4.560316562652588,
      "learning_rate": 4.996518811778858e-06,
      "logits/chosen": 0.9419007301330566,
      "logits/rejected": 0.8434064984321594,
      "logps/chosen": -193.9005889892578,
      "logps/rejected": -196.77064514160156,
      "loss": 0.5937,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.10783553123474121,
      "rewards/margins": 0.35865020751953125,
      "rewards/rejected": -0.25081467628479004,
      "step": 1440
    },
    {
      "epoch": 1.16,
      "grad_norm": 4.446018218994141,
      "learning_rate": 4.996140783891085e-06,
      "logits/chosen": 0.8335108757019043,
      "logits/rejected": 0.7712218165397644,
      "logps/chosen": -200.7833251953125,
      "logps/rejected": -217.6037139892578,
      "loss": 0.5632,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -0.268947035074234,
      "rewards/margins": 0.4497573971748352,
      "rewards/rejected": -0.7187044024467468,
      "step": 1450
    },
    {
      "epoch": 1.168,
      "grad_norm": 4.578011989593506,
      "learning_rate": 4.9957432905960575e-06,
      "logits/chosen": 0.8321472406387329,
      "logits/rejected": 0.8203474283218384,
      "logps/chosen": -179.63121032714844,
      "logps/rejected": -218.509033203125,
      "loss": 0.5283,
      "rewards/accuracies": 0.7125000357627869,
      "rewards/chosen": -0.2659996449947357,
      "rewards/margins": 0.5776376724243164,
      "rewards/rejected": -0.8436372876167297,
      "step": 1460
    },
    {
      "epoch": 1.176,
      "grad_norm": 8.904568672180176,
      "learning_rate": 4.995326334993508e-06,
      "logits/chosen": 0.7332701086997986,
      "logits/rejected": 0.7367414832115173,
      "logps/chosen": -215.7530975341797,
      "logps/rejected": -235.0543670654297,
      "loss": 0.6247,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.6151739954948425,
      "rewards/margins": 0.4263072907924652,
      "rewards/rejected": -1.041481375694275,
      "step": 1470
    },
    {
      "epoch": 1.184,
      "grad_norm": 4.632490158081055,
      "learning_rate": 4.994889920334939e-06,
      "logits/chosen": 0.8015623092651367,
      "logits/rejected": 0.6672269701957703,
      "logps/chosen": -206.82408142089844,
      "logps/rejected": -212.19155883789062,
      "loss": 0.5282,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.25984275341033936,
      "rewards/margins": 0.636847734451294,
      "rewards/rejected": -0.8966904878616333,
      "step": 1480
    },
    {
      "epoch": 1.192,
      "grad_norm": 5.93597412109375,
      "learning_rate": 4.994434050023601e-06,
      "logits/chosen": 0.8352071642875671,
      "logits/rejected": 0.796085000038147,
      "logps/chosen": -200.97604370117188,
      "logps/rejected": -198.58670043945312,
      "loss": 0.6777,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.5017086267471313,
      "rewards/margins": 0.23309734463691711,
      "rewards/rejected": -0.7348060011863708,
      "step": 1490
    },
    {
      "epoch": 1.2,
      "grad_norm": 6.238118648529053,
      "learning_rate": 4.993958727614462e-06,
      "logits/chosen": 0.7685348391532898,
      "logits/rejected": 0.6558945775032043,
      "logps/chosen": -204.9226531982422,
      "logps/rejected": -204.36001586914062,
      "loss": 0.591,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.33830520510673523,
      "rewards/margins": 0.46260887384414673,
      "rewards/rejected": -0.8009141087532043,
      "step": 1500
    },
    {
      "epoch": 1.208,
      "grad_norm": 7.67111873626709,
      "learning_rate": 4.993463956814181e-06,
      "logits/chosen": 0.8750897645950317,
      "logits/rejected": 0.9166715741157532,
      "logps/chosen": -189.0762176513672,
      "logps/rejected": -201.63124084472656,
      "loss": 0.6032,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.2800356447696686,
      "rewards/margins": 0.3452514708042145,
      "rewards/rejected": -0.6252870559692383,
      "step": 1510
    },
    {
      "epoch": 1.216,
      "grad_norm": 4.525489330291748,
      "learning_rate": 4.99294974148108e-06,
      "logits/chosen": 0.8436128497123718,
      "logits/rejected": 0.8338537216186523,
      "logps/chosen": -201.09213256835938,
      "logps/rejected": -220.0704803466797,
      "loss": 0.5747,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.21393656730651855,
      "rewards/margins": 0.4898119568824768,
      "rewards/rejected": -0.7037484645843506,
      "step": 1520
    },
    {
      "epoch": 1.224,
      "grad_norm": 7.136298656463623,
      "learning_rate": 4.992416085625115e-06,
      "logits/chosen": 0.9371658563613892,
      "logits/rejected": 0.9421829581260681,
      "logps/chosen": -187.67381286621094,
      "logps/rejected": -210.39036560058594,
      "loss": 0.6005,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.47332143783569336,
      "rewards/margins": 0.3808668553829193,
      "rewards/rejected": -0.8541883826255798,
      "step": 1530
    },
    {
      "epoch": 1.232,
      "grad_norm": 5.2598700523376465,
      "learning_rate": 4.991862993407841e-06,
      "logits/chosen": 0.9011389017105103,
      "logits/rejected": 0.8639345169067383,
      "logps/chosen": -204.3238525390625,
      "logps/rejected": -222.8467254638672,
      "loss": 0.546,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.5745847821235657,
      "rewards/margins": 0.592379629611969,
      "rewards/rejected": -1.1669644117355347,
      "step": 1540
    },
    {
      "epoch": 1.24,
      "grad_norm": 6.515147686004639,
      "learning_rate": 4.99129046914238e-06,
      "logits/chosen": 0.855043888092041,
      "logits/rejected": 0.895084798336029,
      "logps/chosen": -178.9577178955078,
      "logps/rejected": -202.40538024902344,
      "loss": 0.525,
      "rewards/accuracies": 0.7125000357627869,
      "rewards/chosen": -0.3232458531856537,
      "rewards/margins": 0.7021111845970154,
      "rewards/rejected": -1.0253571271896362,
      "step": 1550
    },
    {
      "epoch": 1.248,
      "grad_norm": 4.3140177726745605,
      "learning_rate": 4.990698517293394e-06,
      "logits/chosen": 0.7897105813026428,
      "logits/rejected": 0.7928025126457214,
      "logps/chosen": -209.9716339111328,
      "logps/rejected": -214.2898712158203,
      "loss": 0.5895,
      "rewards/accuracies": 0.7125000357627869,
      "rewards/chosen": -0.16820655763149261,
      "rewards/margins": 0.539027988910675,
      "rewards/rejected": -0.7072345614433289,
      "step": 1560
    },
    {
      "epoch": 1.256,
      "grad_norm": 7.281121730804443,
      "learning_rate": 4.990087142477042e-06,
      "logits/chosen": 0.797493577003479,
      "logits/rejected": 0.841319739818573,
      "logps/chosen": -211.1555633544922,
      "logps/rejected": -206.3491668701172,
      "loss": 0.5963,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.420600026845932,
      "rewards/margins": 0.395569771528244,
      "rewards/rejected": -0.8161698579788208,
      "step": 1570
    },
    {
      "epoch": 1.264,
      "grad_norm": 6.5268683433532715,
      "learning_rate": 4.989456349460946e-06,
      "logits/chosen": 0.8825798034667969,
      "logits/rejected": 0.7889868021011353,
      "logps/chosen": -214.40603637695312,
      "logps/rejected": -223.7728729248047,
      "loss": 0.556,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.4342477023601532,
      "rewards/margins": 0.5031339526176453,
      "rewards/rejected": -0.9373816847801208,
      "step": 1580
    },
    {
      "epoch": 1.272,
      "grad_norm": 5.648573398590088,
      "learning_rate": 4.988806143164159e-06,
      "logits/chosen": 0.8476539850234985,
      "logits/rejected": 0.7688016891479492,
      "logps/chosen": -213.4077911376953,
      "logps/rejected": -233.77456665039062,
      "loss": 0.5371,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7733410000801086,
      "rewards/margins": 0.6128751039505005,
      "rewards/rejected": -1.3862160444259644,
      "step": 1590
    },
    {
      "epoch": 1.28,
      "grad_norm": 5.341320037841797,
      "learning_rate": 4.988136528657118e-06,
      "logits/chosen": 0.8851507306098938,
      "logits/rejected": 0.9027878046035767,
      "logps/chosen": -213.1574249267578,
      "logps/rejected": -234.6796112060547,
      "loss": 0.5464,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.7365319132804871,
      "rewards/margins": 0.615622878074646,
      "rewards/rejected": -1.3521548509597778,
      "step": 1600
    },
    {
      "epoch": 1.288,
      "grad_norm": 8.493136405944824,
      "learning_rate": 4.987447511161613e-06,
      "logits/chosen": 0.8196334838867188,
      "logits/rejected": 0.833379864692688,
      "logps/chosen": -220.86338806152344,
      "logps/rejected": -230.03433227539062,
      "loss": 0.656,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.6297645568847656,
      "rewards/margins": 0.4346865713596344,
      "rewards/rejected": -1.0644512176513672,
      "step": 1610
    },
    {
      "epoch": 1.296,
      "grad_norm": 7.547203540802002,
      "learning_rate": 4.98673909605074e-06,
      "logits/chosen": 0.8613888025283813,
      "logits/rejected": 0.8237080574035645,
      "logps/chosen": -210.69900512695312,
      "logps/rejected": -225.7193145751953,
      "loss": 0.5646,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.6549606323242188,
      "rewards/margins": 0.5880576968193054,
      "rewards/rejected": -1.243018388748169,
      "step": 1620
    },
    {
      "epoch": 1.304,
      "grad_norm": 4.983480453491211,
      "learning_rate": 4.986011288848863e-06,
      "logits/chosen": 0.9473403096199036,
      "logits/rejected": 0.8395648002624512,
      "logps/chosen": -205.3673858642578,
      "logps/rejected": -201.21458435058594,
      "loss": 0.5895,
      "rewards/accuracies": 0.6500000357627869,
      "rewards/chosen": -0.6703227758407593,
      "rewards/margins": 0.437595933675766,
      "rewards/rejected": -1.1079187393188477,
      "step": 1630
    },
    {
      "epoch": 1.312,
      "grad_norm": 4.282800674438477,
      "learning_rate": 4.985264095231568e-06,
      "logits/chosen": 1.042983889579773,
      "logits/rejected": 0.9440056085586548,
      "logps/chosen": -218.52037048339844,
      "logps/rejected": -235.33702087402344,
      "loss": 0.5297,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7287598848342896,
      "rewards/margins": 0.6097584366798401,
      "rewards/rejected": -1.3385183811187744,
      "step": 1640
    },
    {
      "epoch": 1.32,
      "grad_norm": 5.741344928741455,
      "learning_rate": 4.984497521025622e-06,
      "logits/chosen": 0.9067613482475281,
      "logits/rejected": 0.907869279384613,
      "logps/chosen": -214.6510772705078,
      "logps/rejected": -233.52578735351562,
      "loss": 0.6452,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.5831167101860046,
      "rewards/margins": 0.4216102659702301,
      "rewards/rejected": -1.0047270059585571,
      "step": 1650
    },
    {
      "epoch": 1.328,
      "grad_norm": 5.609917640686035,
      "learning_rate": 4.9837115722089235e-06,
      "logits/chosen": 1.0167735815048218,
      "logits/rejected": 1.0690789222717285,
      "logps/chosen": -198.3705291748047,
      "logps/rejected": -214.2977752685547,
      "loss": 0.591,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.6117779612541199,
      "rewards/margins": 0.41639161109924316,
      "rewards/rejected": -1.0281696319580078,
      "step": 1660
    },
    {
      "epoch": 1.336,
      "grad_norm": 5.828847408294678,
      "learning_rate": 4.982906254910459e-06,
      "logits/chosen": 0.9671093821525574,
      "logits/rejected": 0.934682309627533,
      "logps/chosen": -224.7041473388672,
      "logps/rejected": -225.40408325195312,
      "loss": 0.6122,
      "rewards/accuracies": 0.7125000357627869,
      "rewards/chosen": -0.4490407407283783,
      "rewards/margins": 0.4267473816871643,
      "rewards/rejected": -0.875788152217865,
      "step": 1670
    },
    {
      "epoch": 1.3439999999999999,
      "grad_norm": 4.787741184234619,
      "learning_rate": 4.982081575410256e-06,
      "logits/chosen": 0.9712103009223938,
      "logits/rejected": 1.061010718345642,
      "logps/chosen": -199.6064910888672,
      "logps/rejected": -225.61083984375,
      "loss": 0.5951,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.35054826736450195,
      "rewards/margins": 0.4877324104309082,
      "rewards/rejected": -0.8382806777954102,
      "step": 1680
    },
    {
      "epoch": 1.3519999999999999,
      "grad_norm": 5.184296607971191,
      "learning_rate": 4.981237540139331e-06,
      "logits/chosen": 0.9483404159545898,
      "logits/rejected": 0.9563024640083313,
      "logps/chosen": -181.48716735839844,
      "logps/rejected": -208.47232055664062,
      "loss": 0.5151,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.23454923927783966,
      "rewards/margins": 0.6790667176246643,
      "rewards/rejected": -0.913615882396698,
      "step": 1690
    },
    {
      "epoch": 1.3599999999999999,
      "grad_norm": 5.253026485443115,
      "learning_rate": 4.980374155679639e-06,
      "logits/chosen": 1.0046643018722534,
      "logits/rejected": 0.9964167475700378,
      "logps/chosen": -196.05494689941406,
      "logps/rejected": -208.57164001464844,
      "loss": 0.5772,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.5216605067253113,
      "rewards/margins": 0.5054008960723877,
      "rewards/rejected": -1.0270613431930542,
      "step": 1700
    },
    {
      "epoch": 1.3679999999999999,
      "grad_norm": 4.707123279571533,
      "learning_rate": 4.9794914287640264e-06,
      "logits/chosen": 0.9587984085083008,
      "logits/rejected": 0.9082703590393066,
      "logps/chosen": -191.87796020507812,
      "logps/rejected": -229.51358032226562,
      "loss": 0.5563,
      "rewards/accuracies": 0.6500000357627869,
      "rewards/chosen": -0.3796873688697815,
      "rewards/margins": 0.5814693570137024,
      "rewards/rejected": -0.9611567854881287,
      "step": 1710
    },
    {
      "epoch": 1.376,
      "grad_norm": 6.676301002502441,
      "learning_rate": 4.978589366276174e-06,
      "logits/chosen": 0.8662906885147095,
      "logits/rejected": 0.7710773348808289,
      "logps/chosen": -192.7714080810547,
      "logps/rejected": -228.8347625732422,
      "loss": 0.55,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.7520737051963806,
      "rewards/margins": 0.6919212341308594,
      "rewards/rejected": -1.4439948797225952,
      "step": 1720
    },
    {
      "epoch": 1.384,
      "grad_norm": 6.470781326293945,
      "learning_rate": 4.977667975250548e-06,
      "logits/chosen": 0.8674607276916504,
      "logits/rejected": 0.8369401097297668,
      "logps/chosen": -204.9677276611328,
      "logps/rejected": -220.662353515625,
      "loss": 0.6208,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.6717852354049683,
      "rewards/margins": 0.5695573687553406,
      "rewards/rejected": -1.241342544555664,
      "step": 1730
    },
    {
      "epoch": 1.392,
      "grad_norm": 7.195312023162842,
      "learning_rate": 4.97672726287234e-06,
      "logits/chosen": 0.9234275817871094,
      "logits/rejected": 0.9190540313720703,
      "logps/chosen": -210.9809112548828,
      "logps/rejected": -206.3329620361328,
      "loss": 0.6337,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.394771546125412,
      "rewards/margins": 0.37511372566223145,
      "rewards/rejected": -0.7698853015899658,
      "step": 1740
    },
    {
      "epoch": 1.4,
      "grad_norm": 6.862684726715088,
      "learning_rate": 4.975767236477413e-06,
      "logits/chosen": 1.0124229192733765,
      "logits/rejected": 1.0320953130722046,
      "logps/chosen": -182.6747589111328,
      "logps/rejected": -206.9286346435547,
      "loss": 0.5748,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.14587950706481934,
      "rewards/margins": 0.5520176887512207,
      "rewards/rejected": -0.6978972554206848,
      "step": 1750
    },
    {
      "epoch": 1.408,
      "grad_norm": 5.418376922607422,
      "learning_rate": 4.974787903552247e-06,
      "logits/chosen": 0.8783740997314453,
      "logits/rejected": 0.7816500067710876,
      "logps/chosen": -213.95645141601562,
      "logps/rejected": -225.9214324951172,
      "loss": 0.6632,
      "rewards/accuracies": 0.6500000357627869,
      "rewards/chosen": -0.7024328708648682,
      "rewards/margins": 0.48432379961013794,
      "rewards/rejected": -1.1867567300796509,
      "step": 1760
    },
    {
      "epoch": 1.416,
      "grad_norm": 5.543606758117676,
      "learning_rate": 4.973789271733877e-06,
      "logits/chosen": 0.8639112710952759,
      "logits/rejected": 0.8347362875938416,
      "logps/chosen": -208.14321899414062,
      "logps/rejected": -232.296875,
      "loss": 0.5753,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.4658367335796356,
      "rewards/margins": 0.6865634322166443,
      "rewards/rejected": -1.152400255203247,
      "step": 1770
    },
    {
      "epoch": 1.424,
      "grad_norm": 5.288059711456299,
      "learning_rate": 4.972771348809834e-06,
      "logits/chosen": 0.7869605422019958,
      "logits/rejected": 0.7716497778892517,
      "logps/chosen": -203.56529235839844,
      "logps/rejected": -216.66299438476562,
      "loss": 0.6529,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.5823510885238647,
      "rewards/margins": 0.38376662135124207,
      "rewards/rejected": -0.9661176800727844,
      "step": 1780
    },
    {
      "epoch": 1.432,
      "grad_norm": 7.8693037033081055,
      "learning_rate": 4.9717341427180855e-06,
      "logits/chosen": 0.8742721676826477,
      "logits/rejected": 0.8222671747207642,
      "logps/chosen": -223.1770477294922,
      "logps/rejected": -230.52334594726562,
      "loss": 0.5733,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.4773767590522766,
      "rewards/margins": 0.528388261795044,
      "rewards/rejected": -1.0057649612426758,
      "step": 1790
    },
    {
      "epoch": 1.44,
      "grad_norm": 5.248528480529785,
      "learning_rate": 4.970677661546972e-06,
      "logits/chosen": 0.7629317045211792,
      "logits/rejected": 0.8210358023643494,
      "logps/chosen": -188.845703125,
      "logps/rejected": -227.7703857421875,
      "loss": 0.5079,
      "rewards/accuracies": 0.7125000357627869,
      "rewards/chosen": -0.4410068690776825,
      "rewards/margins": 0.8114088177680969,
      "rewards/rejected": -1.252415657043457,
      "step": 1800
    },
    {
      "epoch": 1.448,
      "grad_norm": 6.2042365074157715,
      "learning_rate": 4.969601913535148e-06,
      "logits/chosen": 0.8677163124084473,
      "logits/rejected": 0.8122463226318359,
      "logps/chosen": -198.2725067138672,
      "logps/rejected": -191.28347778320312,
      "loss": 0.616,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5268740057945251,
      "rewards/margins": 0.3900458514690399,
      "rewards/rejected": -0.9169198274612427,
      "step": 1810
    },
    {
      "epoch": 1.456,
      "grad_norm": 3.95033597946167,
      "learning_rate": 4.9685069070715105e-06,
      "logits/chosen": 0.8852386474609375,
      "logits/rejected": 0.8015106320381165,
      "logps/chosen": -213.7109832763672,
      "logps/rejected": -232.56704711914062,
      "loss": 0.538,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.778232753276825,
      "rewards/margins": 0.8279777765274048,
      "rewards/rejected": -1.6062105894088745,
      "step": 1820
    },
    {
      "epoch": 1.464,
      "grad_norm": 7.496504783630371,
      "learning_rate": 4.967392650695141e-06,
      "logits/chosen": 0.8542248010635376,
      "logits/rejected": 0.8511247634887695,
      "logps/chosen": -208.1324005126953,
      "logps/rejected": -228.3732452392578,
      "loss": 0.6527,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.2442353963851929,
      "rewards/margins": 0.4487527906894684,
      "rewards/rejected": -1.6929882764816284,
      "step": 1830
    },
    {
      "epoch": 1.472,
      "grad_norm": 11.455583572387695,
      "learning_rate": 4.966259153095235e-06,
      "logits/chosen": 0.9046268463134766,
      "logits/rejected": 0.8542624711990356,
      "logps/chosen": -218.9237823486328,
      "logps/rejected": -226.2521209716797,
      "loss": 0.5853,
      "rewards/accuracies": 0.6500000357627869,
      "rewards/chosen": -1.0356435775756836,
      "rewards/margins": 0.6369892954826355,
      "rewards/rejected": -1.6726330518722534,
      "step": 1840
    },
    {
      "epoch": 1.48,
      "grad_norm": 4.398818016052246,
      "learning_rate": 4.965106423111033e-06,
      "logits/chosen": 0.8832312822341919,
      "logits/rejected": 0.9116541743278503,
      "logps/chosen": -205.5150909423828,
      "logps/rejected": -219.7370147705078,
      "loss": 0.64,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.2404916286468506,
      "rewards/margins": 0.47511014342308044,
      "rewards/rejected": -1.7156018018722534,
      "step": 1850
    },
    {
      "epoch": 1.488,
      "grad_norm": 7.927629470825195,
      "learning_rate": 4.963934469731756e-06,
      "logits/chosen": 0.7935855984687805,
      "logits/rejected": 0.8751907348632812,
      "logps/chosen": -198.9912567138672,
      "logps/rejected": -239.27891540527344,
      "loss": 0.5558,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.2322111129760742,
      "rewards/margins": 0.5577873587608337,
      "rewards/rejected": -1.7899982929229736,
      "step": 1860
    },
    {
      "epoch": 1.496,
      "grad_norm": 8.568868637084961,
      "learning_rate": 4.962743302096532e-06,
      "logits/chosen": 0.8752376437187195,
      "logits/rejected": 0.9415783286094666,
      "logps/chosen": -202.5786895751953,
      "logps/rejected": -227.0806121826172,
      "loss": 0.6257,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.2775661945343018,
      "rewards/margins": 0.5068507790565491,
      "rewards/rejected": -1.7844170331954956,
      "step": 1870
    },
    {
      "epoch": 1.504,
      "grad_norm": 7.778118133544922,
      "learning_rate": 4.961532929494325e-06,
      "logits/chosen": 0.8835949301719666,
      "logits/rejected": 0.9183811545372009,
      "logps/chosen": -220.37466430664062,
      "logps/rejected": -255.732421875,
      "loss": 0.5707,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.3042207956314087,
      "rewards/margins": 0.6345095634460449,
      "rewards/rejected": -1.938730239868164,
      "step": 1880
    },
    {
      "epoch": 1.512,
      "grad_norm": 8.768423080444336,
      "learning_rate": 4.960303361363863e-06,
      "logits/chosen": 0.9741565585136414,
      "logits/rejected": 0.9229215979576111,
      "logps/chosen": -193.06396484375,
      "logps/rejected": -212.60093688964844,
      "loss": 0.5493,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.825711190700531,
      "rewards/margins": 0.6793455481529236,
      "rewards/rejected": -1.5050567388534546,
      "step": 1890
    },
    {
      "epoch": 1.52,
      "grad_norm": 8.654824256896973,
      "learning_rate": 4.959054607293567e-06,
      "logits/chosen": 0.8612321019172668,
      "logits/rejected": 0.8589774966239929,
      "logps/chosen": -211.7546844482422,
      "logps/rejected": -245.64761352539062,
      "loss": 0.5589,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.3008898496627808,
      "rewards/margins": 0.6522501111030579,
      "rewards/rejected": -1.9531399011611938,
      "step": 1900
    },
    {
      "epoch": 1.528,
      "grad_norm": 8.161283493041992,
      "learning_rate": 4.9577866770214715e-06,
      "logits/chosen": 0.9014043807983398,
      "logits/rejected": 0.972660481929779,
      "logps/chosen": -225.20498657226562,
      "logps/rejected": -247.1440887451172,
      "loss": 0.5944,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.402032732963562,
      "rewards/margins": 0.5216811299324036,
      "rewards/rejected": -1.9237139225006104,
      "step": 1910
    },
    {
      "epoch": 1.536,
      "grad_norm": 5.097872257232666,
      "learning_rate": 4.95649958043515e-06,
      "logits/chosen": 1.0036065578460693,
      "logits/rejected": 0.9283719062805176,
      "logps/chosen": -189.35348510742188,
      "logps/rejected": -208.98095703125,
      "loss": 0.6005,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.9046257138252258,
      "rewards/margins": 0.6082975268363953,
      "rewards/rejected": -1.5129234790802002,
      "step": 1920
    },
    {
      "epoch": 1.544,
      "grad_norm": 6.37450647354126,
      "learning_rate": 4.955193327571643e-06,
      "logits/chosen": 0.9055408835411072,
      "logits/rejected": 0.8177289366722107,
      "logps/chosen": -201.5961151123047,
      "logps/rejected": -227.8748016357422,
      "loss": 0.5858,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.2569955587387085,
      "rewards/margins": 0.5454604029655457,
      "rewards/rejected": -1.8024559020996094,
      "step": 1930
    },
    {
      "epoch": 1.552,
      "grad_norm": 7.622455596923828,
      "learning_rate": 4.95386792861737e-06,
      "logits/chosen": 0.917728841304779,
      "logits/rejected": 0.9097478985786438,
      "logps/chosen": -226.21763610839844,
      "logps/rejected": -250.62930297851562,
      "loss": 0.6142,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.4412829875946045,
      "rewards/margins": 0.5837629437446594,
      "rewards/rejected": -2.025045871734619,
      "step": 1940
    },
    {
      "epoch": 1.56,
      "grad_norm": 7.772380828857422,
      "learning_rate": 4.952523393908059e-06,
      "logits/chosen": 0.9898624420166016,
      "logits/rejected": 0.9073271155357361,
      "logps/chosen": -230.84971618652344,
      "logps/rejected": -236.9692840576172,
      "loss": 0.6086,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.1479434967041016,
      "rewards/margins": 0.5934609770774841,
      "rewards/rejected": -1.7414045333862305,
      "step": 1950
    },
    {
      "epoch": 1.568,
      "grad_norm": 4.825398921966553,
      "learning_rate": 4.951159733928663e-06,
      "logits/chosen": 0.9667159914970398,
      "logits/rejected": 0.9257230162620544,
      "logps/chosen": -201.62188720703125,
      "logps/rejected": -218.0967254638672,
      "loss": 0.565,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.7356204986572266,
      "rewards/margins": 0.5399088263511658,
      "rewards/rejected": -1.2755292654037476,
      "step": 1960
    },
    {
      "epoch": 1.576,
      "grad_norm": 5.7298760414123535,
      "learning_rate": 4.949776959313275e-06,
      "logits/chosen": 1.0072981119155884,
      "logits/rejected": 1.023651123046875,
      "logps/chosen": -182.70799255371094,
      "logps/rejected": -193.568359375,
      "loss": 0.5054,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.4652971923351288,
      "rewards/margins": 0.755893349647522,
      "rewards/rejected": -1.2211906909942627,
      "step": 1970
    },
    {
      "epoch": 1.584,
      "grad_norm": 5.249616622924805,
      "learning_rate": 4.94837508084505e-06,
      "logits/chosen": 1.0406349897384644,
      "logits/rejected": 1.0460046529769897,
      "logps/chosen": -207.79454040527344,
      "logps/rejected": -222.9483642578125,
      "loss": 0.5672,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.7943063974380493,
      "rewards/margins": 0.5170424580574036,
      "rewards/rejected": -1.311348795890808,
      "step": 1980
    },
    {
      "epoch": 1.592,
      "grad_norm": 5.766063213348389,
      "learning_rate": 4.9469541094561185e-06,
      "logits/chosen": 1.1073545217514038,
      "logits/rejected": 1.0138510465621948,
      "logps/chosen": -207.35740661621094,
      "logps/rejected": -210.7662353515625,
      "loss": 0.5451,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.6396881937980652,
      "rewards/margins": 0.5360140204429626,
      "rewards/rejected": -1.1757020950317383,
      "step": 1990
    },
    {
      "epoch": 1.6,
      "grad_norm": 4.6534624099731445,
      "learning_rate": 4.945514056227499e-06,
      "logits/chosen": 1.0667814016342163,
      "logits/rejected": 1.0090528726577759,
      "logps/chosen": -190.9570770263672,
      "logps/rejected": -224.06448364257812,
      "loss": 0.5949,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.903643786907196,
      "rewards/margins": 0.4600391387939453,
      "rewards/rejected": -1.3636829853057861,
      "step": 2000
    },
    {
      "epoch": 1.608,
      "grad_norm": 6.801830291748047,
      "learning_rate": 4.944054932389018e-06,
      "logits/chosen": 0.9834865927696228,
      "logits/rejected": 0.9897794723510742,
      "logps/chosen": -197.6669158935547,
      "logps/rejected": -222.3367156982422,
      "loss": 0.7015,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.5120326280593872,
      "rewards/margins": 0.37569552659988403,
      "rewards/rejected": -0.8877281546592712,
      "step": 2010
    },
    {
      "epoch": 1.616,
      "grad_norm": 4.7985663414001465,
      "learning_rate": 4.942576749319215e-06,
      "logits/chosen": 0.9592499136924744,
      "logits/rejected": 0.9662960171699524,
      "logps/chosen": -208.7536163330078,
      "logps/rejected": -221.6714324951172,
      "loss": 0.6111,
      "rewards/accuracies": 0.6500000357627869,
      "rewards/chosen": -0.6202236413955688,
      "rewards/margins": 0.34536442160606384,
      "rewards/rejected": -0.9655880331993103,
      "step": 2020
    },
    {
      "epoch": 1.624,
      "grad_norm": 6.010429382324219,
      "learning_rate": 4.9410795185452584e-06,
      "logits/chosen": 0.9691095352172852,
      "logits/rejected": 0.970175564289093,
      "logps/chosen": -202.74681091308594,
      "logps/rejected": -219.4949951171875,
      "loss": 0.6405,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.6648755669593811,
      "rewards/margins": 0.40488871932029724,
      "rewards/rejected": -1.069764256477356,
      "step": 2030
    },
    {
      "epoch": 1.6320000000000001,
      "grad_norm": 3.876993179321289,
      "learning_rate": 4.9395632517428546e-06,
      "logits/chosen": 0.7599179148674011,
      "logits/rejected": 0.8189982771873474,
      "logps/chosen": -203.71568298339844,
      "logps/rejected": -221.14488220214844,
      "loss": 0.5479,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.6682913303375244,
      "rewards/margins": 0.7366959452629089,
      "rewards/rejected": -1.4049873352050781,
      "step": 2040
    },
    {
      "epoch": 1.6400000000000001,
      "grad_norm": 4.428971767425537,
      "learning_rate": 4.938027960736158e-06,
      "logits/chosen": 0.8731341361999512,
      "logits/rejected": 0.9049884080886841,
      "logps/chosen": -195.8953399658203,
      "logps/rejected": -219.8108673095703,
      "loss": 0.572,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.802505612373352,
      "rewards/margins": 0.5797830820083618,
      "rewards/rejected": -1.3822888135910034,
      "step": 2050
    },
    {
      "epoch": 1.6480000000000001,
      "grad_norm": 4.308506011962891,
      "learning_rate": 4.936473657497674e-06,
      "logits/chosen": 0.8433103561401367,
      "logits/rejected": 0.8728559613227844,
      "logps/chosen": -189.4270477294922,
      "logps/rejected": -208.40074157714844,
      "loss": 0.5589,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.5602220892906189,
      "rewards/margins": 0.6132001280784607,
      "rewards/rejected": -1.17342209815979,
      "step": 2060
    },
    {
      "epoch": 1.6560000000000001,
      "grad_norm": 5.429026126861572,
      "learning_rate": 4.934900354148173e-06,
      "logits/chosen": 0.8006834387779236,
      "logits/rejected": 0.8655485510826111,
      "logps/chosen": -196.53811645507812,
      "logps/rejected": -226.0628662109375,
      "loss": 0.5333,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.5719932913780212,
      "rewards/margins": 0.6289103627204895,
      "rewards/rejected": -1.2009037733078003,
      "step": 2070
    },
    {
      "epoch": 1.6640000000000001,
      "grad_norm": 7.226114749908447,
      "learning_rate": 4.933308062956591e-06,
      "logits/chosen": 0.9665275812149048,
      "logits/rejected": 0.9210414886474609,
      "logps/chosen": -216.1051483154297,
      "logps/rejected": -238.590576171875,
      "loss": 0.6039,
      "rewards/accuracies": 0.6500000357627869,
      "rewards/chosen": -1.081291913986206,
      "rewards/margins": 0.5662848353385925,
      "rewards/rejected": -1.6475766897201538,
      "step": 2080
    },
    {
      "epoch": 1.6720000000000002,
      "grad_norm": 5.951044082641602,
      "learning_rate": 4.931696796339933e-06,
      "logits/chosen": 0.8365780711174011,
      "logits/rejected": 0.9172168970108032,
      "logps/chosen": -222.9845733642578,
      "logps/rejected": -241.9564666748047,
      "loss": 0.5248,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6349353790283203,
      "rewards/margins": 0.626502513885498,
      "rewards/rejected": -1.2614378929138184,
      "step": 2090
    },
    {
      "epoch": 1.6800000000000002,
      "grad_norm": 5.7516279220581055,
      "learning_rate": 4.930066566863182e-06,
      "logits/chosen": 0.8466065526008606,
      "logits/rejected": 0.8476532101631165,
      "logps/chosen": -221.70811462402344,
      "logps/rejected": -249.4915313720703,
      "loss": 0.5935,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.7990935444831848,
      "rewards/margins": 0.5039268732070923,
      "rewards/rejected": -1.3030204772949219,
      "step": 2100
    },
    {
      "epoch": 1.688,
      "grad_norm": 4.736870288848877,
      "learning_rate": 4.9284173872391925e-06,
      "logits/chosen": 0.9319866299629211,
      "logits/rejected": 0.7861051559448242,
      "logps/chosen": -211.2753143310547,
      "logps/rejected": -213.59410095214844,
      "loss": 0.513,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.5237230658531189,
      "rewards/margins": 0.7074744701385498,
      "rewards/rejected": -1.231197476387024,
      "step": 2110
    },
    {
      "epoch": 1.696,
      "grad_norm": 6.545098304748535,
      "learning_rate": 4.9267492703286005e-06,
      "logits/chosen": 0.8851673007011414,
      "logits/rejected": 0.9201962351799011,
      "logps/chosen": -205.5138702392578,
      "logps/rejected": -210.7584991455078,
      "loss": 0.6149,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.6310408115386963,
      "rewards/margins": 0.48075610399246216,
      "rewards/rejected": -1.1117968559265137,
      "step": 2120
    },
    {
      "epoch": 1.704,
      "grad_norm": 5.556862831115723,
      "learning_rate": 4.9250622291397144e-06,
      "logits/chosen": 0.8860400319099426,
      "logits/rejected": 0.9521721005439758,
      "logps/chosen": -203.97494506835938,
      "logps/rejected": -223.2489776611328,
      "loss": 0.624,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.8290230631828308,
      "rewards/margins": 0.4605173170566559,
      "rewards/rejected": -1.2895405292510986,
      "step": 2130
    },
    {
      "epoch": 1.712,
      "grad_norm": 4.479294776916504,
      "learning_rate": 4.923356276828422e-06,
      "logits/chosen": 0.9692792892456055,
      "logits/rejected": 0.9220939874649048,
      "logps/chosen": -206.8193817138672,
      "logps/rejected": -212.7848663330078,
      "loss": 0.5211,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.5379689335823059,
      "rewards/margins": 0.6115099787712097,
      "rewards/rejected": -1.149478793144226,
      "step": 2140
    },
    {
      "epoch": 1.72,
      "grad_norm": 4.822434902191162,
      "learning_rate": 4.921631426698082e-06,
      "logits/chosen": 0.9812706112861633,
      "logits/rejected": 0.8581036925315857,
      "logps/chosen": -204.3318328857422,
      "logps/rejected": -208.5980224609375,
      "loss": 0.6165,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.41802969574928284,
      "rewards/margins": 0.469400554895401,
      "rewards/rejected": -0.8874303102493286,
      "step": 2150
    },
    {
      "epoch": 1.728,
      "grad_norm": 7.955755710601807,
      "learning_rate": 4.919887692199423e-06,
      "logits/chosen": 1.019281268119812,
      "logits/rejected": 0.985063374042511,
      "logps/chosen": -196.29539489746094,
      "logps/rejected": -185.47842407226562,
      "loss": 0.5773,
      "rewards/accuracies": 0.7125000357627869,
      "rewards/chosen": -0.2744506895542145,
      "rewards/margins": 0.5127472877502441,
      "rewards/rejected": -0.787198007106781,
      "step": 2160
    },
    {
      "epoch": 1.736,
      "grad_norm": 3.87949275970459,
      "learning_rate": 4.918125086930435e-06,
      "logits/chosen": 0.8921056985855103,
      "logits/rejected": 0.8641147613525391,
      "logps/chosen": -208.4770965576172,
      "logps/rejected": -232.86192321777344,
      "loss": 0.5582,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.398175448179245,
      "rewards/margins": 0.586793839931488,
      "rewards/rejected": -0.9849693179130554,
      "step": 2170
    },
    {
      "epoch": 1.744,
      "grad_norm": 5.009764671325684,
      "learning_rate": 4.91634362463627e-06,
      "logits/chosen": 0.8331074118614197,
      "logits/rejected": 0.8069505095481873,
      "logps/chosen": -194.30528259277344,
      "logps/rejected": -228.6878204345703,
      "loss": 0.5577,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.46142521500587463,
      "rewards/margins": 0.6860167384147644,
      "rewards/rejected": -1.1474418640136719,
      "step": 2180
    },
    {
      "epoch": 1.752,
      "grad_norm": 6.22864294052124,
      "learning_rate": 4.914543319209126e-06,
      "logits/chosen": 0.9490287899971008,
      "logits/rejected": 0.7996692061424255,
      "logps/chosen": -213.89102172851562,
      "logps/rejected": -235.63926696777344,
      "loss": 0.6154,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6507563591003418,
      "rewards/margins": 0.5299718976020813,
      "rewards/rejected": -1.1807281970977783,
      "step": 2190
    },
    {
      "epoch": 1.76,
      "grad_norm": 6.673972129821777,
      "learning_rate": 4.912724184688149e-06,
      "logits/chosen": 0.7972329258918762,
      "logits/rejected": 0.7832920551300049,
      "logps/chosen": -216.7205810546875,
      "logps/rejected": -232.1858367919922,
      "loss": 0.6573,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7054974436759949,
      "rewards/margins": 0.4817284643650055,
      "rewards/rejected": -1.1872259378433228,
      "step": 2200
    },
    {
      "epoch": 1.768,
      "grad_norm": 4.410974979400635,
      "learning_rate": 4.910886235259315e-06,
      "logits/chosen": 0.8468648791313171,
      "logits/rejected": 0.8238309025764465,
      "logps/chosen": -206.13621520996094,
      "logps/rejected": -206.95458984375,
      "loss": 0.5607,
      "rewards/accuracies": 0.7125000357627869,
      "rewards/chosen": -0.4652062952518463,
      "rewards/margins": 0.5647412538528442,
      "rewards/rejected": -1.0299476385116577,
      "step": 2210
    },
    {
      "epoch": 1.776,
      "grad_norm": 5.151928424835205,
      "learning_rate": 4.909029485255321e-06,
      "logits/chosen": 0.883913516998291,
      "logits/rejected": 0.8439237475395203,
      "logps/chosen": -216.8835906982422,
      "logps/rejected": -225.7892303466797,
      "loss": 0.6172,
      "rewards/accuracies": 0.6500000357627869,
      "rewards/chosen": -0.736081063747406,
      "rewards/margins": 0.49011898040771484,
      "rewards/rejected": -1.2262001037597656,
      "step": 2220
    },
    {
      "epoch": 1.784,
      "grad_norm": 4.777335166931152,
      "learning_rate": 4.907153949155479e-06,
      "logits/chosen": 0.8213723301887512,
      "logits/rejected": 0.77827388048172,
      "logps/chosen": -216.30235290527344,
      "logps/rejected": -234.95713806152344,
      "loss": 0.5687,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.8275079131126404,
      "rewards/margins": 0.5740241408348083,
      "rewards/rejected": -1.4015320539474487,
      "step": 2230
    },
    {
      "epoch": 1.792,
      "grad_norm": 5.356624126434326,
      "learning_rate": 4.905259641585594e-06,
      "logits/chosen": 1.0409396886825562,
      "logits/rejected": 0.9675580263137817,
      "logps/chosen": -189.8865966796875,
      "logps/rejected": -210.8790283203125,
      "loss": 0.5454,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.5686191916465759,
      "rewards/margins": 0.596284806728363,
      "rewards/rejected": -1.164903998374939,
      "step": 2240
    },
    {
      "epoch": 1.8,
      "grad_norm": 7.358102798461914,
      "learning_rate": 4.903346577317859e-06,
      "logits/chosen": 0.9212884306907654,
      "logits/rejected": 0.9235758185386658,
      "logps/chosen": -205.32864379882812,
      "logps/rejected": -209.7302703857422,
      "loss": 0.6964,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.7906255125999451,
      "rewards/margins": 0.18452060222625732,
      "rewards/rejected": -0.9751461148262024,
      "step": 2250
    },
    {
      "epoch": 1.808,
      "grad_norm": 6.358880043029785,
      "learning_rate": 4.901414771270732e-06,
      "logits/chosen": 1.0110472440719604,
      "logits/rejected": 0.9830945134162903,
      "logps/chosen": -196.3888397216797,
      "logps/rejected": -196.31834411621094,
      "loss": 0.5517,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.44832611083984375,
      "rewards/margins": 0.5208678841590881,
      "rewards/rejected": -0.9691939353942871,
      "step": 2260
    },
    {
      "epoch": 1.8159999999999998,
      "grad_norm": 4.602325439453125,
      "learning_rate": 4.899464238508826e-06,
      "logits/chosen": 0.9746546149253845,
      "logits/rejected": 0.9642091989517212,
      "logps/chosen": -196.0602569580078,
      "logps/rejected": -214.0269012451172,
      "loss": 0.5631,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.30714574456214905,
      "rewards/margins": 0.5362817049026489,
      "rewards/rejected": -0.8434274792671204,
      "step": 2270
    },
    {
      "epoch": 1.8239999999999998,
      "grad_norm": 4.84414005279541,
      "learning_rate": 4.8974949942427854e-06,
      "logits/chosen": 0.9626126289367676,
      "logits/rejected": 0.9172550439834595,
      "logps/chosen": -193.07933044433594,
      "logps/rejected": -198.24754333496094,
      "loss": 0.625,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.4726344645023346,
      "rewards/margins": 0.4540925920009613,
      "rewards/rejected": -0.9267271161079407,
      "step": 2280
    },
    {
      "epoch": 1.8319999999999999,
      "grad_norm": 8.682156562805176,
      "learning_rate": 4.895507053829174e-06,
      "logits/chosen": 0.8741292953491211,
      "logits/rejected": 0.9165781140327454,
      "logps/chosen": -206.250732421875,
      "logps/rejected": -249.11558532714844,
      "loss": 0.5665,
      "rewards/accuracies": 0.6500000357627869,
      "rewards/chosen": -0.6650144457817078,
      "rewards/margins": 0.7068294286727905,
      "rewards/rejected": -1.3718438148498535,
      "step": 2290
    },
    {
      "epoch": 1.8399999999999999,
      "grad_norm": 5.111639499664307,
      "learning_rate": 4.893500432770349e-06,
      "logits/chosen": 0.9301668405532837,
      "logits/rejected": 0.9247567057609558,
      "logps/chosen": -206.8374786376953,
      "logps/rejected": -210.31265258789062,
      "loss": 0.5961,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.7106897234916687,
      "rewards/margins": 0.40209847688674927,
      "rewards/rejected": -1.112788200378418,
      "step": 2300
    },
    {
      "epoch": 1.8479999999999999,
      "grad_norm": 5.505365371704102,
      "learning_rate": 4.891475146714348e-06,
      "logits/chosen": 0.9037653207778931,
      "logits/rejected": 0.8538966178894043,
      "logps/chosen": -199.22640991210938,
      "logps/rejected": -212.10537719726562,
      "loss": 0.5586,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.9016987085342407,
      "rewards/margins": 0.6126433610916138,
      "rewards/rejected": -1.514341950416565,
      "step": 2310
    },
    {
      "epoch": 1.8559999999999999,
      "grad_norm": 4.674638748168945,
      "learning_rate": 4.889431211454753e-06,
      "logits/chosen": 0.8724443316459656,
      "logits/rejected": 0.9939247965812683,
      "logps/chosen": -213.60525512695312,
      "logps/rejected": -227.02359008789062,
      "loss": 0.5838,
      "rewards/accuracies": 0.7125000357627869,
      "rewards/chosen": -1.1054712533950806,
      "rewards/margins": 0.44871193170547485,
      "rewards/rejected": -1.5541832447052002,
      "step": 2320
    },
    {
      "epoch": 1.8639999999999999,
      "grad_norm": 7.4808220863342285,
      "learning_rate": 4.887368642930588e-06,
      "logits/chosen": 0.9019695520401001,
      "logits/rejected": 0.8304522633552551,
      "logps/chosen": -211.55125427246094,
      "logps/rejected": -225.2754364013672,
      "loss": 0.6556,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.171556830406189,
      "rewards/margins": 0.2245822697877884,
      "rewards/rejected": -1.396139144897461,
      "step": 2330
    },
    {
      "epoch": 1.8719999999999999,
      "grad_norm": 6.583737373352051,
      "learning_rate": 4.8852874572261715e-06,
      "logits/chosen": 0.9607311487197876,
      "logits/rejected": 0.8902657628059387,
      "logps/chosen": -194.19366455078125,
      "logps/rejected": -233.3009796142578,
      "loss": 0.5089,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -0.9743821024894714,
      "rewards/margins": 0.8457622528076172,
      "rewards/rejected": -1.8201442956924438,
      "step": 2340
    },
    {
      "epoch": 1.88,
      "grad_norm": 8.004322052001953,
      "learning_rate": 4.88318767057101e-06,
      "logits/chosen": 0.6777879595756531,
      "logits/rejected": 0.7692098021507263,
      "logps/chosen": -226.11428833007812,
      "logps/rejected": -252.94114685058594,
      "loss": 0.6428,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.4815189838409424,
      "rewards/margins": 0.4003751873970032,
      "rewards/rejected": -1.8818941116333008,
      "step": 2350
    },
    {
      "epoch": 1.888,
      "grad_norm": 5.834934234619141,
      "learning_rate": 4.881069299339662e-06,
      "logits/chosen": 0.9836352467536926,
      "logits/rejected": 0.925758957862854,
      "logps/chosen": -196.276611328125,
      "logps/rejected": -216.8465118408203,
      "loss": 0.5801,
      "rewards/accuracies": 0.7125000357627869,
      "rewards/chosen": -0.6051527261734009,
      "rewards/margins": 0.5349343419075012,
      "rewards/rejected": -1.1400870084762573,
      "step": 2360
    },
    {
      "epoch": 1.896,
      "grad_norm": 3.793811321258545,
      "learning_rate": 4.878932360051611e-06,
      "logits/chosen": 0.9329185485839844,
      "logits/rejected": 0.8791740536689758,
      "logps/chosen": -198.0451202392578,
      "logps/rejected": -204.83177185058594,
      "loss": 0.5089,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -0.7063581943511963,
      "rewards/margins": 0.798491895198822,
      "rewards/rejected": -1.5048500299453735,
      "step": 2370
    },
    {
      "epoch": 1.904,
      "grad_norm": 6.470005512237549,
      "learning_rate": 4.876776869371139e-06,
      "logits/chosen": 0.9098548889160156,
      "logits/rejected": 0.9413707852363586,
      "logps/chosen": -205.4551239013672,
      "logps/rejected": -206.733642578125,
      "loss": 0.543,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.8138889670372009,
      "rewards/margins": 0.5941287279129028,
      "rewards/rejected": -1.4080175161361694,
      "step": 2380
    },
    {
      "epoch": 1.912,
      "grad_norm": 7.808496475219727,
      "learning_rate": 4.874602844107195e-06,
      "logits/chosen": 1.01150643825531,
      "logits/rejected": 0.873560905456543,
      "logps/chosen": -209.4512939453125,
      "logps/rejected": -219.6645965576172,
      "loss": 0.649,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.0077110528945923,
      "rewards/margins": 0.44567227363586426,
      "rewards/rejected": -1.453383445739746,
      "step": 2390
    },
    {
      "epoch": 1.92,
      "grad_norm": 10.799213409423828,
      "learning_rate": 4.872410301213265e-06,
      "logits/chosen": 0.9560382962226868,
      "logits/rejected": 0.9300372004508972,
      "logps/chosen": -196.2581329345703,
      "logps/rejected": -212.28335571289062,
      "loss": 0.582,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.8118780255317688,
      "rewards/margins": 0.4887621998786926,
      "rewards/rejected": -1.3006402254104614,
      "step": 2400
    },
    {
      "epoch": 1.928,
      "grad_norm": 4.009180068969727,
      "learning_rate": 4.87019925778724e-06,
      "logits/chosen": 0.7739871144294739,
      "logits/rejected": 0.7527830004692078,
      "logps/chosen": -202.1708221435547,
      "logps/rejected": -212.5369415283203,
      "loss": 0.5072,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.6656922101974487,
      "rewards/margins": 0.811048150062561,
      "rewards/rejected": -1.4767402410507202,
      "step": 2410
    },
    {
      "epoch": 1.936,
      "grad_norm": 4.5736799240112305,
      "learning_rate": 4.867969731071279e-06,
      "logits/chosen": 0.7770849466323853,
      "logits/rejected": 0.8255655169487,
      "logps/chosen": -192.43284606933594,
      "logps/rejected": -222.0802001953125,
      "loss": 0.5829,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.6381308436393738,
      "rewards/margins": 0.6074991226196289,
      "rewards/rejected": -1.245630145072937,
      "step": 2420
    },
    {
      "epoch": 1.944,
      "grad_norm": 7.796564102172852,
      "learning_rate": 4.86572173845168e-06,
      "logits/chosen": 0.9963685870170593,
      "logits/rejected": 0.986523449420929,
      "logps/chosen": -213.4120635986328,
      "logps/rejected": -214.7560577392578,
      "loss": 0.664,
      "rewards/accuracies": 0.5875000357627869,
      "rewards/chosen": -0.7294261455535889,
      "rewards/margins": 0.35328245162963867,
      "rewards/rejected": -1.0827085971832275,
      "step": 2430
    },
    {
      "epoch": 1.952,
      "grad_norm": 5.976226329803467,
      "learning_rate": 4.863455297458741e-06,
      "logits/chosen": 0.8093425631523132,
      "logits/rejected": 0.8130960464477539,
      "logps/chosen": -223.12828063964844,
      "logps/rejected": -246.886474609375,
      "loss": 0.6872,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.014013648033142,
      "rewards/margins": 0.4549499452114105,
      "rewards/rejected": -1.468963861465454,
      "step": 2440
    },
    {
      "epoch": 1.96,
      "grad_norm": 5.015947341918945,
      "learning_rate": 4.861170425766625e-06,
      "logits/chosen": 0.7283975481987,
      "logits/rejected": 0.7141422629356384,
      "logps/chosen": -223.5131378173828,
      "logps/rejected": -231.6356964111328,
      "loss": 0.5357,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.8296670913696289,
      "rewards/margins": 0.6724082827568054,
      "rewards/rejected": -1.5020753145217896,
      "step": 2450
    },
    {
      "epoch": 1.968,
      "grad_norm": 5.242801666259766,
      "learning_rate": 4.8588671411932195e-06,
      "logits/chosen": 0.8053815960884094,
      "logits/rejected": 0.8061729669570923,
      "logps/chosen": -214.3092041015625,
      "logps/rejected": -251.5323486328125,
      "loss": 0.5139,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.6689096689224243,
      "rewards/margins": 0.6673839688301086,
      "rewards/rejected": -1.3362935781478882,
      "step": 2460
    },
    {
      "epoch": 1.976,
      "grad_norm": 5.3611578941345215,
      "learning_rate": 4.8565454616999995e-06,
      "logits/chosen": 0.7702573537826538,
      "logits/rejected": 0.8161222338676453,
      "logps/chosen": -194.3104248046875,
      "logps/rejected": -229.4737091064453,
      "loss": 0.5365,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.5571290850639343,
      "rewards/margins": 0.699532151222229,
      "rewards/rejected": -1.2566611766815186,
      "step": 2470
    },
    {
      "epoch": 1.984,
      "grad_norm": 6.883541107177734,
      "learning_rate": 4.85420540539189e-06,
      "logits/chosen": 0.8040187954902649,
      "logits/rejected": 0.8295624852180481,
      "logps/chosen": -201.51536560058594,
      "logps/rejected": -207.3661651611328,
      "loss": 0.7064,
      "rewards/accuracies": 0.6500000357627869,
      "rewards/chosen": -0.622946560382843,
      "rewards/margins": 0.24029460549354553,
      "rewards/rejected": -0.8632411956787109,
      "step": 2480
    },
    {
      "epoch": 1.992,
      "grad_norm": 6.731320381164551,
      "learning_rate": 4.851846990517118e-06,
      "logits/chosen": 0.8790884017944336,
      "logits/rejected": 0.8123799562454224,
      "logps/chosen": -206.55789184570312,
      "logps/rejected": -234.25,
      "loss": 0.5478,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.8480615019798279,
      "rewards/margins": 0.6503373980522156,
      "rewards/rejected": -1.498399019241333,
      "step": 2490
    },
    {
      "epoch": 2.0,
      "grad_norm": 5.269995212554932,
      "learning_rate": 4.849470235467079e-06,
      "logits/chosen": 0.9131700396537781,
      "logits/rejected": 0.8961302638053894,
      "logps/chosen": -205.96377563476562,
      "logps/rejected": -208.3470458984375,
      "loss": 0.643,
      "rewards/accuracies": 0.5875000357627869,
      "rewards/chosen": -0.4859640598297119,
      "rewards/margins": 0.346769243478775,
      "rewards/rejected": -0.8327333331108093,
      "step": 2500
    },
    {
      "epoch": 2.008,
      "grad_norm": 4.879722595214844,
      "learning_rate": 4.847075158776183e-06,
      "logits/chosen": 0.8211914300918579,
      "logits/rejected": 0.8813266754150391,
      "logps/chosen": -202.9598846435547,
      "logps/rejected": -227.003173828125,
      "loss": 0.5717,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.9907218813896179,
      "rewards/margins": 0.5864070057868958,
      "rewards/rejected": -1.5771290063858032,
      "step": 2510
    },
    {
      "epoch": 2.016,
      "grad_norm": 4.4429168701171875,
      "learning_rate": 4.844661779121723e-06,
      "logits/chosen": 0.8528118133544922,
      "logits/rejected": 0.8727533221244812,
      "logps/chosen": -206.079833984375,
      "logps/rejected": -226.8438262939453,
      "loss": 0.4943,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.6906086206436157,
      "rewards/margins": 0.6708667874336243,
      "rewards/rejected": -1.3614753484725952,
      "step": 2520
    },
    {
      "epoch": 2.024,
      "grad_norm": 5.297170162200928,
      "learning_rate": 4.842230115323715e-06,
      "logits/chosen": 0.8990702629089355,
      "logits/rejected": 0.8813673257827759,
      "logps/chosen": -210.3358917236328,
      "logps/rejected": -215.9623565673828,
      "loss": 0.4864,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.9391742944717407,
      "rewards/margins": 0.8787464499473572,
      "rewards/rejected": -1.8179206848144531,
      "step": 2530
    },
    {
      "epoch": 2.032,
      "grad_norm": 5.085737228393555,
      "learning_rate": 4.839780186344763e-06,
      "logits/chosen": 0.7243828177452087,
      "logits/rejected": 0.5710186958312988,
      "logps/chosen": -218.251708984375,
      "logps/rejected": -233.02012634277344,
      "loss": 0.5227,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.8471910357475281,
      "rewards/margins": 0.7661673426628113,
      "rewards/rejected": -1.6133583784103394,
      "step": 2540
    },
    {
      "epoch": 2.04,
      "grad_norm": 6.776291847229004,
      "learning_rate": 4.837312011289907e-06,
      "logits/chosen": 0.8092204332351685,
      "logits/rejected": 0.7919169664382935,
      "logps/chosen": -233.19627380371094,
      "logps/rejected": -239.8424835205078,
      "loss": 0.5437,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3999868631362915,
      "rewards/margins": 0.5697715878486633,
      "rewards/rejected": -1.96975839138031,
      "step": 2550
    },
    {
      "epoch": 2.048,
      "grad_norm": 4.587131500244141,
      "learning_rate": 4.834825609406469e-06,
      "logits/chosen": 0.8447713851928711,
      "logits/rejected": 0.8319129347801208,
      "logps/chosen": -197.89254760742188,
      "logps/rejected": -224.28733825683594,
      "loss": 0.4837,
      "rewards/accuracies": 0.8375000357627869,
      "rewards/chosen": -0.7428638339042664,
      "rewards/margins": 0.8988615870475769,
      "rewards/rejected": -1.6417255401611328,
      "step": 2560
    },
    {
      "epoch": 2.056,
      "grad_norm": 5.485004425048828,
      "learning_rate": 4.832321000083912e-06,
      "logits/chosen": 0.6010708212852478,
      "logits/rejected": 0.6102247834205627,
      "logps/chosen": -233.9095001220703,
      "logps/rejected": -258.9781188964844,
      "loss": 0.5354,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.8256379961967468,
      "rewards/margins": 0.7594703435897827,
      "rewards/rejected": -1.5851082801818848,
      "step": 2570
    },
    {
      "epoch": 2.064,
      "grad_norm": 4.678139686584473,
      "learning_rate": 4.829798202853683e-06,
      "logits/chosen": 0.7654294371604919,
      "logits/rejected": 0.827416718006134,
      "logps/chosen": -204.60549926757812,
      "logps/rejected": -226.6314239501953,
      "loss": 0.5546,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.819423496723175,
      "rewards/margins": 0.5961082577705383,
      "rewards/rejected": -1.4155317544937134,
      "step": 2580
    },
    {
      "epoch": 2.072,
      "grad_norm": 7.408326625823975,
      "learning_rate": 4.82725723738906e-06,
      "logits/chosen": 0.9313848614692688,
      "logits/rejected": 0.9375463724136353,
      "logps/chosen": -222.64830017089844,
      "logps/rejected": -225.4774169921875,
      "loss": 0.6477,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.9466853141784668,
      "rewards/margins": 0.3815837800502777,
      "rewards/rejected": -1.328269124031067,
      "step": 2590
    },
    {
      "epoch": 2.08,
      "grad_norm": 7.954169273376465,
      "learning_rate": 4.824698123505004e-06,
      "logits/chosen": 0.8060113191604614,
      "logits/rejected": 0.7566053867340088,
      "logps/chosen": -192.39781188964844,
      "logps/rejected": -221.01512145996094,
      "loss": 0.5003,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.4853571057319641,
      "rewards/margins": 0.7401863932609558,
      "rewards/rejected": -1.2255436182022095,
      "step": 2600
    },
    {
      "epoch": 2.088,
      "grad_norm": 6.623762607574463,
      "learning_rate": 4.822120881157998e-06,
      "logits/chosen": 0.8647942543029785,
      "logits/rejected": 0.8719661831855774,
      "logps/chosen": -221.8902587890625,
      "logps/rejected": -240.48338317871094,
      "loss": 0.5385,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.855373203754425,
      "rewards/margins": 0.6286361813545227,
      "rewards/rejected": -1.4840092658996582,
      "step": 2610
    },
    {
      "epoch": 2.096,
      "grad_norm": 5.830476760864258,
      "learning_rate": 4.8195255304458945e-06,
      "logits/chosen": 0.8587938547134399,
      "logits/rejected": 0.7880618572235107,
      "logps/chosen": -208.57481384277344,
      "logps/rejected": -225.7516632080078,
      "loss": 0.5388,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.8550947308540344,
      "rewards/margins": 0.61111980676651,
      "rewards/rejected": -1.4662145376205444,
      "step": 2620
    },
    {
      "epoch": 2.104,
      "grad_norm": 5.968774795532227,
      "learning_rate": 4.8169120916077626e-06,
      "logits/chosen": 0.8810015916824341,
      "logits/rejected": 0.8329893946647644,
      "logps/chosen": -206.7748260498047,
      "logps/rejected": -212.00332641601562,
      "loss": 0.4447,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.9596864581108093,
      "rewards/margins": 0.9688228964805603,
      "rewards/rejected": -1.9285091161727905,
      "step": 2630
    },
    {
      "epoch": 2.112,
      "grad_norm": 4.1383161544799805,
      "learning_rate": 4.81428058502372e-06,
      "logits/chosen": 0.7850375175476074,
      "logits/rejected": 0.7830752730369568,
      "logps/chosen": -215.91201782226562,
      "logps/rejected": -243.4402313232422,
      "loss": 0.5962,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.5484000444412231,
      "rewards/margins": 0.7390782237052917,
      "rewards/rejected": -2.28747820854187,
      "step": 2640
    },
    {
      "epoch": 2.12,
      "grad_norm": 5.95521879196167,
      "learning_rate": 4.811631031214787e-06,
      "logits/chosen": 0.7586268782615662,
      "logits/rejected": 0.7738537192344666,
      "logps/chosen": -226.69517517089844,
      "logps/rejected": -252.53208923339844,
      "loss": 0.4531,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -1.1299831867218018,
      "rewards/margins": 1.046120047569275,
      "rewards/rejected": -2.176103353500366,
      "step": 2650
    },
    {
      "epoch": 2.128,
      "grad_norm": 5.219895839691162,
      "learning_rate": 4.808963450842713e-06,
      "logits/chosen": 0.9560911059379578,
      "logits/rejected": 0.9116310477256775,
      "logps/chosen": -205.336181640625,
      "logps/rejected": -206.9613800048828,
      "loss": 0.5378,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.8686642050743103,
      "rewards/margins": 0.6341574788093567,
      "rewards/rejected": -1.5028215646743774,
      "step": 2660
    },
    {
      "epoch": 2.136,
      "grad_norm": 4.962176322937012,
      "learning_rate": 4.806277864709828e-06,
      "logits/chosen": 0.9082285165786743,
      "logits/rejected": 0.8512415885925293,
      "logps/chosen": -217.08535766601562,
      "logps/rejected": -218.56155395507812,
      "loss": 0.4624,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.5144981741905212,
      "rewards/margins": 0.8028311133384705,
      "rewards/rejected": -1.3173291683197021,
      "step": 2670
    },
    {
      "epoch": 2.144,
      "grad_norm": 6.617359161376953,
      "learning_rate": 4.803574293758873e-06,
      "logits/chosen": 0.8195849657058716,
      "logits/rejected": 0.8258503079414368,
      "logps/chosen": -227.69168090820312,
      "logps/rejected": -232.21450805664062,
      "loss": 0.5883,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4341284036636353,
      "rewards/margins": 0.5511090755462646,
      "rewards/rejected": -1.9852374792099,
      "step": 2680
    },
    {
      "epoch": 2.152,
      "grad_norm": 4.731860637664795,
      "learning_rate": 4.800852759072834e-06,
      "logits/chosen": 1.0406241416931152,
      "logits/rejected": 0.9129249453544617,
      "logps/chosen": -212.03970336914062,
      "logps/rejected": -217.3359832763672,
      "loss": 0.5685,
      "rewards/accuracies": 0.7125000357627869,
      "rewards/chosen": -1.189060926437378,
      "rewards/margins": 0.6306447386741638,
      "rewards/rejected": -1.819705605506897,
      "step": 2690
    },
    {
      "epoch": 2.16,
      "grad_norm": 6.71656608581543,
      "learning_rate": 4.798113281874788e-06,
      "logits/chosen": 0.8752555847167969,
      "logits/rejected": 0.8828164935112,
      "logps/chosen": -216.815673828125,
      "logps/rejected": -230.06640625,
      "loss": 0.5298,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9842235445976257,
      "rewards/margins": 0.7561108469963074,
      "rewards/rejected": -1.740334391593933,
      "step": 2700
    },
    {
      "epoch": 2.168,
      "grad_norm": 5.923402309417725,
      "learning_rate": 4.795355883527727e-06,
      "logits/chosen": 1.119627833366394,
      "logits/rejected": 1.0217770338058472,
      "logps/chosen": -214.1453094482422,
      "logps/rejected": -225.8303680419922,
      "loss": 0.4973,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4931985139846802,
      "rewards/margins": 0.8571271300315857,
      "rewards/rejected": -2.350325584411621,
      "step": 2710
    },
    {
      "epoch": 2.176,
      "grad_norm": 5.549647331237793,
      "learning_rate": 4.792580585534398e-06,
      "logits/chosen": 0.976405918598175,
      "logits/rejected": 1.0035079717636108,
      "logps/chosen": -200.0377960205078,
      "logps/rejected": -236.29800415039062,
      "loss": 0.5329,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.0612291097640991,
      "rewards/margins": 0.7105744481086731,
      "rewards/rejected": -1.7718034982681274,
      "step": 2720
    },
    {
      "epoch": 2.184,
      "grad_norm": 8.303404808044434,
      "learning_rate": 4.789787409537131e-06,
      "logits/chosen": 0.9948248267173767,
      "logits/rejected": 0.940967857837677,
      "logps/chosen": -207.9303436279297,
      "logps/rejected": -244.0626220703125,
      "loss": 0.5104,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.080901026725769,
      "rewards/margins": 0.7775439620018005,
      "rewards/rejected": -1.8584450483322144,
      "step": 2730
    },
    {
      "epoch": 2.192,
      "grad_norm": 6.788060188293457,
      "learning_rate": 4.786976377317676e-06,
      "logits/chosen": 0.9980520606040955,
      "logits/rejected": 0.9337381720542908,
      "logps/chosen": -205.30735778808594,
      "logps/rejected": -212.91213989257812,
      "loss": 0.5431,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.287164568901062,
      "rewards/margins": 0.6435604691505432,
      "rewards/rejected": -1.93072509765625,
      "step": 2740
    },
    {
      "epoch": 2.2,
      "grad_norm": 5.9607133865356445,
      "learning_rate": 4.784147510797024e-06,
      "logits/chosen": 0.956340491771698,
      "logits/rejected": 0.8990178108215332,
      "logps/chosen": -207.7548828125,
      "logps/rejected": -227.23081970214844,
      "loss": 0.5219,
      "rewards/accuracies": 0.7125000357627869,
      "rewards/chosen": -1.1541296243667603,
      "rewards/margins": 0.8181205987930298,
      "rewards/rejected": -1.97225022315979,
      "step": 2750
    },
    {
      "epoch": 2.208,
      "grad_norm": 4.65596342086792,
      "learning_rate": 4.7813008320352475e-06,
      "logits/chosen": 0.886645495891571,
      "logits/rejected": 0.8505622148513794,
      "logps/chosen": -201.58055114746094,
      "logps/rejected": -216.75845336914062,
      "loss": 0.5183,
      "rewards/accuracies": 0.7125000357627869,
      "rewards/chosen": -0.5563634037971497,
      "rewards/margins": 0.788402259349823,
      "rewards/rejected": -1.344765543937683,
      "step": 2760
    },
    {
      "epoch": 2.216,
      "grad_norm": 3.351832628250122,
      "learning_rate": 4.778436363231317e-06,
      "logits/chosen": 0.9784888625144958,
      "logits/rejected": 0.9678120613098145,
      "logps/chosen": -209.824462890625,
      "logps/rejected": -236.84239196777344,
      "loss": 0.5421,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1734176874160767,
      "rewards/margins": 0.7453652024269104,
      "rewards/rejected": -1.9187828302383423,
      "step": 2770
    },
    {
      "epoch": 2.224,
      "grad_norm": 5.160139560699463,
      "learning_rate": 4.775554126722935e-06,
      "logits/chosen": 1.0381357669830322,
      "logits/rejected": 0.8997783064842224,
      "logps/chosen": -215.1231689453125,
      "logps/rejected": -221.33921813964844,
      "loss": 0.4999,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8012914657592773,
      "rewards/margins": 0.8245387077331543,
      "rewards/rejected": -1.625830054283142,
      "step": 2780
    },
    {
      "epoch": 2.232,
      "grad_norm": 6.553884983062744,
      "learning_rate": 4.772654144986364e-06,
      "logits/chosen": 1.0697473287582397,
      "logits/rejected": 0.9900814890861511,
      "logps/chosen": -190.27476501464844,
      "logps/rejected": -185.47320556640625,
      "loss": 0.5253,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.6551761031150818,
      "rewards/margins": 0.7400442361831665,
      "rewards/rejected": -1.395220398902893,
      "step": 2790
    },
    {
      "epoch": 2.24,
      "grad_norm": 7.37640905380249,
      "learning_rate": 4.7697364406362415e-06,
      "logits/chosen": 0.9499724507331848,
      "logits/rejected": 0.9506826400756836,
      "logps/chosen": -211.59814453125,
      "logps/rejected": -214.0547637939453,
      "loss": 0.5319,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.9677931070327759,
      "rewards/margins": 0.7696712613105774,
      "rewards/rejected": -1.7374645471572876,
      "step": 2800
    },
    {
      "epoch": 2.248,
      "grad_norm": 7.318824291229248,
      "learning_rate": 4.766801036425413e-06,
      "logits/chosen": 0.9329463243484497,
      "logits/rejected": 0.9652411341667175,
      "logps/chosen": -198.66159057617188,
      "logps/rejected": -218.34695434570312,
      "loss": 0.4791,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.636730432510376,
      "rewards/margins": 0.9497951865196228,
      "rewards/rejected": -1.5865256786346436,
      "step": 2810
    },
    {
      "epoch": 2.2560000000000002,
      "grad_norm": 8.074874877929688,
      "learning_rate": 4.763847955244749e-06,
      "logits/chosen": 0.9467241168022156,
      "logits/rejected": 0.9240646362304688,
      "logps/chosen": -195.85824584960938,
      "logps/rejected": -213.454345703125,
      "loss": 0.4331,
      "rewards/accuracies": 0.8375000357627869,
      "rewards/chosen": -0.5387411117553711,
      "rewards/margins": 0.931182861328125,
      "rewards/rejected": -1.4699242115020752,
      "step": 2820
    },
    {
      "epoch": 2.2640000000000002,
      "grad_norm": 5.222074508666992,
      "learning_rate": 4.760877220122972e-06,
      "logits/chosen": 1.0361279249191284,
      "logits/rejected": 0.9784205555915833,
      "logps/chosen": -217.472412109375,
      "logps/rejected": -220.15403747558594,
      "loss": 0.4908,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.6365905404090881,
      "rewards/margins": 0.8387727737426758,
      "rewards/rejected": -1.4753633737564087,
      "step": 2830
    },
    {
      "epoch": 2.2720000000000002,
      "grad_norm": 12.525833129882812,
      "learning_rate": 4.757888854226469e-06,
      "logits/chosen": 0.980974018573761,
      "logits/rejected": 0.8848344683647156,
      "logps/chosen": -217.03807067871094,
      "logps/rejected": -220.2837371826172,
      "loss": 0.6158,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.099002480506897,
      "rewards/margins": 0.5908426642417908,
      "rewards/rejected": -1.689845085144043,
      "step": 2840
    },
    {
      "epoch": 2.2800000000000002,
      "grad_norm": 7.606281280517578,
      "learning_rate": 4.75488288085912e-06,
      "logits/chosen": 0.886518120765686,
      "logits/rejected": 0.8294790387153625,
      "logps/chosen": -223.3667449951172,
      "logps/rejected": -241.37620544433594,
      "loss": 0.5392,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.5162465572357178,
      "rewards/margins": 0.6705383658409119,
      "rewards/rejected": -2.1867847442626953,
      "step": 2850
    },
    {
      "epoch": 2.288,
      "grad_norm": 6.877048015594482,
      "learning_rate": 4.751859323462106e-06,
      "logits/chosen": 0.8719585537910461,
      "logits/rejected": 0.8641806840896606,
      "logps/chosen": -196.090576171875,
      "logps/rejected": -222.262451171875,
      "loss": 0.4964,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.1291723251342773,
      "rewards/margins": 0.9274942278862,
      "rewards/rejected": -2.056666612625122,
      "step": 2860
    },
    {
      "epoch": 2.296,
      "grad_norm": 8.89814281463623,
      "learning_rate": 4.748818205613738e-06,
      "logits/chosen": 0.9758926630020142,
      "logits/rejected": 0.8365556597709656,
      "logps/chosen": -208.0929412841797,
      "logps/rejected": -219.41197204589844,
      "loss": 0.5806,
      "rewards/accuracies": 0.6500000357627869,
      "rewards/chosen": -0.8481144905090332,
      "rewards/margins": 0.6624639630317688,
      "rewards/rejected": -1.5105783939361572,
      "step": 2870
    },
    {
      "epoch": 2.304,
      "grad_norm": 10.317713737487793,
      "learning_rate": 4.7457595510292615e-06,
      "logits/chosen": 0.9229365587234497,
      "logits/rejected": 0.7857539057731628,
      "logps/chosen": -219.9251251220703,
      "logps/rejected": -253.33775329589844,
      "loss": 0.6269,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.2590759992599487,
      "rewards/margins": 0.696280300617218,
      "rewards/rejected": -1.9553560018539429,
      "step": 2880
    },
    {
      "epoch": 2.312,
      "grad_norm": 4.831161022186279,
      "learning_rate": 4.7426833835606815e-06,
      "logits/chosen": 0.977482259273529,
      "logits/rejected": 0.9541126489639282,
      "logps/chosen": -201.36666870117188,
      "logps/rejected": -229.928466796875,
      "loss": 0.5535,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.8802324533462524,
      "rewards/margins": 0.7807623744010925,
      "rewards/rejected": -1.6609947681427002,
      "step": 2890
    },
    {
      "epoch": 2.32,
      "grad_norm": 10.042828559875488,
      "learning_rate": 4.7395897271965676e-06,
      "logits/chosen": 0.9359084963798523,
      "logits/rejected": 0.8896707892417908,
      "logps/chosen": -226.84707641601562,
      "logps/rejected": -231.924072265625,
      "loss": 0.5353,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.173598051071167,
      "rewards/margins": 0.6731023192405701,
      "rewards/rejected": -1.8467003107070923,
      "step": 2900
    },
    {
      "epoch": 2.328,
      "grad_norm": 7.612265110015869,
      "learning_rate": 4.736478606061876e-06,
      "logits/chosen": 0.9218583106994629,
      "logits/rejected": 0.8763092160224915,
      "logps/chosen": -189.9997100830078,
      "logps/rejected": -222.7259521484375,
      "loss": 0.4865,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.624975323677063,
      "rewards/margins": 0.9700756072998047,
      "rewards/rejected": -1.5950509309768677,
      "step": 2910
    },
    {
      "epoch": 2.336,
      "grad_norm": 7.228297710418701,
      "learning_rate": 4.733350044417752e-06,
      "logits/chosen": 0.8519703149795532,
      "logits/rejected": 0.9009062051773071,
      "logps/chosen": -211.00601196289062,
      "logps/rejected": -227.3970184326172,
      "loss": 0.5175,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.1423438787460327,
      "rewards/margins": 0.9418218731880188,
      "rewards/rejected": -2.084165573120117,
      "step": 2920
    },
    {
      "epoch": 2.344,
      "grad_norm": 7.3916192054748535,
      "learning_rate": 4.730204066661349e-06,
      "logits/chosen": 0.8180250525474548,
      "logits/rejected": 0.8404077887535095,
      "logps/chosen": -219.1990509033203,
      "logps/rejected": -247.86865234375,
      "loss": 0.4079,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -1.1498545408248901,
      "rewards/margins": 1.2002915143966675,
      "rewards/rejected": -2.3501460552215576,
      "step": 2930
    },
    {
      "epoch": 2.352,
      "grad_norm": 7.41680383682251,
      "learning_rate": 4.727040697325634e-06,
      "logits/chosen": 0.8754854202270508,
      "logits/rejected": 0.8143682479858398,
      "logps/chosen": -216.9535675048828,
      "logps/rejected": -224.46226501464844,
      "loss": 0.4467,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.9588058590888977,
      "rewards/margins": 0.9411051869392395,
      "rewards/rejected": -1.8999111652374268,
      "step": 2940
    },
    {
      "epoch": 2.36,
      "grad_norm": 9.07734203338623,
      "learning_rate": 4.723859961079196e-06,
      "logits/chosen": 0.9282005429267883,
      "logits/rejected": 1.0409669876098633,
      "logps/chosen": -200.03123474121094,
      "logps/rejected": -205.33560180664062,
      "loss": 0.5902,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.9528915286064148,
      "rewards/margins": 0.5995592474937439,
      "rewards/rejected": -1.5524507761001587,
      "step": 2950
    },
    {
      "epoch": 2.368,
      "grad_norm": 7.95607328414917,
      "learning_rate": 4.720661882726054e-06,
      "logits/chosen": 0.8046501278877258,
      "logits/rejected": 0.8820120096206665,
      "logps/chosen": -198.43272399902344,
      "logps/rejected": -228.85073852539062,
      "loss": 0.5915,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.2360581159591675,
      "rewards/margins": 0.611167848110199,
      "rewards/rejected": -1.8472260236740112,
      "step": 2960
    },
    {
      "epoch": 2.376,
      "grad_norm": 9.31917667388916,
      "learning_rate": 4.717446487205466e-06,
      "logits/chosen": 1.0128583908081055,
      "logits/rejected": 0.8825269937515259,
      "logps/chosen": -221.35952758789062,
      "logps/rejected": -233.3060760498047,
      "loss": 0.5001,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.7609128952026367,
      "rewards/margins": 0.8846578598022461,
      "rewards/rejected": -1.6455707550048828,
      "step": 2970
    },
    {
      "epoch": 2.384,
      "grad_norm": 6.160464763641357,
      "learning_rate": 4.714213799591733e-06,
      "logits/chosen": 0.9421942830085754,
      "logits/rejected": 0.9018292427062988,
      "logps/chosen": -205.28750610351562,
      "logps/rejected": -214.02294921875,
      "loss": 0.5016,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7612765431404114,
      "rewards/margins": 0.7555828094482422,
      "rewards/rejected": -1.5168594121932983,
      "step": 2980
    },
    {
      "epoch": 2.392,
      "grad_norm": 5.200317859649658,
      "learning_rate": 4.710963845094003e-06,
      "logits/chosen": 0.8887328505516052,
      "logits/rejected": 0.8375994563102722,
      "logps/chosen": -196.9802703857422,
      "logps/rejected": -206.02444458007812,
      "loss": 0.5104,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.6040834784507751,
      "rewards/margins": 0.7478589415550232,
      "rewards/rejected": -1.3519423007965088,
      "step": 2990
    },
    {
      "epoch": 2.4,
      "grad_norm": 5.419322490692139,
      "learning_rate": 4.707696649056073e-06,
      "logits/chosen": 0.8664781451225281,
      "logits/rejected": 0.8220338821411133,
      "logps/chosen": -191.3097686767578,
      "logps/rejected": -210.8022918701172,
      "loss": 0.4085,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4698951244354248,
      "rewards/margins": 1.0510791540145874,
      "rewards/rejected": -1.5209741592407227,
      "step": 3000
    },
    {
      "epoch": 2.408,
      "grad_norm": 7.53326416015625,
      "learning_rate": 4.704412236956194e-06,
      "logits/chosen": 0.8981558680534363,
      "logits/rejected": 0.9666721224784851,
      "logps/chosen": -210.7949981689453,
      "logps/rejected": -252.54966735839844,
      "loss": 0.4686,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.2617671489715576,
      "rewards/margins": 1.07135808467865,
      "rewards/rejected": -2.333125352859497,
      "step": 3010
    },
    {
      "epoch": 2.416,
      "grad_norm": 4.284425258636475,
      "learning_rate": 4.701110634406871e-06,
      "logits/chosen": 0.7835731506347656,
      "logits/rejected": 0.7178624272346497,
      "logps/chosen": -197.4336395263672,
      "logps/rejected": -236.43325805664062,
      "loss": 0.456,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.1816260814666748,
      "rewards/margins": 1.00633704662323,
      "rewards/rejected": -2.1879632472991943,
      "step": 3020
    },
    {
      "epoch": 2.424,
      "grad_norm": 8.3016357421875,
      "learning_rate": 4.6977918671546635e-06,
      "logits/chosen": 0.9753482937812805,
      "logits/rejected": 0.8922098278999329,
      "logps/chosen": -211.79405212402344,
      "logps/rejected": -218.29739379882812,
      "loss": 0.5311,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.0505523681640625,
      "rewards/margins": 0.8706418871879578,
      "rewards/rejected": -1.9211944341659546,
      "step": 3030
    },
    {
      "epoch": 2.432,
      "grad_norm": 9.126633644104004,
      "learning_rate": 4.6944559610799865e-06,
      "logits/chosen": 1.0277721881866455,
      "logits/rejected": 1.0039381980895996,
      "logps/chosen": -183.63748168945312,
      "logps/rejected": -211.2511749267578,
      "loss": 0.5654,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.6382626891136169,
      "rewards/margins": 0.8599988222122192,
      "rewards/rejected": -1.4982614517211914,
      "step": 3040
    },
    {
      "epoch": 2.44,
      "grad_norm": 7.879682540893555,
      "learning_rate": 4.691102942196905e-06,
      "logits/chosen": 0.9624242186546326,
      "logits/rejected": 1.0566548109054565,
      "logps/chosen": -197.49008178710938,
      "logps/rejected": -237.42349243164062,
      "loss": 0.5254,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.352699875831604,
      "rewards/margins": 0.8264394998550415,
      "rewards/rejected": -2.1791393756866455,
      "step": 3050
    },
    {
      "epoch": 2.448,
      "grad_norm": 8.244893074035645,
      "learning_rate": 4.687732836652935e-06,
      "logits/chosen": 1.102777361869812,
      "logits/rejected": 0.9323251843452454,
      "logps/chosen": -211.6886749267578,
      "logps/rejected": -223.81761169433594,
      "loss": 0.5689,
      "rewards/accuracies": 0.6500000357627869,
      "rewards/chosen": -1.4250367879867554,
      "rewards/margins": 0.8165454864501953,
      "rewards/rejected": -2.2415823936462402,
      "step": 3060
    },
    {
      "epoch": 2.456,
      "grad_norm": 10.064545631408691,
      "learning_rate": 4.684345670728835e-06,
      "logits/chosen": 0.9085485339164734,
      "logits/rejected": 0.9594566226005554,
      "logps/chosen": -216.97305297851562,
      "logps/rejected": -245.9871368408203,
      "loss": 0.5879,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.5272258520126343,
      "rewards/margins": 0.6840270161628723,
      "rewards/rejected": -2.2112529277801514,
      "step": 3070
    },
    {
      "epoch": 2.464,
      "grad_norm": 5.9822773933410645,
      "learning_rate": 4.680941470838405e-06,
      "logits/chosen": 0.9992641806602478,
      "logits/rejected": 0.9386343955993652,
      "logps/chosen": -211.3744659423828,
      "logps/rejected": -215.39097595214844,
      "loss": 0.5522,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.8485382199287415,
      "rewards/margins": 0.7580586075782776,
      "rewards/rejected": -1.606596827507019,
      "step": 3080
    },
    {
      "epoch": 2.472,
      "grad_norm": 9.379515647888184,
      "learning_rate": 4.67752026352828e-06,
      "logits/chosen": 0.9449491500854492,
      "logits/rejected": 0.8973454833030701,
      "logps/chosen": -229.10166931152344,
      "logps/rejected": -235.14395141601562,
      "loss": 0.5425,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.0061733722686768,
      "rewards/margins": 0.7214813828468323,
      "rewards/rejected": -1.7276546955108643,
      "step": 3090
    },
    {
      "epoch": 2.48,
      "grad_norm": 7.775414943695068,
      "learning_rate": 4.674082075477724e-06,
      "logits/chosen": 0.9458308219909668,
      "logits/rejected": 0.9328064322471619,
      "logps/chosen": -194.5631561279297,
      "logps/rejected": -212.3477325439453,
      "loss": 0.5018,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9718823432922363,
      "rewards/margins": 0.8911674618721008,
      "rewards/rejected": -1.8630497455596924,
      "step": 3100
    },
    {
      "epoch": 2.488,
      "grad_norm": 8.869393348693848,
      "learning_rate": 4.670626933498415e-06,
      "logits/chosen": 1.0055527687072754,
      "logits/rejected": 0.8751175999641418,
      "logps/chosen": -205.0562744140625,
      "logps/rejected": -214.41934204101562,
      "loss": 0.5698,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.790256679058075,
      "rewards/margins": 0.7495672106742859,
      "rewards/rejected": -1.5398238897323608,
      "step": 3110
    },
    {
      "epoch": 2.496,
      "grad_norm": 7.518993854522705,
      "learning_rate": 4.667154864534245e-06,
      "logits/chosen": 0.9388038516044617,
      "logits/rejected": 0.9028251767158508,
      "logps/chosen": -189.31138610839844,
      "logps/rejected": -246.3251495361328,
      "loss": 0.4711,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.7512499690055847,
      "rewards/margins": 0.9976223111152649,
      "rewards/rejected": -1.7488723993301392,
      "step": 3120
    },
    {
      "epoch": 2.504,
      "grad_norm": 5.921288967132568,
      "learning_rate": 4.663665895661107e-06,
      "logits/chosen": 0.9502741694450378,
      "logits/rejected": 0.9423877596855164,
      "logps/chosen": -201.31776428222656,
      "logps/rejected": -228.7478790283203,
      "loss": 0.6146,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.0320862531661987,
      "rewards/margins": 0.730940580368042,
      "rewards/rejected": -1.7630270719528198,
      "step": 3130
    },
    {
      "epoch": 2.512,
      "grad_norm": 7.64976692199707,
      "learning_rate": 4.6601600540866794e-06,
      "logits/chosen": 0.9315633773803711,
      "logits/rejected": 0.9705595374107361,
      "logps/chosen": -207.4699249267578,
      "logps/rejected": -223.7519989013672,
      "loss": 0.5229,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.7369431257247925,
      "rewards/margins": 0.7962962985038757,
      "rewards/rejected": -1.5332393646240234,
      "step": 3140
    },
    {
      "epoch": 2.52,
      "grad_norm": 10.3628511428833,
      "learning_rate": 4.65663736715022e-06,
      "logits/chosen": 1.0306199789047241,
      "logits/rejected": 0.9668065309524536,
      "logps/chosen": -216.719970703125,
      "logps/rejected": -218.1455535888672,
      "loss": 0.5989,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.6501615047454834,
      "rewards/margins": 0.6095995306968689,
      "rewards/rejected": -1.259761095046997,
      "step": 3150
    },
    {
      "epoch": 2.528,
      "grad_norm": 8.640954971313477,
      "learning_rate": 4.653097862322347e-06,
      "logits/chosen": 0.9273595809936523,
      "logits/rejected": 0.9403362274169922,
      "logps/chosen": -213.16714477539062,
      "logps/rejected": -236.9595489501953,
      "loss": 0.5293,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.7404825687408447,
      "rewards/margins": 0.8145327568054199,
      "rewards/rejected": -1.5550154447555542,
      "step": 3160
    },
    {
      "epoch": 2.536,
      "grad_norm": 7.425879001617432,
      "learning_rate": 4.6495415672048336e-06,
      "logits/chosen": 0.8542930483818054,
      "logits/rejected": 0.880113422870636,
      "logps/chosen": -213.308837890625,
      "logps/rejected": -242.27099609375,
      "loss": 0.4891,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.4495139718055725,
      "rewards/margins": 0.8645074963569641,
      "rewards/rejected": -1.3140214681625366,
      "step": 3170
    },
    {
      "epoch": 2.544,
      "grad_norm": 6.936334609985352,
      "learning_rate": 4.645968509530381e-06,
      "logits/chosen": 0.985787034034729,
      "logits/rejected": 0.970470130443573,
      "logps/chosen": -205.81802368164062,
      "logps/rejected": -199.0405731201172,
      "loss": 0.5015,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6486034393310547,
      "rewards/margins": 0.8106253743171692,
      "rewards/rejected": -1.4592288732528687,
      "step": 3180
    },
    {
      "epoch": 2.552,
      "grad_norm": 9.339189529418945,
      "learning_rate": 4.642378717162411e-06,
      "logits/chosen": 0.9108757972717285,
      "logits/rejected": 0.8992031216621399,
      "logps/chosen": -212.0107879638672,
      "logps/rejected": -223.214599609375,
      "loss": 0.5662,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.43249455094337463,
      "rewards/margins": 0.7586955428123474,
      "rewards/rejected": -1.1911901235580444,
      "step": 3190
    },
    {
      "epoch": 2.56,
      "grad_norm": 10.31650161743164,
      "learning_rate": 4.638772218094847e-06,
      "logits/chosen": 0.8744648098945618,
      "logits/rejected": 0.9359402060508728,
      "logps/chosen": -211.1424560546875,
      "logps/rejected": -240.16343688964844,
      "loss": 0.5185,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.9992405772209167,
      "rewards/margins": 0.8215592503547668,
      "rewards/rejected": -1.820799708366394,
      "step": 3200
    },
    {
      "epoch": 2.568,
      "grad_norm": 9.806978225708008,
      "learning_rate": 4.635149040451891e-06,
      "logits/chosen": 1.0247632265090942,
      "logits/rejected": 0.971980094909668,
      "logps/chosen": -204.77330017089844,
      "logps/rejected": -208.4816131591797,
      "loss": 0.5257,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.39921775460243225,
      "rewards/margins": 0.7709429860115051,
      "rewards/rejected": -1.1701607704162598,
      "step": 3210
    },
    {
      "epoch": 2.576,
      "grad_norm": 7.172482490539551,
      "learning_rate": 4.631509212487812e-06,
      "logits/chosen": 0.9954729080200195,
      "logits/rejected": 0.9880996942520142,
      "logps/chosen": -210.8682098388672,
      "logps/rejected": -213.47573852539062,
      "loss": 0.5339,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -0.9985690116882324,
      "rewards/margins": 0.739185094833374,
      "rewards/rejected": -1.7377541065216064,
      "step": 3220
    },
    {
      "epoch": 2.584,
      "grad_norm": 11.274581909179688,
      "learning_rate": 4.627852762586718e-06,
      "logits/chosen": 0.9896817207336426,
      "logits/rejected": 1.021188735961914,
      "logps/chosen": -189.18344116210938,
      "logps/rejected": -215.19497680664062,
      "loss": 0.5474,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.4847024381160736,
      "rewards/margins": 0.7209543585777283,
      "rewards/rejected": -1.205656886100769,
      "step": 3230
    },
    {
      "epoch": 2.592,
      "grad_norm": 5.447055816650391,
      "learning_rate": 4.624179719262342e-06,
      "logits/chosen": 1.0467568635940552,
      "logits/rejected": 1.0570539236068726,
      "logps/chosen": -201.77415466308594,
      "logps/rejected": -206.077392578125,
      "loss": 0.4613,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.5657867789268494,
      "rewards/margins": 0.8899927139282227,
      "rewards/rejected": -1.4557795524597168,
      "step": 3240
    },
    {
      "epoch": 2.6,
      "grad_norm": 7.587062358856201,
      "learning_rate": 4.62049011115781e-06,
      "logits/chosen": 0.9699912071228027,
      "logits/rejected": 1.017883539199829,
      "logps/chosen": -205.2608642578125,
      "logps/rejected": -220.08792114257812,
      "loss": 0.4848,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.7234296798706055,
      "rewards/margins": 0.8834452033042908,
      "rewards/rejected": -1.6068748235702515,
      "step": 3250
    },
    {
      "epoch": 2.608,
      "grad_norm": 6.428370475769043,
      "learning_rate": 4.616783967045432e-06,
      "logits/chosen": 1.0170371532440186,
      "logits/rejected": 0.9278216361999512,
      "logps/chosen": -205.9370574951172,
      "logps/rejected": -226.574462890625,
      "loss": 0.443,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.777590274810791,
      "rewards/margins": 0.9760915637016296,
      "rewards/rejected": -1.753682017326355,
      "step": 3260
    },
    {
      "epoch": 2.616,
      "grad_norm": 9.805668830871582,
      "learning_rate": 4.6130613158264605e-06,
      "logits/chosen": 0.8976919054985046,
      "logits/rejected": 0.9280143976211548,
      "logps/chosen": -206.73876953125,
      "logps/rejected": -206.44371032714844,
      "loss": 0.5154,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8372632265090942,
      "rewards/margins": 0.7562478184700012,
      "rewards/rejected": -1.5935109853744507,
      "step": 3270
    },
    {
      "epoch": 2.624,
      "grad_norm": 8.653728485107422,
      "learning_rate": 4.6093221865308795e-06,
      "logits/chosen": 0.9815517663955688,
      "logits/rejected": 0.926008403301239,
      "logps/chosen": -196.36875915527344,
      "logps/rejected": -215.28271484375,
      "loss": 0.5294,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -0.9871358871459961,
      "rewards/margins": 0.8871564865112305,
      "rewards/rejected": -1.874292254447937,
      "step": 3280
    },
    {
      "epoch": 2.632,
      "grad_norm": 4.758111953735352,
      "learning_rate": 4.605566608317169e-06,
      "logits/chosen": 0.8787722587585449,
      "logits/rejected": 0.8375174403190613,
      "logps/chosen": -190.63404846191406,
      "logps/rejected": -210.22802734375,
      "loss": 0.5017,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5020423531532288,
      "rewards/margins": 0.8334178328514099,
      "rewards/rejected": -1.3354600667953491,
      "step": 3290
    },
    {
      "epoch": 2.64,
      "grad_norm": 7.596653938293457,
      "learning_rate": 4.601794610472083e-06,
      "logits/chosen": 0.9046363830566406,
      "logits/rejected": 0.8183378577232361,
      "logps/chosen": -215.7941436767578,
      "logps/rejected": -241.56838989257812,
      "loss": 0.4736,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.9933833479881287,
      "rewards/margins": 0.9869853854179382,
      "rewards/rejected": -1.9803688526153564,
      "step": 3300
    },
    {
      "epoch": 2.648,
      "grad_norm": 12.559988975524902,
      "learning_rate": 4.598006222410419e-06,
      "logits/chosen": 0.9887520670890808,
      "logits/rejected": 1.0198968648910522,
      "logps/chosen": -188.9550018310547,
      "logps/rejected": -219.6782684326172,
      "loss": 0.4814,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -0.820119321346283,
      "rewards/margins": 0.9246525168418884,
      "rewards/rejected": -1.7447718381881714,
      "step": 3310
    },
    {
      "epoch": 2.656,
      "grad_norm": 5.864845275878906,
      "learning_rate": 4.594201473674788e-06,
      "logits/chosen": 0.9294392466545105,
      "logits/rejected": 0.796425461769104,
      "logps/chosen": -197.6046905517578,
      "logps/rejected": -234.70787048339844,
      "loss": 0.4557,
      "rewards/accuracies": 0.8375000357627869,
      "rewards/chosen": -1.0228973627090454,
      "rewards/margins": 0.9283467531204224,
      "rewards/rejected": -1.9512439966201782,
      "step": 3320
    },
    {
      "epoch": 2.664,
      "grad_norm": 8.346921920776367,
      "learning_rate": 4.590380393935383e-06,
      "logits/chosen": 0.8741597533226013,
      "logits/rejected": 0.836887001991272,
      "logps/chosen": -214.54185485839844,
      "logps/rejected": -223.0530242919922,
      "loss": 0.5895,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.2219938039779663,
      "rewards/margins": 0.6594648361206055,
      "rewards/rejected": -1.8814586400985718,
      "step": 3330
    },
    {
      "epoch": 2.672,
      "grad_norm": 7.899364948272705,
      "learning_rate": 4.5865430129897536e-06,
      "logits/chosen": 0.857835590839386,
      "logits/rejected": 0.8607529997825623,
      "logps/chosen": -210.548095703125,
      "logps/rejected": -236.50767517089844,
      "loss": 0.5514,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.091759443283081,
      "rewards/margins": 0.8923540115356445,
      "rewards/rejected": -1.984113335609436,
      "step": 3340
    },
    {
      "epoch": 2.68,
      "grad_norm": 6.950599193572998,
      "learning_rate": 4.5826893607625665e-06,
      "logits/chosen": 0.8608657717704773,
      "logits/rejected": 0.9021877646446228,
      "logps/chosen": -219.5095672607422,
      "logps/rejected": -248.8926239013672,
      "loss": 0.4884,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.17706298828125,
      "rewards/margins": 1.054115653038025,
      "rewards/rejected": -2.2311787605285645,
      "step": 3350
    },
    {
      "epoch": 2.6879999999999997,
      "grad_norm": 6.3925065994262695,
      "learning_rate": 4.578819467305375e-06,
      "logits/chosen": 0.8635089993476868,
      "logits/rejected": 0.9291839599609375,
      "logps/chosen": -208.9047393798828,
      "logps/rejected": -242.42764282226562,
      "loss": 0.5177,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.8883923888206482,
      "rewards/margins": 0.8316335678100586,
      "rewards/rejected": -1.7200260162353516,
      "step": 3360
    },
    {
      "epoch": 2.6959999999999997,
      "grad_norm": 7.390092372894287,
      "learning_rate": 4.5749333627963886e-06,
      "logits/chosen": 1.025161623954773,
      "logits/rejected": 0.9390385746955872,
      "logps/chosen": -192.0789031982422,
      "logps/rejected": -208.17300415039062,
      "loss": 0.6558,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.7664794325828552,
      "rewards/margins": 0.47816377878189087,
      "rewards/rejected": -1.244643211364746,
      "step": 3370
    },
    {
      "epoch": 2.7039999999999997,
      "grad_norm": 6.525977611541748,
      "learning_rate": 4.571031077540227e-06,
      "logits/chosen": 0.8534032702445984,
      "logits/rejected": 0.847669780254364,
      "logps/chosen": -206.7920684814453,
      "logps/rejected": -226.60240173339844,
      "loss": 0.5259,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.8559421896934509,
      "rewards/margins": 0.7720093727111816,
      "rewards/rejected": -1.6279516220092773,
      "step": 3380
    },
    {
      "epoch": 2.7119999999999997,
      "grad_norm": 11.586974143981934,
      "learning_rate": 4.567112641967697e-06,
      "logits/chosen": 0.849672794342041,
      "logits/rejected": 0.7306337356567383,
      "logps/chosen": -230.2295379638672,
      "logps/rejected": -227.7930908203125,
      "loss": 0.5625,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.9319648146629333,
      "rewards/margins": 0.6750860214233398,
      "rewards/rejected": -1.607050895690918,
      "step": 3390
    },
    {
      "epoch": 2.7199999999999998,
      "grad_norm": 13.69837474822998,
      "learning_rate": 4.563178086635546e-06,
      "logits/chosen": 0.9492778182029724,
      "logits/rejected": 0.9037938117980957,
      "logps/chosen": -210.29478454589844,
      "logps/rejected": -216.89903259277344,
      "loss": 0.5038,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -0.8451706171035767,
      "rewards/margins": 0.8125400543212891,
      "rewards/rejected": -1.6577106714248657,
      "step": 3400
    },
    {
      "epoch": 2.7279999999999998,
      "grad_norm": 4.132405757904053,
      "learning_rate": 4.559227442226226e-06,
      "logits/chosen": 0.9789943695068359,
      "logits/rejected": 0.9167888760566711,
      "logps/chosen": -191.22715759277344,
      "logps/rejected": -232.78164672851562,
      "loss": 0.4735,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -0.7555204629898071,
      "rewards/margins": 0.9841014742851257,
      "rewards/rejected": -1.7396221160888672,
      "step": 3410
    },
    {
      "epoch": 2.7359999999999998,
      "grad_norm": 6.608091831207275,
      "learning_rate": 4.555260739547657e-06,
      "logits/chosen": 1.0244665145874023,
      "logits/rejected": 0.8857207298278809,
      "logps/chosen": -203.6723175048828,
      "logps/rejected": -212.9291534423828,
      "loss": 0.5356,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.6175267100334167,
      "rewards/margins": 0.7223233580589294,
      "rewards/rejected": -1.3398501873016357,
      "step": 3420
    },
    {
      "epoch": 2.7439999999999998,
      "grad_norm": 9.938222885131836,
      "learning_rate": 4.551278009532981e-06,
      "logits/chosen": 1.0124415159225464,
      "logits/rejected": 0.9431573152542114,
      "logps/chosen": -207.00718688964844,
      "logps/rejected": -225.10806274414062,
      "loss": 0.5292,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.5922741889953613,
      "rewards/margins": 0.7229223251342773,
      "rewards/rejected": -1.3151965141296387,
      "step": 3430
    },
    {
      "epoch": 2.752,
      "grad_norm": 6.640532970428467,
      "learning_rate": 4.5472792832403295e-06,
      "logits/chosen": 0.9901018142700195,
      "logits/rejected": 0.8438584208488464,
      "logps/chosen": -192.44761657714844,
      "logps/rejected": -208.34683227539062,
      "loss": 0.4769,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.6061093211174011,
      "rewards/margins": 0.9215426445007324,
      "rewards/rejected": -1.5276520252227783,
      "step": 3440
    },
    {
      "epoch": 2.76,
      "grad_norm": 7.896131992340088,
      "learning_rate": 4.543264591852572e-06,
      "logits/chosen": 0.9401613473892212,
      "logits/rejected": 1.0364055633544922,
      "logps/chosen": -212.4904022216797,
      "logps/rejected": -249.1276092529297,
      "loss": 0.6105,
      "rewards/accuracies": 0.6500000357627869,
      "rewards/chosen": -0.9996709823608398,
      "rewards/margins": 0.6035251021385193,
      "rewards/rejected": -1.6031960248947144,
      "step": 3450
    },
    {
      "epoch": 2.768,
      "grad_norm": 14.408479690551758,
      "learning_rate": 4.539233966677078e-06,
      "logits/chosen": 0.9975506067276001,
      "logits/rejected": 1.0087366104125977,
      "logps/chosen": -220.4411163330078,
      "logps/rejected": -230.56089782714844,
      "loss": 0.5005,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.6962399482727051,
      "rewards/margins": 0.7583475112915039,
      "rewards/rejected": -1.4545873403549194,
      "step": 3460
    },
    {
      "epoch": 2.776,
      "grad_norm": 9.091312408447266,
      "learning_rate": 4.535187439145473e-06,
      "logits/chosen": 1.004744529724121,
      "logits/rejected": 0.956881046295166,
      "logps/chosen": -208.3164520263672,
      "logps/rejected": -249.3081512451172,
      "loss": 0.4817,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8332870602607727,
      "rewards/margins": 0.9917869567871094,
      "rewards/rejected": -1.8250740766525269,
      "step": 3470
    },
    {
      "epoch": 2.784,
      "grad_norm": 9.282960891723633,
      "learning_rate": 4.531125040813392e-06,
      "logits/chosen": 0.9015275835990906,
      "logits/rejected": 0.8595914244651794,
      "logps/chosen": -206.90184020996094,
      "logps/rejected": -241.4942626953125,
      "loss": 0.4265,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.8733939528465271,
      "rewards/margins": 1.2572206258773804,
      "rewards/rejected": -2.1306145191192627,
      "step": 3480
    },
    {
      "epoch": 2.792,
      "grad_norm": 7.599782943725586,
      "learning_rate": 4.527046803360232e-06,
      "logits/chosen": 1.0097182989120483,
      "logits/rejected": 0.9868324398994446,
      "logps/chosen": -197.34820556640625,
      "logps/rejected": -220.2039337158203,
      "loss": 0.4665,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.28118035197257996,
      "rewards/margins": 0.9197006225585938,
      "rewards/rejected": -1.200881004333496,
      "step": 3490
    },
    {
      "epoch": 2.8,
      "grad_norm": 10.817337036132812,
      "learning_rate": 4.522952758588909e-06,
      "logits/chosen": 1.014154314994812,
      "logits/rejected": 1.0231674909591675,
      "logps/chosen": -199.9979705810547,
      "logps/rejected": -226.4349822998047,
      "loss": 0.4632,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.3697163462638855,
      "rewards/margins": 0.8839020133018494,
      "rewards/rejected": -1.2536184787750244,
      "step": 3500
    },
    {
      "epoch": 2.808,
      "grad_norm": 10.987738609313965,
      "learning_rate": 4.518842938425606e-06,
      "logits/chosen": 1.0434938669204712,
      "logits/rejected": 0.9931101202964783,
      "logps/chosen": -193.0313262939453,
      "logps/rejected": -202.6310272216797,
      "loss": 0.493,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.3265318274497986,
      "rewards/margins": 0.9020771980285645,
      "rewards/rejected": -1.2286089658737183,
      "step": 3510
    },
    {
      "epoch": 2.816,
      "grad_norm": 6.466224193572998,
      "learning_rate": 4.514717374919525e-06,
      "logits/chosen": 1.0711123943328857,
      "logits/rejected": 0.9438812136650085,
      "logps/chosen": -204.62571716308594,
      "logps/rejected": -227.7462921142578,
      "loss": 0.4658,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.47518759965896606,
      "rewards/margins": 1.0582878589630127,
      "rewards/rejected": -1.5334755182266235,
      "step": 3520
    },
    {
      "epoch": 2.824,
      "grad_norm": 9.006815910339355,
      "learning_rate": 4.510576100242642e-06,
      "logits/chosen": 1.0240957736968994,
      "logits/rejected": 0.9844247698783875,
      "logps/chosen": -198.17991638183594,
      "logps/rejected": -212.7672576904297,
      "loss": 0.5084,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.45504727959632874,
      "rewards/margins": 0.9179355502128601,
      "rewards/rejected": -1.3729829788208008,
      "step": 3530
    },
    {
      "epoch": 2.832,
      "grad_norm": 7.647313594818115,
      "learning_rate": 4.506419146689445e-06,
      "logits/chosen": 1.0858877897262573,
      "logits/rejected": 0.9952969551086426,
      "logps/chosen": -204.3494110107422,
      "logps/rejected": -224.9398651123047,
      "loss": 0.4831,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -0.022721266373991966,
      "rewards/margins": 1.0130819082260132,
      "rewards/rejected": -1.035803198814392,
      "step": 3540
    },
    {
      "epoch": 2.84,
      "grad_norm": 10.288926124572754,
      "learning_rate": 4.502246546676697e-06,
      "logits/chosen": 1.0694109201431274,
      "logits/rejected": 1.054747462272644,
      "logps/chosen": -194.4237518310547,
      "logps/rejected": -223.91714477539062,
      "loss": 0.5075,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.7783587574958801,
      "rewards/margins": 0.9766228795051575,
      "rewards/rejected": -1.7549816370010376,
      "step": 3550
    },
    {
      "epoch": 2.848,
      "grad_norm": 7.374194145202637,
      "learning_rate": 4.498058332743168e-06,
      "logits/chosen": 0.7898425459861755,
      "logits/rejected": 0.7930797934532166,
      "logps/chosen": -203.2722625732422,
      "logps/rejected": -237.77330017089844,
      "loss": 0.5055,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -0.446158230304718,
      "rewards/margins": 0.8784043192863464,
      "rewards/rejected": -1.3245625495910645,
      "step": 3560
    },
    {
      "epoch": 2.856,
      "grad_norm": 9.789468765258789,
      "learning_rate": 4.493854537549393e-06,
      "logits/chosen": 1.0634657144546509,
      "logits/rejected": 1.1062793731689453,
      "logps/chosen": -202.4827117919922,
      "logps/rejected": -214.6761474609375,
      "loss": 0.49,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -0.2086171656847,
      "rewards/margins": 0.8339020609855652,
      "rewards/rejected": -1.042519211769104,
      "step": 3570
    },
    {
      "epoch": 2.864,
      "grad_norm": 6.90875244140625,
      "learning_rate": 4.48963519387741e-06,
      "logits/chosen": 1.0072237253189087,
      "logits/rejected": 0.9182701110839844,
      "logps/chosen": -202.2296905517578,
      "logps/rejected": -228.3877410888672,
      "loss": 0.5129,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5820229649543762,
      "rewards/margins": 0.7529212832450867,
      "rewards/rejected": -1.334944248199463,
      "step": 3580
    },
    {
      "epoch": 2.872,
      "grad_norm": 7.405145645141602,
      "learning_rate": 4.485400334630511e-06,
      "logits/chosen": 1.0315786600112915,
      "logits/rejected": 1.0412896871566772,
      "logps/chosen": -208.01670837402344,
      "logps/rejected": -233.53164672851562,
      "loss": 0.4206,
      "rewards/accuracies": 0.8375000357627869,
      "rewards/chosen": -0.788216233253479,
      "rewards/margins": 1.1600385904312134,
      "rewards/rejected": -1.9482548236846924,
      "step": 3590
    },
    {
      "epoch": 2.88,
      "grad_norm": 8.326019287109375,
      "learning_rate": 4.4811499928329775e-06,
      "logits/chosen": 0.8572309613227844,
      "logits/rejected": 0.8392788171768188,
      "logps/chosen": -199.2486572265625,
      "logps/rejected": -243.4100799560547,
      "loss": 0.5167,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.2861894369125366,
      "rewards/margins": 0.9100348353385925,
      "rewards/rejected": -2.1962242126464844,
      "step": 3600
    },
    {
      "epoch": 2.888,
      "grad_norm": 5.889350414276123,
      "learning_rate": 4.4768842016298275e-06,
      "logits/chosen": 1.1370327472686768,
      "logits/rejected": 0.966783344745636,
      "logps/chosen": -203.1497039794922,
      "logps/rejected": -216.1260223388672,
      "loss": 0.5539,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.37005722522735596,
      "rewards/margins": 0.8371629118919373,
      "rewards/rejected": -1.2072200775146484,
      "step": 3610
    },
    {
      "epoch": 2.896,
      "grad_norm": 5.19056510925293,
      "learning_rate": 4.472602994286559e-06,
      "logits/chosen": 1.003334641456604,
      "logits/rejected": 0.9512959718704224,
      "logps/chosen": -205.01858520507812,
      "logps/rejected": -227.35317993164062,
      "loss": 0.629,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.9021452069282532,
      "rewards/margins": 0.6625142097473145,
      "rewards/rejected": -1.5646594762802124,
      "step": 3620
    },
    {
      "epoch": 2.904,
      "grad_norm": 8.15114974975586,
      "learning_rate": 4.468306404188887e-06,
      "logits/chosen": 0.9551017880439758,
      "logits/rejected": 0.9154602289199829,
      "logps/chosen": -198.5496368408203,
      "logps/rejected": -231.066650390625,
      "loss": 0.5031,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.8837252855300903,
      "rewards/margins": 0.8877946734428406,
      "rewards/rejected": -1.7715200185775757,
      "step": 3630
    },
    {
      "epoch": 2.912,
      "grad_norm": 6.599959373474121,
      "learning_rate": 4.463994464842485e-06,
      "logits/chosen": 1.1515620946884155,
      "logits/rejected": 1.0820724964141846,
      "logps/chosen": -196.33172607421875,
      "logps/rejected": -196.94976806640625,
      "loss": 0.5572,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.42906999588012695,
      "rewards/margins": 0.6470549702644348,
      "rewards/rejected": -1.0761250257492065,
      "step": 3640
    },
    {
      "epoch": 2.92,
      "grad_norm": 11.319299697875977,
      "learning_rate": 4.45966720987272e-06,
      "logits/chosen": 0.982757031917572,
      "logits/rejected": 0.9923986792564392,
      "logps/chosen": -221.4713134765625,
      "logps/rejected": -228.6186065673828,
      "loss": 0.5141,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.6713235974311829,
      "rewards/margins": 0.912000834941864,
      "rewards/rejected": -1.5833243131637573,
      "step": 3650
    },
    {
      "epoch": 2.928,
      "grad_norm": 6.565591335296631,
      "learning_rate": 4.455324673024396e-06,
      "logits/chosen": 1.0351612567901611,
      "logits/rejected": 0.9620069861412048,
      "logps/chosen": -205.81106567382812,
      "logps/rejected": -224.9451446533203,
      "loss": 0.466,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.5482096076011658,
      "rewards/margins": 0.8861121535301208,
      "rewards/rejected": -1.434321641921997,
      "step": 3660
    },
    {
      "epoch": 2.936,
      "grad_norm": 11.906221389770508,
      "learning_rate": 4.45096688816149e-06,
      "logits/chosen": 0.9011246562004089,
      "logits/rejected": 0.9497923254966736,
      "logps/chosen": -201.8857421875,
      "logps/rejected": -207.8782501220703,
      "loss": 0.6158,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.971345067024231,
      "rewards/margins": 0.47949621081352234,
      "rewards/rejected": -1.4508411884307861,
      "step": 3670
    },
    {
      "epoch": 2.944,
      "grad_norm": 4.588380336761475,
      "learning_rate": 4.4465938892668815e-06,
      "logits/chosen": 1.0076476335525513,
      "logits/rejected": 1.000885009765625,
      "logps/chosen": -192.5887451171875,
      "logps/rejected": -214.83604431152344,
      "loss": 0.4861,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -0.916580319404602,
      "rewards/margins": 0.8697047233581543,
      "rewards/rejected": -1.7862850427627563,
      "step": 3680
    },
    {
      "epoch": 2.952,
      "grad_norm": 7.142757415771484,
      "learning_rate": 4.442205710442095e-06,
      "logits/chosen": 1.0882282257080078,
      "logits/rejected": 1.0283339023590088,
      "logps/chosen": -202.76055908203125,
      "logps/rejected": -213.7969512939453,
      "loss": 0.5747,
      "rewards/accuracies": 0.7125000357627869,
      "rewards/chosen": -1.2140840291976929,
      "rewards/margins": 0.6454777121543884,
      "rewards/rejected": -1.8595619201660156,
      "step": 3690
    },
    {
      "epoch": 2.96,
      "grad_norm": 7.612401962280273,
      "learning_rate": 4.43780238590703e-06,
      "logits/chosen": 1.0234251022338867,
      "logits/rejected": 1.0618809461593628,
      "logps/chosen": -208.89512634277344,
      "logps/rejected": -240.69761657714844,
      "loss": 0.5233,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4141120910644531,
      "rewards/margins": 1.018404245376587,
      "rewards/rejected": -2.432516574859619,
      "step": 3700
    },
    {
      "epoch": 2.968,
      "grad_norm": 7.064319133758545,
      "learning_rate": 4.433383949999695e-06,
      "logits/chosen": 0.9859750866889954,
      "logits/rejected": 0.86090087890625,
      "logps/chosen": -195.9196319580078,
      "logps/rejected": -218.8886260986328,
      "loss": 0.4801,
      "rewards/accuracies": 0.8375000357627869,
      "rewards/chosen": -1.1885597705841064,
      "rewards/margins": 0.8579304814338684,
      "rewards/rejected": -2.04649019241333,
      "step": 3710
    },
    {
      "epoch": 2.976,
      "grad_norm": 6.925068378448486,
      "learning_rate": 4.428950437175944e-06,
      "logits/chosen": 0.9019951224327087,
      "logits/rejected": 0.8458378911018372,
      "logps/chosen": -232.447265625,
      "logps/rejected": -239.73648071289062,
      "loss": 0.5006,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.1259241104125977,
      "rewards/margins": 0.8236715197563171,
      "rewards/rejected": -1.9495956897735596,
      "step": 3720
    },
    {
      "epoch": 2.984,
      "grad_norm": 9.741296768188477,
      "learning_rate": 4.4245018820091975e-06,
      "logits/chosen": 0.8749257326126099,
      "logits/rejected": 0.8770486116409302,
      "logps/chosen": -232.95010375976562,
      "logps/rejected": -250.5192413330078,
      "loss": 0.5261,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.2567474842071533,
      "rewards/margins": 1.0741055011749268,
      "rewards/rejected": -2.33085298538208,
      "step": 3730
    },
    {
      "epoch": 2.992,
      "grad_norm": 8.8729887008667,
      "learning_rate": 4.420038319190184e-06,
      "logits/chosen": 0.8761041760444641,
      "logits/rejected": 0.9457497000694275,
      "logps/chosen": -199.91307067871094,
      "logps/rejected": -205.45762634277344,
      "loss": 0.4907,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.8159521222114563,
      "rewards/margins": 0.788399338722229,
      "rewards/rejected": -1.6043514013290405,
      "step": 3740
    },
    {
      "epoch": 3.0,
      "grad_norm": 6.5494585037231445,
      "learning_rate": 4.415559783526661e-06,
      "logits/chosen": 0.9953758120536804,
      "logits/rejected": 0.9608160257339478,
      "logps/chosen": -203.28005981445312,
      "logps/rejected": -234.4387664794922,
      "loss": 0.5245,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.4376920461654663,
      "rewards/margins": 0.7289747595787048,
      "rewards/rejected": -2.1666667461395264,
      "step": 3750
    },
    {
      "epoch": 3.008,
      "grad_norm": 7.342889785766602,
      "learning_rate": 4.411066309943151e-06,
      "logits/chosen": 0.911847710609436,
      "logits/rejected": 0.8547344207763672,
      "logps/chosen": -229.86656188964844,
      "logps/rejected": -246.57359313964844,
      "loss": 0.39,
      "rewards/accuracies": 0.8375000357627869,
      "rewards/chosen": -0.783231258392334,
      "rewards/margins": 1.1452419757843018,
      "rewards/rejected": -1.9284733533859253,
      "step": 3760
    },
    {
      "epoch": 3.016,
      "grad_norm": 6.122635364532471,
      "learning_rate": 4.406557933480665e-06,
      "logits/chosen": 0.8730871081352234,
      "logits/rejected": 0.7891945242881775,
      "logps/chosen": -230.88916015625,
      "logps/rejected": -229.0500946044922,
      "loss": 0.4251,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.818146824836731,
      "rewards/margins": 1.2087650299072266,
      "rewards/rejected": -2.026911973953247,
      "step": 3770
    },
    {
      "epoch": 3.024,
      "grad_norm": 7.4118452072143555,
      "learning_rate": 4.402034689296425e-06,
      "logits/chosen": 0.9826356768608093,
      "logits/rejected": 0.8032142519950867,
      "logps/chosen": -197.12989807128906,
      "logps/rejected": -208.55838012695312,
      "loss": 0.3441,
      "rewards/accuracies": 0.9000000357627869,
      "rewards/chosen": -0.5323260426521301,
      "rewards/margins": 1.3038690090179443,
      "rewards/rejected": -1.8361949920654297,
      "step": 3780
    },
    {
      "epoch": 3.032,
      "grad_norm": 9.49242877960205,
      "learning_rate": 4.397496612663599e-06,
      "logits/chosen": 1.0212730169296265,
      "logits/rejected": 0.8553698658943176,
      "logps/chosen": -213.5210418701172,
      "logps/rejected": -232.46238708496094,
      "loss": 0.427,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.1634958982467651,
      "rewards/margins": 1.099713921546936,
      "rewards/rejected": -2.263209819793701,
      "step": 3790
    },
    {
      "epoch": 3.04,
      "grad_norm": 7.7141571044921875,
      "learning_rate": 4.392943738971021e-06,
      "logits/chosen": 0.9747546315193176,
      "logits/rejected": 0.974277675151825,
      "logps/chosen": -211.902099609375,
      "logps/rejected": -224.82958984375,
      "loss": 0.3788,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.9616455435752869,
      "rewards/margins": 1.1425886154174805,
      "rewards/rejected": -2.104234457015991,
      "step": 3800
    },
    {
      "epoch": 3.048,
      "grad_norm": 6.494782447814941,
      "learning_rate": 4.388376103722914e-06,
      "logits/chosen": 0.9183564186096191,
      "logits/rejected": 0.8766688704490662,
      "logps/chosen": -192.46058654785156,
      "logps/rejected": -252.86399841308594,
      "loss": 0.4398,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.0373767614364624,
      "rewards/margins": 1.24557363986969,
      "rewards/rejected": -2.2829504013061523,
      "step": 3810
    },
    {
      "epoch": 3.056,
      "grad_norm": 12.855714797973633,
      "learning_rate": 4.383793742538615e-06,
      "logits/chosen": 0.9224297404289246,
      "logits/rejected": 0.8424360156059265,
      "logps/chosen": -188.82408142089844,
      "logps/rejected": -220.83970642089844,
      "loss": 0.5444,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5318169593811035,
      "rewards/margins": 0.8576032519340515,
      "rewards/rejected": -1.3894203901290894,
      "step": 3820
    },
    {
      "epoch": 3.064,
      "grad_norm": 6.39134407043457,
      "learning_rate": 4.3791966911522985e-06,
      "logits/chosen": 0.9185449481010437,
      "logits/rejected": 0.9570236206054688,
      "logps/chosen": -198.7986602783203,
      "logps/rejected": -196.16566467285156,
      "loss": 0.5775,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.8139546513557434,
      "rewards/margins": 0.7184473872184753,
      "rewards/rejected": -1.5324020385742188,
      "step": 3830
    },
    {
      "epoch": 3.072,
      "grad_norm": 7.5484185218811035,
      "learning_rate": 4.374584985412692e-06,
      "logits/chosen": 1.0025938749313354,
      "logits/rejected": 0.965260922908783,
      "logps/chosen": -184.16038513183594,
      "logps/rejected": -217.509765625,
      "loss": 0.4271,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -0.8147037625312805,
      "rewards/margins": 1.0747199058532715,
      "rewards/rejected": -1.8894237279891968,
      "step": 3840
    },
    {
      "epoch": 3.08,
      "grad_norm": 10.4950532913208,
      "learning_rate": 4.369958661282805e-06,
      "logits/chosen": 0.9348158240318298,
      "logits/rejected": 0.8544149398803711,
      "logps/chosen": -199.47950744628906,
      "logps/rejected": -233.7744598388672,
      "loss": 0.4727,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -0.8798511624336243,
      "rewards/margins": 1.0782015323638916,
      "rewards/rejected": -1.958052635192871,
      "step": 3850
    },
    {
      "epoch": 3.088,
      "grad_norm": 10.140469551086426,
      "learning_rate": 4.365317754839643e-06,
      "logits/chosen": 0.8552842140197754,
      "logits/rejected": 0.8293384909629822,
      "logps/chosen": -229.6714324951172,
      "logps/rejected": -237.9065399169922,
      "loss": 0.4637,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.158323884010315,
      "rewards/margins": 1.1132620573043823,
      "rewards/rejected": -2.2715859413146973,
      "step": 3860
    },
    {
      "epoch": 3.096,
      "grad_norm": 10.660223960876465,
      "learning_rate": 4.360662302273926e-06,
      "logits/chosen": 0.9855637550354004,
      "logits/rejected": 0.8762430548667908,
      "logps/chosen": -197.29660034179688,
      "logps/rejected": -210.25186157226562,
      "loss": 0.4684,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.9172877669334412,
      "rewards/margins": 0.9678840637207031,
      "rewards/rejected": -1.885171890258789,
      "step": 3870
    },
    {
      "epoch": 3.104,
      "grad_norm": 7.666329383850098,
      "learning_rate": 4.355992339889806e-06,
      "logits/chosen": 0.946201741695404,
      "logits/rejected": 0.8815839886665344,
      "logps/chosen": -197.9135284423828,
      "logps/rejected": -215.1569366455078,
      "loss": 0.4383,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.3328570127487183,
      "rewards/margins": 1.0127038955688477,
      "rewards/rejected": -2.3455610275268555,
      "step": 3880
    },
    {
      "epoch": 3.112,
      "grad_norm": 14.594318389892578,
      "learning_rate": 4.3513079041045925e-06,
      "logits/chosen": 1.000898003578186,
      "logits/rejected": 1.0313762426376343,
      "logps/chosen": -207.04624938964844,
      "logps/rejected": -221.084228515625,
      "loss": 0.4328,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.088545560836792,
      "rewards/margins": 0.945988118648529,
      "rewards/rejected": -2.0345335006713867,
      "step": 3890
    },
    {
      "epoch": 3.12,
      "grad_norm": 7.375593185424805,
      "learning_rate": 4.3466090314484526e-06,
      "logits/chosen": 0.8702503442764282,
      "logits/rejected": 0.9601964950561523,
      "logps/chosen": -216.6647491455078,
      "logps/rejected": -237.7848663330078,
      "loss": 0.429,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -1.0842136144638062,
      "rewards/margins": 1.1683653593063354,
      "rewards/rejected": -2.2525792121887207,
      "step": 3900
    },
    {
      "epoch": 3.128,
      "grad_norm": 5.803628921508789,
      "learning_rate": 4.341895758564141e-06,
      "logits/chosen": 0.9758888483047485,
      "logits/rejected": 0.9623239636421204,
      "logps/chosen": -195.2889404296875,
      "logps/rejected": -224.08889770507812,
      "loss": 0.4119,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.9340219497680664,
      "rewards/margins": 1.1046533584594727,
      "rewards/rejected": -2.038675308227539,
      "step": 3910
    },
    {
      "epoch": 3.136,
      "grad_norm": 13.526612281799316,
      "learning_rate": 4.3371681222067065e-06,
      "logits/chosen": 0.9235677719116211,
      "logits/rejected": 0.8304759860038757,
      "logps/chosen": -211.0590362548828,
      "logps/rejected": -219.9079132080078,
      "loss": 0.4242,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -0.7414934039115906,
      "rewards/margins": 1.2431910037994385,
      "rewards/rejected": -1.9846843481063843,
      "step": 3920
    },
    {
      "epoch": 3.144,
      "grad_norm": 7.47207498550415,
      "learning_rate": 4.332426159243206e-06,
      "logits/chosen": 0.9469828009605408,
      "logits/rejected": 0.8730935454368591,
      "logps/chosen": -202.57708740234375,
      "logps/rejected": -234.28994750976562,
      "loss": 0.3326,
      "rewards/accuracies": 0.9000000357627869,
      "rewards/chosen": -0.8033515214920044,
      "rewards/margins": 1.5122345685958862,
      "rewards/rejected": -2.3155860900878906,
      "step": 3930
    },
    {
      "epoch": 3.152,
      "grad_norm": 10.290471076965332,
      "learning_rate": 4.327669906652421e-06,
      "logits/chosen": 0.9347248077392578,
      "logits/rejected": 0.9168369174003601,
      "logps/chosen": -205.8367156982422,
      "logps/rejected": -241.17527770996094,
      "loss": 0.4048,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.078516960144043,
      "rewards/margins": 1.3800959587097168,
      "rewards/rejected": -2.4586129188537598,
      "step": 3940
    },
    {
      "epoch": 3.16,
      "grad_norm": 10.458824157714844,
      "learning_rate": 4.322899401524563e-06,
      "logits/chosen": 0.8786640167236328,
      "logits/rejected": 0.726817786693573,
      "logps/chosen": -206.63316345214844,
      "logps/rejected": -241.03616333007812,
      "loss": 0.417,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.183452844619751,
      "rewards/margins": 1.3412961959838867,
      "rewards/rejected": -2.5247490406036377,
      "step": 3950
    },
    {
      "epoch": 3.168,
      "grad_norm": 11.669900894165039,
      "learning_rate": 4.318114681060989e-06,
      "logits/chosen": 0.9407995343208313,
      "logits/rejected": 0.9570455551147461,
      "logps/chosen": -201.372802734375,
      "logps/rejected": -219.628662109375,
      "loss": 0.4584,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.7632571458816528,
      "rewards/margins": 1.0025835037231445,
      "rewards/rejected": -1.765840768814087,
      "step": 3960
    },
    {
      "epoch": 3.176,
      "grad_norm": 12.478621482849121,
      "learning_rate": 4.313315782573914e-06,
      "logits/chosen": 0.8294739127159119,
      "logits/rejected": 0.8217317461967468,
      "logps/chosen": -222.06484985351562,
      "logps/rejected": -248.57861328125,
      "loss": 0.474,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.5710614919662476,
      "rewards/margins": 1.08151113986969,
      "rewards/rejected": -2.6525726318359375,
      "step": 3970
    },
    {
      "epoch": 3.184,
      "grad_norm": 17.692319869995117,
      "learning_rate": 4.308502743486107e-06,
      "logits/chosen": 0.9584707617759705,
      "logits/rejected": 0.8594500422477722,
      "logps/chosen": -223.4303436279297,
      "logps/rejected": -239.1450653076172,
      "loss": 0.4356,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.8885783553123474,
      "rewards/margins": 1.3923364877700806,
      "rewards/rejected": -2.280914783477783,
      "step": 3980
    },
    {
      "epoch": 3.192,
      "grad_norm": 7.768544673919678,
      "learning_rate": 4.303675601330618e-06,
      "logits/chosen": 0.8310710191726685,
      "logits/rejected": 0.8502975702285767,
      "logps/chosen": -201.81480407714844,
      "logps/rejected": -245.815673828125,
      "loss": 0.3742,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9760616421699524,
      "rewards/margins": 1.6183117628097534,
      "rewards/rejected": -2.5943734645843506,
      "step": 3990
    },
    {
      "epoch": 3.2,
      "grad_norm": 11.38988208770752,
      "learning_rate": 4.298834393750469e-06,
      "logits/chosen": 0.9716154336929321,
      "logits/rejected": 0.9836306571960449,
      "logps/chosen": -210.5645294189453,
      "logps/rejected": -230.19566345214844,
      "loss": 0.4073,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -1.0508493185043335,
      "rewards/margins": 1.2152557373046875,
      "rewards/rejected": -2.2661049365997314,
      "step": 4000
    },
    {
      "epoch": 3.208,
      "grad_norm": 8.679203987121582,
      "learning_rate": 4.2939791584983695e-06,
      "logits/chosen": 0.9018089175224304,
      "logits/rejected": 0.8700457811355591,
      "logps/chosen": -199.3931427001953,
      "logps/rejected": -236.6988983154297,
      "loss": 0.3966,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.3499828577041626,
      "rewards/margins": 1.516105055809021,
      "rewards/rejected": -2.8660879135131836,
      "step": 4010
    },
    {
      "epoch": 3.216,
      "grad_norm": 6.8652777671813965,
      "learning_rate": 4.28910993343642e-06,
      "logits/chosen": 0.9662749171257019,
      "logits/rejected": 0.9868101477622986,
      "logps/chosen": -201.87808227539062,
      "logps/rejected": -243.437744140625,
      "loss": 0.3566,
      "rewards/accuracies": 0.8375000357627869,
      "rewards/chosen": -1.0853939056396484,
      "rewards/margins": 1.5348894596099854,
      "rewards/rejected": -2.620283365249634,
      "step": 4020
    },
    {
      "epoch": 3.224,
      "grad_norm": 10.237499237060547,
      "learning_rate": 4.284226756535814e-06,
      "logits/chosen": 0.899558961391449,
      "logits/rejected": 0.8322281241416931,
      "logps/chosen": -207.8240203857422,
      "logps/rejected": -255.903564453125,
      "loss": 0.42,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.0343433618545532,
      "rewards/margins": 1.4542030096054077,
      "rewards/rejected": -2.488546371459961,
      "step": 4030
    },
    {
      "epoch": 3.232,
      "grad_norm": 8.353890419006348,
      "learning_rate": 4.279329665876548e-06,
      "logits/chosen": 0.8721134066581726,
      "logits/rejected": 0.8629803657531738,
      "logps/chosen": -199.42417907714844,
      "logps/rejected": -235.1962432861328,
      "loss": 0.404,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0176727771759033,
      "rewards/margins": 1.266229510307312,
      "rewards/rejected": -2.283902406692505,
      "step": 4040
    },
    {
      "epoch": 3.24,
      "grad_norm": 8.403268814086914,
      "learning_rate": 4.274418699647117e-06,
      "logits/chosen": 0.9170069694519043,
      "logits/rejected": 0.836732029914856,
      "logps/chosen": -224.29885864257812,
      "logps/rejected": -224.2794189453125,
      "loss": 0.475,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -1.4468005895614624,
      "rewards/margins": 1.152286410331726,
      "rewards/rejected": -2.5990869998931885,
      "step": 4050
    },
    {
      "epoch": 3.248,
      "grad_norm": 13.75932502746582,
      "learning_rate": 4.269493896144224e-06,
      "logits/chosen": 0.809971809387207,
      "logits/rejected": 0.8430444598197937,
      "logps/chosen": -188.1841278076172,
      "logps/rejected": -231.8311767578125,
      "loss": 0.4046,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.8959644436836243,
      "rewards/margins": 1.3504829406738281,
      "rewards/rejected": -2.2464473247528076,
      "step": 4060
    },
    {
      "epoch": 3.2560000000000002,
      "grad_norm": 15.792646408081055,
      "learning_rate": 4.264555293772475e-06,
      "logits/chosen": 0.8807941675186157,
      "logits/rejected": 0.8839223980903625,
      "logps/chosen": -203.93402099609375,
      "logps/rejected": -227.60142517089844,
      "loss": 0.4881,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9412260055541992,
      "rewards/margins": 1.0077519416809082,
      "rewards/rejected": -1.9489777088165283,
      "step": 4070
    },
    {
      "epoch": 3.2640000000000002,
      "grad_norm": 13.880685806274414,
      "learning_rate": 4.2596029310440826e-06,
      "logits/chosen": 0.8665573000907898,
      "logits/rejected": 0.9192334413528442,
      "logps/chosen": -225.70999145507812,
      "logps/rejected": -229.2024383544922,
      "loss": 0.4918,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.1684324741363525,
      "rewards/margins": 1.0852488279342651,
      "rewards/rejected": -2.253680944442749,
      "step": 4080
    },
    {
      "epoch": 3.2720000000000002,
      "grad_norm": 12.61258602142334,
      "learning_rate": 4.254636846578567e-06,
      "logits/chosen": 0.8059272766113281,
      "logits/rejected": 0.7835027575492859,
      "logps/chosen": -214.0414581298828,
      "logps/rejected": -231.42832946777344,
      "loss": 0.4407,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.8260923624038696,
      "rewards/margins": 1.0982822179794312,
      "rewards/rejected": -1.9243745803833008,
      "step": 4090
    },
    {
      "epoch": 3.2800000000000002,
      "grad_norm": 14.153593063354492,
      "learning_rate": 4.249657079102452e-06,
      "logits/chosen": 0.9150283932685852,
      "logits/rejected": 0.7976348996162415,
      "logps/chosen": -203.6976776123047,
      "logps/rejected": -214.48025512695312,
      "loss": 0.4579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.8042591214179993,
      "rewards/margins": 1.0714467763900757,
      "rewards/rejected": -1.8757059574127197,
      "step": 4100
    },
    {
      "epoch": 3.288,
      "grad_norm": 12.573663711547852,
      "learning_rate": 4.244663667448965e-06,
      "logits/chosen": 0.9341398477554321,
      "logits/rejected": 0.8945412039756775,
      "logps/chosen": -218.30126953125,
      "logps/rejected": -234.64599609375,
      "loss": 0.462,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.2913658618927002,
      "rewards/margins": 1.1530601978302002,
      "rewards/rejected": -2.4444260597229004,
      "step": 4110
    },
    {
      "epoch": 3.296,
      "grad_norm": 9.752543449401855,
      "learning_rate": 4.239656650557733e-06,
      "logits/chosen": 0.9419944882392883,
      "logits/rejected": 0.9461938738822937,
      "logps/chosen": -205.2966766357422,
      "logps/rejected": -216.1801300048828,
      "loss": 0.4384,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -1.2133740186691284,
      "rewards/margins": 1.447343111038208,
      "rewards/rejected": -2.660717010498047,
      "step": 4120
    },
    {
      "epoch": 3.304,
      "grad_norm": 19.97236442565918,
      "learning_rate": 4.234636067474481e-06,
      "logits/chosen": 0.8762012720108032,
      "logits/rejected": 0.8730892539024353,
      "logps/chosen": -209.2789306640625,
      "logps/rejected": -225.71426391601562,
      "loss": 0.5437,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.9765220880508423,
      "rewards/margins": 0.9421109557151794,
      "rewards/rejected": -1.9186328649520874,
      "step": 4130
    },
    {
      "epoch": 3.312,
      "grad_norm": 7.913999080657959,
      "learning_rate": 4.229601957350722e-06,
      "logits/chosen": 0.8649017214775085,
      "logits/rejected": 0.7728471159934998,
      "logps/chosen": -236.4957733154297,
      "logps/rejected": -252.7061767578125,
      "loss": 0.4567,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.455244779586792,
      "rewards/margins": 1.1092432737350464,
      "rewards/rejected": -2.564488172531128,
      "step": 4140
    },
    {
      "epoch": 3.32,
      "grad_norm": 12.79985523223877,
      "learning_rate": 4.224554359443459e-06,
      "logits/chosen": 0.9708512425422668,
      "logits/rejected": 0.8335834741592407,
      "logps/chosen": -202.499755859375,
      "logps/rejected": -231.90538024902344,
      "loss": 0.4786,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.0350658893585205,
      "rewards/margins": 1.2157402038574219,
      "rewards/rejected": -2.2508060932159424,
      "step": 4150
    },
    {
      "epoch": 3.328,
      "grad_norm": 9.839788436889648,
      "learning_rate": 4.219493313114875e-06,
      "logits/chosen": 0.9519194960594177,
      "logits/rejected": 0.9104518294334412,
      "logps/chosen": -205.2983856201172,
      "logps/rejected": -224.93663024902344,
      "loss": 0.3966,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.8869916796684265,
      "rewards/margins": 1.4354019165039062,
      "rewards/rejected": -2.3223936557769775,
      "step": 4160
    },
    {
      "epoch": 3.336,
      "grad_norm": 12.0596284866333,
      "learning_rate": 4.214418857832025e-06,
      "logits/chosen": 0.8763806223869324,
      "logits/rejected": 0.8765867352485657,
      "logps/chosen": -222.2722625732422,
      "logps/rejected": -257.39453125,
      "loss": 0.4153,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -1.6189930438995361,
      "rewards/margins": 1.1976702213287354,
      "rewards/rejected": -2.8166635036468506,
      "step": 4170
    },
    {
      "epoch": 3.344,
      "grad_norm": 8.58606243133545,
      "learning_rate": 4.209331033166532e-06,
      "logits/chosen": 0.9239550828933716,
      "logits/rejected": 0.8988777995109558,
      "logps/chosen": -198.4449920654297,
      "logps/rejected": -243.26878356933594,
      "loss": 0.4353,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.5305871963500977,
      "rewards/margins": 1.0142645835876465,
      "rewards/rejected": -2.544851779937744,
      "step": 4180
    },
    {
      "epoch": 3.352,
      "grad_norm": 15.175363540649414,
      "learning_rate": 4.2042298787942735e-06,
      "logits/chosen": 0.843410313129425,
      "logits/rejected": 0.8987080454826355,
      "logps/chosen": -211.9822235107422,
      "logps/rejected": -244.9770965576172,
      "loss": 0.4034,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.0648707151412964,
      "rewards/margins": 1.016554355621338,
      "rewards/rejected": -2.081425189971924,
      "step": 4190
    },
    {
      "epoch": 3.36,
      "grad_norm": 8.164642333984375,
      "learning_rate": 4.1991154344950755e-06,
      "logits/chosen": 0.9261566996574402,
      "logits/rejected": 0.897598385810852,
      "logps/chosen": -202.40428161621094,
      "logps/rejected": -241.23191833496094,
      "loss": 0.3954,
      "rewards/accuracies": 0.8375000357627869,
      "rewards/chosen": -0.40160098671913147,
      "rewards/margins": 1.1215957403182983,
      "rewards/rejected": -1.5231966972351074,
      "step": 4200
    },
    {
      "epoch": 3.368,
      "grad_norm": 10.852888107299805,
      "learning_rate": 4.193987740152404e-06,
      "logits/chosen": 0.955554187297821,
      "logits/rejected": 0.9004291892051697,
      "logps/chosen": -184.4699249267578,
      "logps/rejected": -220.79690551757812,
      "loss": 0.3409,
      "rewards/accuracies": 0.9000000357627869,
      "rewards/chosen": -0.6415241956710815,
      "rewards/margins": 1.315872073173523,
      "rewards/rejected": -1.957396388053894,
      "step": 4210
    },
    {
      "epoch": 3.376,
      "grad_norm": 18.212907791137695,
      "learning_rate": 4.188846835753047e-06,
      "logits/chosen": 0.8005386590957642,
      "logits/rejected": 0.8098655939102173,
      "logps/chosen": -215.5693359375,
      "logps/rejected": -231.4944610595703,
      "loss": 0.3963,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.9665080308914185,
      "rewards/margins": 1.293958306312561,
      "rewards/rejected": -2.2604663372039795,
      "step": 4220
    },
    {
      "epoch": 3.384,
      "grad_norm": 10.81216812133789,
      "learning_rate": 4.183692761386813e-06,
      "logits/chosen": 0.9972082376480103,
      "logits/rejected": 0.9668437242507935,
      "logps/chosen": -207.8662872314453,
      "logps/rejected": -236.48428344726562,
      "loss": 0.4591,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5786136984825134,
      "rewards/margins": 1.239148497581482,
      "rewards/rejected": -1.8177622556686401,
      "step": 4230
    },
    {
      "epoch": 3.392,
      "grad_norm": 12.289898872375488,
      "learning_rate": 4.178525557246207e-06,
      "logits/chosen": 0.8920055627822876,
      "logits/rejected": 0.8146273493766785,
      "logps/chosen": -197.36410522460938,
      "logps/rejected": -226.280029296875,
      "loss": 0.4908,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0728448629379272,
      "rewards/margins": 1.341575264930725,
      "rewards/rejected": -2.4144201278686523,
      "step": 4240
    },
    {
      "epoch": 3.4,
      "grad_norm": 10.308980941772461,
      "learning_rate": 4.173345263626125e-06,
      "logits/chosen": 0.9153737425804138,
      "logits/rejected": 0.9260154962539673,
      "logps/chosen": -215.0824737548828,
      "logps/rejected": -243.21629333496094,
      "loss": 0.479,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.110467791557312,
      "rewards/margins": 1.1035948991775513,
      "rewards/rejected": -2.214062452316284,
      "step": 4250
    },
    {
      "epoch": 3.408,
      "grad_norm": 9.071174621582031,
      "learning_rate": 4.168151920923536e-06,
      "logits/chosen": 0.8960357904434204,
      "logits/rejected": 0.8135896921157837,
      "logps/chosen": -216.384765625,
      "logps/rejected": -231.6706085205078,
      "loss": 0.4235,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.950639545917511,
      "rewards/margins": 1.0974748134613037,
      "rewards/rejected": -2.048114538192749,
      "step": 4260
    },
    {
      "epoch": 3.416,
      "grad_norm": 7.223389148712158,
      "learning_rate": 4.162945569637174e-06,
      "logits/chosen": 0.7265322804450989,
      "logits/rejected": 0.7518787384033203,
      "logps/chosen": -207.7421112060547,
      "logps/rejected": -234.2729034423828,
      "loss": 0.4978,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.4523930549621582,
      "rewards/margins": 1.2414056062698364,
      "rewards/rejected": -2.693798542022705,
      "step": 4270
    },
    {
      "epoch": 3.424,
      "grad_norm": 12.135030746459961,
      "learning_rate": 4.157726250367208e-06,
      "logits/chosen": 0.8626713752746582,
      "logits/rejected": 0.8069788217544556,
      "logps/chosen": -228.7198944091797,
      "logps/rejected": -217.19143676757812,
      "loss": 0.412,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -1.6234177350997925,
      "rewards/margins": 1.184737205505371,
      "rewards/rejected": -2.808155059814453,
      "step": 4280
    },
    {
      "epoch": 3.432,
      "grad_norm": 10.609066009521484,
      "learning_rate": 4.152494003814939e-06,
      "logits/chosen": 0.93280029296875,
      "logits/rejected": 0.8937816619873047,
      "logps/chosen": -212.73155212402344,
      "logps/rejected": -226.97402954101562,
      "loss": 0.5399,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.471534013748169,
      "rewards/margins": 0.9502825736999512,
      "rewards/rejected": -2.42181658744812,
      "step": 4290
    },
    {
      "epoch": 3.44,
      "grad_norm": 12.865897178649902,
      "learning_rate": 4.147248870782477e-06,
      "logits/chosen": 0.9401634335517883,
      "logits/rejected": 0.8570839166641235,
      "logps/chosen": -236.3987274169922,
      "logps/rejected": -248.67503356933594,
      "loss": 0.4249,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.5956907272338867,
      "rewards/margins": 1.3058620691299438,
      "rewards/rejected": -2.901552677154541,
      "step": 4300
    },
    {
      "epoch": 3.448,
      "grad_norm": 10.754268646240234,
      "learning_rate": 4.141990892172424e-06,
      "logits/chosen": 0.9375637173652649,
      "logits/rejected": 0.8381093144416809,
      "logps/chosen": -232.57603454589844,
      "logps/rejected": -257.6446838378906,
      "loss": 0.4866,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.8424854278564453,
      "rewards/margins": 1.1537357568740845,
      "rewards/rejected": -2.9962213039398193,
      "step": 4310
    },
    {
      "epoch": 3.456,
      "grad_norm": 5.4626054763793945,
      "learning_rate": 4.136720108987552e-06,
      "logits/chosen": 1.1030884981155396,
      "logits/rejected": 0.9064838290214539,
      "logps/chosen": -203.1874542236328,
      "logps/rejected": -221.27049255371094,
      "loss": 0.3856,
      "rewards/accuracies": 0.8375000357627869,
      "rewards/chosen": -1.246989130973816,
      "rewards/margins": 1.3378962278366089,
      "rewards/rejected": -2.584885358810425,
      "step": 4320
    },
    {
      "epoch": 3.464,
      "grad_norm": 12.412184715270996,
      "learning_rate": 4.131436562330488e-06,
      "logits/chosen": 0.9628559350967407,
      "logits/rejected": 0.9677176475524902,
      "logps/chosen": -211.6782684326172,
      "logps/rejected": -234.12564086914062,
      "loss": 0.4181,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.5418589115142822,
      "rewards/margins": 1.1167868375778198,
      "rewards/rejected": -2.6586456298828125,
      "step": 4330
    },
    {
      "epoch": 3.472,
      "grad_norm": 15.786921501159668,
      "learning_rate": 4.126140293403389e-06,
      "logits/chosen": 0.98765629529953,
      "logits/rejected": 0.9222535490989685,
      "logps/chosen": -210.3227081298828,
      "logps/rejected": -228.1774444580078,
      "loss": 0.4448,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -1.6453129053115845,
      "rewards/margins": 1.0853195190429688,
      "rewards/rejected": -2.7306325435638428,
      "step": 4340
    },
    {
      "epoch": 3.48,
      "grad_norm": 10.291638374328613,
      "learning_rate": 4.1208313435076255e-06,
      "logits/chosen": 0.9463958740234375,
      "logits/rejected": 0.9481539130210876,
      "logps/chosen": -212.87197875976562,
      "logps/rejected": -221.46165466308594,
      "loss": 0.5025,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.4104286432266235,
      "rewards/margins": 1.028719425201416,
      "rewards/rejected": -2.43914794921875,
      "step": 4350
    },
    {
      "epoch": 3.488,
      "grad_norm": 18.95135498046875,
      "learning_rate": 4.115509754043454e-06,
      "logits/chosen": 0.9782568216323853,
      "logits/rejected": 0.9336418509483337,
      "logps/chosen": -213.4448699951172,
      "logps/rejected": -235.00161743164062,
      "loss": 0.4065,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.11874504387378693,
      "rewards/margins": 1.4726828336715698,
      "rewards/rejected": -1.5914280414581299,
      "step": 4360
    },
    {
      "epoch": 3.496,
      "grad_norm": 15.36201000213623,
      "learning_rate": 4.1101755665097e-06,
      "logits/chosen": 1.0215367078781128,
      "logits/rejected": 0.9443756937980652,
      "logps/chosen": -210.01756286621094,
      "logps/rejected": -214.49449157714844,
      "loss": 0.4452,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.8719436526298523,
      "rewards/margins": 1.059888482093811,
      "rewards/rejected": -1.931832194328308,
      "step": 4370
    },
    {
      "epoch": 3.504,
      "grad_norm": 14.627403259277344,
      "learning_rate": 4.104828822503427e-06,
      "logits/chosen": 1.0366023778915405,
      "logits/rejected": 0.9075769782066345,
      "logps/chosen": -215.0442657470703,
      "logps/rejected": -228.04214477539062,
      "loss": 0.4347,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.925746738910675,
      "rewards/margins": 1.1247776746749878,
      "rewards/rejected": -2.0505244731903076,
      "step": 4380
    },
    {
      "epoch": 3.512,
      "grad_norm": 12.368705749511719,
      "learning_rate": 4.09946956371962e-06,
      "logits/chosen": 0.8762325644493103,
      "logits/rejected": 0.9048509001731873,
      "logps/chosen": -215.4369659423828,
      "logps/rejected": -259.8838806152344,
      "loss": 0.4074,
      "rewards/accuracies": 0.8375000357627869,
      "rewards/chosen": -1.2534805536270142,
      "rewards/margins": 1.3236781358718872,
      "rewards/rejected": -2.5771586894989014,
      "step": 4390
    },
    {
      "epoch": 3.52,
      "grad_norm": 11.062219619750977,
      "learning_rate": 4.094097831950855e-06,
      "logits/chosen": 0.9681398272514343,
      "logits/rejected": 0.8761087656021118,
      "logps/chosen": -194.627685546875,
      "logps/rejected": -231.01333618164062,
      "loss": 0.4311,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.7227877974510193,
      "rewards/margins": 1.3572759628295898,
      "rewards/rejected": -2.080063819885254,
      "step": 4400
    },
    {
      "epoch": 3.528,
      "grad_norm": 13.982263565063477,
      "learning_rate": 4.0887136690869774e-06,
      "logits/chosen": 0.9537173509597778,
      "logits/rejected": 0.9426767230033875,
      "logps/chosen": -201.9144287109375,
      "logps/rejected": -229.7262420654297,
      "loss": 0.3984,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.8084962964057922,
      "rewards/margins": 1.3087875843048096,
      "rewards/rejected": -2.117284059524536,
      "step": 4410
    },
    {
      "epoch": 3.536,
      "grad_norm": 12.993025779724121,
      "learning_rate": 4.0833171171147675e-06,
      "logits/chosen": 0.8449978828430176,
      "logits/rejected": 0.8342208862304688,
      "logps/chosen": -216.42979431152344,
      "logps/rejected": -257.1995849609375,
      "loss": 0.3506,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.0627737045288086,
      "rewards/margins": 1.590333342552185,
      "rewards/rejected": -2.653106927871704,
      "step": 4420
    },
    {
      "epoch": 3.544,
      "grad_norm": 16.3622989654541,
      "learning_rate": 4.077908218117625e-06,
      "logits/chosen": 1.0056699514389038,
      "logits/rejected": 0.9356996417045593,
      "logps/chosen": -223.9595947265625,
      "logps/rejected": -241.6196746826172,
      "loss": 0.3655,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.1103066205978394,
      "rewards/margins": 1.436450481414795,
      "rewards/rejected": -2.546757459640503,
      "step": 4430
    },
    {
      "epoch": 3.552,
      "grad_norm": 4.724842548370361,
      "learning_rate": 4.072487014275228e-06,
      "logits/chosen": 0.9575614929199219,
      "logits/rejected": 0.7959676384925842,
      "logps/chosen": -206.49546813964844,
      "logps/rejected": -259.52972412109375,
      "loss": 0.4052,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.4658904075622559,
      "rewards/margins": 1.5798839330673218,
      "rewards/rejected": -3.045774221420288,
      "step": 4440
    },
    {
      "epoch": 3.56,
      "grad_norm": 11.298309326171875,
      "learning_rate": 4.067053547863215e-06,
      "logits/chosen": 0.9113892912864685,
      "logits/rejected": 0.837977409362793,
      "logps/chosen": -218.4698486328125,
      "logps/rejected": -254.52427673339844,
      "loss": 0.4225,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.5362995862960815,
      "rewards/margins": 1.178178071975708,
      "rewards/rejected": -2.7144775390625,
      "step": 4450
    },
    {
      "epoch": 3.568,
      "grad_norm": 6.332326889038086,
      "learning_rate": 4.061607861252848e-06,
      "logits/chosen": 0.870576024055481,
      "logits/rejected": 0.8328754305839539,
      "logps/chosen": -217.90184020996094,
      "logps/rejected": -243.92466735839844,
      "loss": 0.4866,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.8576223850250244,
      "rewards/margins": 1.264044165611267,
      "rewards/rejected": -3.121666669845581,
      "step": 4460
    },
    {
      "epoch": 3.576,
      "grad_norm": 19.035297393798828,
      "learning_rate": 4.056149996910683e-06,
      "logits/chosen": 0.8594030737876892,
      "logits/rejected": 0.7456435561180115,
      "logps/chosen": -213.825439453125,
      "logps/rejected": -235.2193145751953,
      "loss": 0.4798,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.9178558588027954,
      "rewards/margins": 1.1325775384902954,
      "rewards/rejected": -3.050433397293091,
      "step": 4470
    },
    {
      "epoch": 3.584,
      "grad_norm": 7.088034152984619,
      "learning_rate": 4.050679997398247e-06,
      "logits/chosen": 0.7786403894424438,
      "logits/rejected": 0.806538999080658,
      "logps/chosen": -202.5958709716797,
      "logps/rejected": -247.24156188964844,
      "loss": 0.4065,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.560505747795105,
      "rewards/margins": 1.615809440612793,
      "rewards/rejected": -3.1763153076171875,
      "step": 4480
    },
    {
      "epoch": 3.592,
      "grad_norm": 7.6185078620910645,
      "learning_rate": 4.045197905371691e-06,
      "logits/chosen": 0.8927198648452759,
      "logits/rejected": 0.8568658232688904,
      "logps/chosen": -200.9806671142578,
      "logps/rejected": -230.39976501464844,
      "loss": 0.4441,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.6182069778442383,
      "rewards/margins": 1.4453877210617065,
      "rewards/rejected": -3.0635945796966553,
      "step": 4490
    },
    {
      "epoch": 3.6,
      "grad_norm": 16.420970916748047,
      "learning_rate": 4.039703763581472e-06,
      "logits/chosen": 0.8568277359008789,
      "logits/rejected": 0.7848814725875854,
      "logps/chosen": -235.65513610839844,
      "logps/rejected": -240.30740356445312,
      "loss": 0.463,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -2.157593250274658,
      "rewards/margins": 1.1094763278961182,
      "rewards/rejected": -3.2670693397521973,
      "step": 4500
    },
    {
      "epoch": 3.608,
      "grad_norm": 9.267051696777344,
      "learning_rate": 4.03419761487201e-06,
      "logits/chosen": 1.004172921180725,
      "logits/rejected": 0.8293590545654297,
      "logps/chosen": -220.4943389892578,
      "logps/rejected": -238.46202087402344,
      "loss": 0.4328,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.6261404752731323,
      "rewards/margins": 1.2516275644302368,
      "rewards/rejected": -2.877768039703369,
      "step": 4510
    },
    {
      "epoch": 3.616,
      "grad_norm": 8.903371810913086,
      "learning_rate": 4.0286795021813595e-06,
      "logits/chosen": 0.7373332977294922,
      "logits/rejected": 0.7830902338027954,
      "logps/chosen": -206.95864868164062,
      "logps/rejected": -238.60684204101562,
      "loss": 0.4642,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.362304449081421,
      "rewards/margins": 1.3245913982391357,
      "rewards/rejected": -2.6868958473205566,
      "step": 4520
    },
    {
      "epoch": 3.624,
      "grad_norm": 9.160982131958008,
      "learning_rate": 4.023149468540871e-06,
      "logits/chosen": 0.8418847918510437,
      "logits/rejected": 0.8112041354179382,
      "logps/chosen": -207.1737060546875,
      "logps/rejected": -238.65518188476562,
      "loss": 0.5013,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.8721437454223633,
      "rewards/margins": 1.152505874633789,
      "rewards/rejected": -3.0246498584747314,
      "step": 4530
    },
    {
      "epoch": 3.632,
      "grad_norm": 8.608299255371094,
      "learning_rate": 4.0176075570748596e-06,
      "logits/chosen": 0.8552350997924805,
      "logits/rejected": 0.8494642376899719,
      "logps/chosen": -219.12474060058594,
      "logps/rejected": -222.3189239501953,
      "loss": 0.4387,
      "rewards/accuracies": 0.8375000357627869,
      "rewards/chosen": -1.2666515111923218,
      "rewards/margins": 1.1172685623168945,
      "rewards/rejected": -2.3839199542999268,
      "step": 4540
    },
    {
      "epoch": 3.64,
      "grad_norm": 11.497233390808105,
      "learning_rate": 4.012053811000262e-06,
      "logits/chosen": 0.919759213924408,
      "logits/rejected": 0.9661205410957336,
      "logps/chosen": -220.15342712402344,
      "logps/rejected": -245.29685974121094,
      "loss": 0.3889,
      "rewards/accuracies": 0.8375000357627869,
      "rewards/chosen": -1.2769368886947632,
      "rewards/margins": 1.3770891427993774,
      "rewards/rejected": -2.6540262699127197,
      "step": 4550
    },
    {
      "epoch": 3.648,
      "grad_norm": 12.795794486999512,
      "learning_rate": 4.006488273626307e-06,
      "logits/chosen": 0.8910170793533325,
      "logits/rejected": 0.8077157139778137,
      "logps/chosen": -208.62158203125,
      "logps/rejected": -234.9666290283203,
      "loss": 0.424,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.1060861349105835,
      "rewards/margins": 1.2906891107559204,
      "rewards/rejected": -2.396775007247925,
      "step": 4560
    },
    {
      "epoch": 3.656,
      "grad_norm": 19.536590576171875,
      "learning_rate": 4.000910988354172e-06,
      "logits/chosen": 0.9590219855308533,
      "logits/rejected": 0.9733330011367798,
      "logps/chosen": -199.6140899658203,
      "logps/rejected": -206.4640655517578,
      "loss": 0.4468,
      "rewards/accuracies": 0.8375000357627869,
      "rewards/chosen": -0.5966355204582214,
      "rewards/margins": 1.0923033952713013,
      "rewards/rejected": -1.6889389753341675,
      "step": 4570
    },
    {
      "epoch": 3.664,
      "grad_norm": 7.018489837646484,
      "learning_rate": 3.995321998676648e-06,
      "logits/chosen": 0.9202627539634705,
      "logits/rejected": 0.8594157099723816,
      "logps/chosen": -203.9952850341797,
      "logps/rejected": -229.83840942382812,
      "loss": 0.4649,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.984645664691925,
      "rewards/margins": 1.0351135730743408,
      "rewards/rejected": -2.0197594165802,
      "step": 4580
    },
    {
      "epoch": 3.672,
      "grad_norm": 10.823237419128418,
      "learning_rate": 3.989721348177801e-06,
      "logits/chosen": 0.9117226004600525,
      "logits/rejected": 0.8806440234184265,
      "logps/chosen": -222.4832763671875,
      "logps/rejected": -247.8332061767578,
      "loss": 0.3441,
      "rewards/accuracies": 0.9000000357627869,
      "rewards/chosen": -1.3591053485870361,
      "rewards/margins": 1.4962059259414673,
      "rewards/rejected": -2.855311155319214,
      "step": 4590
    },
    {
      "epoch": 3.68,
      "grad_norm": 14.88288402557373,
      "learning_rate": 3.984109080532627e-06,
      "logits/chosen": 0.9188446998596191,
      "logits/rejected": 0.8930045366287231,
      "logps/chosen": -190.447509765625,
      "logps/rejected": -212.60386657714844,
      "loss": 0.4098,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -1.0938667058944702,
      "rewards/margins": 1.1657882928848267,
      "rewards/rejected": -2.259654998779297,
      "step": 4600
    },
    {
      "epoch": 3.6879999999999997,
      "grad_norm": 15.50723934173584,
      "learning_rate": 3.978485239506717e-06,
      "logits/chosen": 0.8951994180679321,
      "logits/rejected": 0.9232072830200195,
      "logps/chosen": -222.2184295654297,
      "logps/rejected": -248.290771484375,
      "loss": 0.4084,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.0719726085662842,
      "rewards/margins": 1.3165521621704102,
      "rewards/rejected": -2.3885247707366943,
      "step": 4610
    },
    {
      "epoch": 3.6959999999999997,
      "grad_norm": 11.021881103515625,
      "learning_rate": 3.972849868955913e-06,
      "logits/chosen": 0.9283590316772461,
      "logits/rejected": 0.8612028360366821,
      "logps/chosen": -208.7410888671875,
      "logps/rejected": -236.5598907470703,
      "loss": 0.2934,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.6821409463882446,
      "rewards/margins": 1.696975588798523,
      "rewards/rejected": -2.3791165351867676,
      "step": 4620
    },
    {
      "epoch": 3.7039999999999997,
      "grad_norm": 7.1401777267456055,
      "learning_rate": 3.967203012825965e-06,
      "logits/chosen": 0.8836082816123962,
      "logits/rejected": 0.8645914196968079,
      "logps/chosen": -213.68142700195312,
      "logps/rejected": -251.0132293701172,
      "loss": 0.4014,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.4805501699447632,
      "rewards/margins": 1.6087449789047241,
      "rewards/rejected": -3.0892951488494873,
      "step": 4630
    },
    {
      "epoch": 3.7119999999999997,
      "grad_norm": 15.971772193908691,
      "learning_rate": 3.961544715152195e-06,
      "logits/chosen": 0.9219527244567871,
      "logits/rejected": 0.8754696249961853,
      "logps/chosen": -201.8592987060547,
      "logps/rejected": -226.51504516601562,
      "loss": 0.4237,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.1449387073516846,
      "rewards/margins": 1.4969040155410767,
      "rewards/rejected": -2.6418426036834717,
      "step": 4640
    },
    {
      "epoch": 3.7199999999999998,
      "grad_norm": 3.4029507637023926,
      "learning_rate": 3.955875020059141e-06,
      "logits/chosen": 0.9539863467216492,
      "logits/rejected": 0.9481542706489563,
      "logps/chosen": -190.33999633789062,
      "logps/rejected": -230.81553649902344,
      "loss": 0.4776,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.0588791370391846,
      "rewards/margins": 1.255807876586914,
      "rewards/rejected": -2.3146870136260986,
      "step": 4650
    },
    {
      "epoch": 3.7279999999999998,
      "grad_norm": 9.8549165725708,
      "learning_rate": 3.950193971760227e-06,
      "logits/chosen": 0.9642109274864197,
      "logits/rejected": 0.9821377992630005,
      "logps/chosen": -210.73709106445312,
      "logps/rejected": -215.7576904296875,
      "loss": 0.4352,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.1254199743270874,
      "rewards/margins": 1.1262983083724976,
      "rewards/rejected": -2.251718282699585,
      "step": 4660
    },
    {
      "epoch": 3.7359999999999998,
      "grad_norm": 13.556317329406738,
      "learning_rate": 3.944501614557408e-06,
      "logits/chosen": 1.0108891725540161,
      "logits/rejected": 0.903409481048584,
      "logps/chosen": -189.86575317382812,
      "logps/rejected": -216.7268524169922,
      "loss": 0.3932,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.1368598937988281,
      "rewards/margins": 1.453155517578125,
      "rewards/rejected": -2.590015172958374,
      "step": 4670
    },
    {
      "epoch": 3.7439999999999998,
      "grad_norm": 12.027454376220703,
      "learning_rate": 3.938797992840828e-06,
      "logits/chosen": 0.9133999943733215,
      "logits/rejected": 0.8714101910591125,
      "logps/chosen": -202.46926879882812,
      "logps/rejected": -219.1940460205078,
      "loss": 0.504,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.0332410335540771,
      "rewards/margins": 1.1814830303192139,
      "rewards/rejected": -2.214724063873291,
      "step": 4680
    },
    {
      "epoch": 3.752,
      "grad_norm": 7.450799465179443,
      "learning_rate": 3.933083151088476e-06,
      "logits/chosen": 0.9655442237854004,
      "logits/rejected": 0.9951929450035095,
      "logps/chosen": -216.5625762939453,
      "logps/rejected": -226.92529296875,
      "loss": 0.4698,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4813133478164673,
      "rewards/margins": 1.0279206037521362,
      "rewards/rejected": -2.5092339515686035,
      "step": 4690
    },
    {
      "epoch": 3.76,
      "grad_norm": 10.197044372558594,
      "learning_rate": 3.927357133865836e-06,
      "logits/chosen": 0.8014942407608032,
      "logits/rejected": 0.7166596055030823,
      "logps/chosen": -206.07383728027344,
      "logps/rejected": -233.2744598388672,
      "loss": 0.4263,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.1761497259140015,
      "rewards/margins": 1.4023005962371826,
      "rewards/rejected": -2.5784504413604736,
      "step": 4700
    },
    {
      "epoch": 3.768,
      "grad_norm": 12.484579086303711,
      "learning_rate": 3.92161998582554e-06,
      "logits/chosen": 0.9144058227539062,
      "logits/rejected": 0.8918857574462891,
      "logps/chosen": -210.94981384277344,
      "logps/rejected": -232.71836853027344,
      "loss": 0.4134,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.373250961303711,
      "rewards/margins": 1.1428226232528687,
      "rewards/rejected": -2.516073703765869,
      "step": 4710
    },
    {
      "epoch": 3.776,
      "grad_norm": 10.925464630126953,
      "learning_rate": 3.9158717517070216e-06,
      "logits/chosen": 0.9928507804870605,
      "logits/rejected": 0.9784805178642273,
      "logps/chosen": -184.71435546875,
      "logps/rejected": -216.792724609375,
      "loss": 0.3318,
      "rewards/accuracies": 0.8375000357627869,
      "rewards/chosen": -1.1705821752548218,
      "rewards/margins": 1.615303874015808,
      "rewards/rejected": -2.78588604927063,
      "step": 4720
    },
    {
      "epoch": 3.784,
      "grad_norm": 12.400118827819824,
      "learning_rate": 3.9101124763361645e-06,
      "logits/chosen": 0.9748711585998535,
      "logits/rejected": 0.9293093681335449,
      "logps/chosen": -217.0461883544922,
      "logps/rejected": -227.57286071777344,
      "loss": 0.4629,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.5834509134292603,
      "rewards/margins": 1.2847626209259033,
      "rewards/rejected": -2.868213415145874,
      "step": 4730
    },
    {
      "epoch": 3.792,
      "grad_norm": 11.517487525939941,
      "learning_rate": 3.904342204624955e-06,
      "logits/chosen": 0.9611164331436157,
      "logits/rejected": 0.9418201446533203,
      "logps/chosen": -217.9922637939453,
      "logps/rejected": -231.400634765625,
      "loss": 0.4182,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.1144764423370361,
      "rewards/margins": 1.261598825454712,
      "rewards/rejected": -2.376075029373169,
      "step": 4740
    },
    {
      "epoch": 3.8,
      "grad_norm": 18.023473739624023,
      "learning_rate": 3.8985609815711315e-06,
      "logits/chosen": 1.0194095373153687,
      "logits/rejected": 0.9130865931510925,
      "logps/chosen": -217.2231903076172,
      "logps/rejected": -231.2641143798828,
      "loss": 0.4554,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.4849942922592163,
      "rewards/margins": 1.1392329931259155,
      "rewards/rejected": -2.624227285385132,
      "step": 4750
    },
    {
      "epoch": 3.808,
      "grad_norm": 17.991451263427734,
      "learning_rate": 3.892768852257831e-06,
      "logits/chosen": 0.9631202816963196,
      "logits/rejected": 0.8651983141899109,
      "logps/chosen": -215.14443969726562,
      "logps/rejected": -251.19363403320312,
      "loss": 0.492,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.984946608543396,
      "rewards/margins": 1.1132451295852661,
      "rewards/rejected": -3.098191499710083,
      "step": 4760
    },
    {
      "epoch": 3.816,
      "grad_norm": 10.07773208618164,
      "learning_rate": 3.886965861853243e-06,
      "logits/chosen": 0.9928179979324341,
      "logits/rejected": 0.9808389544487,
      "logps/chosen": -209.1436309814453,
      "logps/rejected": -228.26318359375,
      "loss": 0.3544,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.264801263809204,
      "rewards/margins": 1.3731207847595215,
      "rewards/rejected": -2.6379220485687256,
      "step": 4770
    },
    {
      "epoch": 3.824,
      "grad_norm": 21.519670486450195,
      "learning_rate": 3.881152055610253e-06,
      "logits/chosen": 0.9541469812393188,
      "logits/rejected": 0.8962286114692688,
      "logps/chosen": -216.61123657226562,
      "logps/rejected": -220.80076599121094,
      "loss": 0.5935,
      "rewards/accuracies": 0.7125000357627869,
      "rewards/chosen": -1.6538161039352417,
      "rewards/margins": 0.6947936415672302,
      "rewards/rejected": -2.348609685897827,
      "step": 4780
    },
    {
      "epoch": 3.832,
      "grad_norm": 12.259000778198242,
      "learning_rate": 3.875327478866089e-06,
      "logits/chosen": 0.8635460734367371,
      "logits/rejected": 0.8118668794631958,
      "logps/chosen": -233.24827575683594,
      "logps/rejected": -245.0417022705078,
      "loss": 0.4181,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.2450588941574097,
      "rewards/margins": 1.115803599357605,
      "rewards/rejected": -2.3608624935150146,
      "step": 4790
    },
    {
      "epoch": 3.84,
      "grad_norm": 14.562280654907227,
      "learning_rate": 3.869492177041971e-06,
      "logits/chosen": 0.9073505401611328,
      "logits/rejected": 0.872600257396698,
      "logps/chosen": -231.16311645507812,
      "logps/rejected": -224.2497100830078,
      "loss": 0.5217,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.0939778089523315,
      "rewards/margins": 0.7748345732688904,
      "rewards/rejected": -1.8688122034072876,
      "step": 4800
    },
    {
      "epoch": 3.848,
      "grad_norm": 14.074769973754883,
      "learning_rate": 3.863646195642754e-06,
      "logits/chosen": 0.9319748282432556,
      "logits/rejected": 0.8828820586204529,
      "logps/chosen": -201.0325164794922,
      "logps/rejected": -209.9548797607422,
      "loss": 0.4524,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -0.7232056856155396,
      "rewards/margins": 1.1827844381332397,
      "rewards/rejected": -1.9059902429580688,
      "step": 4810
    },
    {
      "epoch": 3.856,
      "grad_norm": 13.172402381896973,
      "learning_rate": 3.857789580256576e-06,
      "logits/chosen": 0.8821210861206055,
      "logits/rejected": 0.8552846312522888,
      "logps/chosen": -204.34593200683594,
      "logps/rejected": -239.4946746826172,
      "loss": 0.5189,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.5751923322677612,
      "rewards/margins": 0.9332385063171387,
      "rewards/rejected": -2.5084309577941895,
      "step": 4820
    },
    {
      "epoch": 3.864,
      "grad_norm": 12.942428588867188,
      "learning_rate": 3.8519223765544985e-06,
      "logits/chosen": 0.9515409469604492,
      "logits/rejected": 0.8108540773391724,
      "logps/chosen": -196.3207550048828,
      "logps/rejected": -233.15110778808594,
      "loss": 0.3708,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.2026864290237427,
      "rewards/margins": 1.3849681615829468,
      "rewards/rejected": -2.5876548290252686,
      "step": 4830
    },
    {
      "epoch": 3.872,
      "grad_norm": 16.509870529174805,
      "learning_rate": 3.8460446302901575e-06,
      "logits/chosen": 0.9374640583992004,
      "logits/rejected": 0.8481482863426208,
      "logps/chosen": -200.82627868652344,
      "logps/rejected": -220.869873046875,
      "loss": 0.4426,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.4088255167007446,
      "rewards/margins": 1.1208895444869995,
      "rewards/rejected": -2.529715061187744,
      "step": 4840
    },
    {
      "epoch": 3.88,
      "grad_norm": 6.809768199920654,
      "learning_rate": 3.840156387299397e-06,
      "logits/chosen": 0.8348701596260071,
      "logits/rejected": 0.8732596635818481,
      "logps/chosen": -216.68138122558594,
      "logps/rejected": -247.988037109375,
      "loss": 0.4629,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.482513189315796,
      "rewards/margins": 1.2046746015548706,
      "rewards/rejected": -2.687187910079956,
      "step": 4850
    },
    {
      "epoch": 3.888,
      "grad_norm": 10.955533981323242,
      "learning_rate": 3.8342576934999184e-06,
      "logits/chosen": 0.8893832564353943,
      "logits/rejected": 0.8247093558311462,
      "logps/chosen": -204.13548278808594,
      "logps/rejected": -241.85671997070312,
      "loss": 0.4514,
      "rewards/accuracies": 0.8375000357627869,
      "rewards/chosen": -1.217315435409546,
      "rewards/margins": 1.1432523727416992,
      "rewards/rejected": -2.360567808151245,
      "step": 4860
    },
    {
      "epoch": 3.896,
      "grad_norm": 11.238479614257812,
      "learning_rate": 3.828348594890923e-06,
      "logits/chosen": 0.8213216662406921,
      "logits/rejected": 0.8109332919120789,
      "logps/chosen": -209.7275848388672,
      "logps/rejected": -236.1708526611328,
      "loss": 0.4234,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.2727683782577515,
      "rewards/margins": 1.410487174987793,
      "rewards/rejected": -2.683255910873413,
      "step": 4870
    },
    {
      "epoch": 3.904,
      "grad_norm": 12.693486213684082,
      "learning_rate": 3.822429137552747e-06,
      "logits/chosen": 0.9051896929740906,
      "logits/rejected": 0.8351686596870422,
      "logps/chosen": -212.6349639892578,
      "logps/rejected": -205.12376403808594,
      "loss": 0.5046,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.834870457649231,
      "rewards/margins": 0.9832606315612793,
      "rewards/rejected": -1.8181308507919312,
      "step": 4880
    },
    {
      "epoch": 3.912,
      "grad_norm": 6.716438293457031,
      "learning_rate": 3.816499367646508e-06,
      "logits/chosen": 1.0042909383773804,
      "logits/rejected": 0.9129732251167297,
      "logps/chosen": -212.87109375,
      "logps/rejected": -230.78567504882812,
      "loss": 0.535,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.2843660116195679,
      "rewards/margins": 1.0389277935028076,
      "rewards/rejected": -2.323293924331665,
      "step": 4890
    },
    {
      "epoch": 3.92,
      "grad_norm": 10.39694595336914,
      "learning_rate": 3.8105593314137434e-06,
      "logits/chosen": 0.9733989834785461,
      "logits/rejected": 0.9892801642417908,
      "logps/chosen": -209.4358367919922,
      "logps/rejected": -221.290771484375,
      "loss": 0.3994,
      "rewards/accuracies": 0.8375000357627869,
      "rewards/chosen": -1.1634297370910645,
      "rewards/margins": 1.246660590171814,
      "rewards/rejected": -2.410090208053589,
      "step": 4900
    },
    {
      "epoch": 3.928,
      "grad_norm": 16.674728393554688,
      "learning_rate": 3.804609075176049e-06,
      "logits/chosen": 0.7904636859893799,
      "logits/rejected": 0.7977269291877747,
      "logps/chosen": -202.6865692138672,
      "logps/rejected": -237.93357849121094,
      "loss": 0.4082,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.5116066932678223,
      "rewards/margins": 1.3030771017074585,
      "rewards/rejected": -2.8146839141845703,
      "step": 4910
    },
    {
      "epoch": 3.936,
      "grad_norm": 11.239212036132812,
      "learning_rate": 3.7986486453347183e-06,
      "logits/chosen": 0.9851727485656738,
      "logits/rejected": 0.8940200805664062,
      "logps/chosen": -206.59951782226562,
      "logps/rejected": -215.1295623779297,
      "loss": 0.4914,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.3703104257583618,
      "rewards/margins": 0.929050624370575,
      "rewards/rejected": -2.299360990524292,
      "step": 4920
    },
    {
      "epoch": 3.944,
      "grad_norm": 14.794110298156738,
      "learning_rate": 3.7926780883703794e-06,
      "logits/chosen": 0.9150724411010742,
      "logits/rejected": 0.8797799944877625,
      "logps/chosen": -209.21240234375,
      "logps/rejected": -228.32688903808594,
      "loss": 0.4379,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.8193817138671875,
      "rewards/margins": 1.1322726011276245,
      "rewards/rejected": -1.9516544342041016,
      "step": 4930
    },
    {
      "epoch": 3.952,
      "grad_norm": 9.821282386779785,
      "learning_rate": 3.7866974508426355e-06,
      "logits/chosen": 0.8630434274673462,
      "logits/rejected": 0.9651856422424316,
      "logps/chosen": -226.81321716308594,
      "logps/rejected": -220.50515747070312,
      "loss": 0.402,
      "rewards/accuracies": 0.8375000357627869,
      "rewards/chosen": -0.8245296478271484,
      "rewards/margins": 1.2236753702163696,
      "rewards/rejected": -2.0482048988342285,
      "step": 4940
    },
    {
      "epoch": 3.96,
      "grad_norm": 11.387616157531738,
      "learning_rate": 3.7807067793897006e-06,
      "logits/chosen": 0.8870008587837219,
      "logits/rejected": 0.8385466933250427,
      "logps/chosen": -230.2690887451172,
      "logps/rejected": -249.96641540527344,
      "loss": 0.5202,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.0625405311584473,
      "rewards/margins": 0.9815382361412048,
      "rewards/rejected": -2.044078826904297,
      "step": 4950
    },
    {
      "epoch": 3.968,
      "grad_norm": 10.633583068847656,
      "learning_rate": 3.7747061207280322e-06,
      "logits/chosen": 0.867588222026825,
      "logits/rejected": 0.78270423412323,
      "logps/chosen": -230.1920928955078,
      "logps/rejected": -261.4850158691406,
      "loss": 0.4374,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.1695181131362915,
      "rewards/margins": 1.3066359758377075,
      "rewards/rejected": -2.476154088973999,
      "step": 4960
    },
    {
      "epoch": 3.976,
      "grad_norm": 11.73788070678711,
      "learning_rate": 3.7686955216519733e-06,
      "logits/chosen": 0.8684186339378357,
      "logits/rejected": 0.9026303291320801,
      "logps/chosen": -202.72027587890625,
      "logps/rejected": -224.8450164794922,
      "loss": 0.3963,
      "rewards/accuracies": 0.8375000357627869,
      "rewards/chosen": -1.1008380651474,
      "rewards/margins": 1.233585238456726,
      "rewards/rejected": -2.334423065185547,
      "step": 4970
    },
    {
      "epoch": 3.984,
      "grad_norm": 12.70168685913086,
      "learning_rate": 3.7626750290333824e-06,
      "logits/chosen": 0.9102399945259094,
      "logits/rejected": 0.789924681186676,
      "logps/chosen": -217.3763885498047,
      "logps/rejected": -246.22720336914062,
      "loss": 0.431,
      "rewards/accuracies": 0.8375000357627869,
      "rewards/chosen": -1.2897388935089111,
      "rewards/margins": 1.2651302814483643,
      "rewards/rejected": -2.5548691749572754,
      "step": 4980
    },
    {
      "epoch": 3.992,
      "grad_norm": 10.833782196044922,
      "learning_rate": 3.7566446898212704e-06,
      "logits/chosen": 0.8599758148193359,
      "logits/rejected": 0.8142706751823425,
      "logps/chosen": -215.34092712402344,
      "logps/rejected": -204.10824584960938,
      "loss": 0.4499,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.1187313795089722,
      "rewards/margins": 1.1984655857086182,
      "rewards/rejected": -2.317196846008301,
      "step": 4990
    },
    {
      "epoch": 4.0,
      "grad_norm": 13.4774169921875,
      "learning_rate": 3.7506045510414337e-06,
      "logits/chosen": 1.0011329650878906,
      "logits/rejected": 0.9479683041572571,
      "logps/chosen": -199.9625701904297,
      "logps/rejected": -239.7847137451172,
      "loss": 0.3563,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0051459074020386,
      "rewards/margins": 1.3487907648086548,
      "rewards/rejected": -2.3539369106292725,
      "step": 5000
    },
    {
      "epoch": 4.008,
      "grad_norm": 8.692243576049805,
      "learning_rate": 3.7445546597960882e-06,
      "logits/chosen": 1.0498260259628296,
      "logits/rejected": 1.0109456777572632,
      "logps/chosen": -207.2587432861328,
      "logps/rejected": -230.1753692626953,
      "loss": 0.3541,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9930923581123352,
      "rewards/margins": 1.3766816854476929,
      "rewards/rejected": -2.3697738647460938,
      "step": 5010
    },
    {
      "epoch": 4.016,
      "grad_norm": 14.994426727294922,
      "learning_rate": 3.7384950632635e-06,
      "logits/chosen": 1.0372326374053955,
      "logits/rejected": 0.9997326135635376,
      "logps/chosen": -206.49755859375,
      "logps/rejected": -229.08706665039062,
      "loss": 0.4148,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.0184519290924072,
      "rewards/margins": 1.1713188886642456,
      "rewards/rejected": -2.1897706985473633,
      "step": 5020
    },
    {
      "epoch": 4.024,
      "grad_norm": 12.80656909942627,
      "learning_rate": 3.732425808697622e-06,
      "logits/chosen": 1.04145085811615,
      "logits/rejected": 0.8659443855285645,
      "logps/chosen": -207.9678192138672,
      "logps/rejected": -230.5495147705078,
      "loss": 0.3428,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.694399356842041,
      "rewards/margins": 1.4684067964553833,
      "rewards/rejected": -2.162806272506714,
      "step": 5030
    },
    {
      "epoch": 4.032,
      "grad_norm": 7.541386604309082,
      "learning_rate": 3.726346943427719e-06,
      "logits/chosen": 0.9582153558731079,
      "logits/rejected": 0.867928683757782,
      "logps/chosen": -208.20193481445312,
      "logps/rejected": -234.29641723632812,
      "loss": 0.3957,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.2386597394943237,
      "rewards/margins": 1.2774535417556763,
      "rewards/rejected": -2.516113042831421,
      "step": 5040
    },
    {
      "epoch": 4.04,
      "grad_norm": 12.571610450744629,
      "learning_rate": 3.720258514858004e-06,
      "logits/chosen": 1.0318713188171387,
      "logits/rejected": 0.8940740823745728,
      "logps/chosen": -195.3436279296875,
      "logps/rejected": -235.5468292236328,
      "loss": 0.3239,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.7823795676231384,
      "rewards/margins": 1.5767382383346558,
      "rewards/rejected": -2.3591177463531494,
      "step": 5050
    },
    {
      "epoch": 4.048,
      "grad_norm": 14.773040771484375,
      "learning_rate": 3.714160570467266e-06,
      "logits/chosen": 0.9699028134346008,
      "logits/rejected": 0.9142343401908875,
      "logps/chosen": -218.6599578857422,
      "logps/rejected": -241.7670440673828,
      "loss": 0.3705,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.231390357017517,
      "rewards/margins": 1.3999671936035156,
      "rewards/rejected": -2.6313576698303223,
      "step": 5060
    },
    {
      "epoch": 4.056,
      "grad_norm": 8.311098098754883,
      "learning_rate": 3.7080531578085e-06,
      "logits/chosen": 0.880449116230011,
      "logits/rejected": 0.8501182794570923,
      "logps/chosen": -207.9825897216797,
      "logps/rejected": -245.84878540039062,
      "loss": 0.3206,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.1131296157836914,
      "rewards/margins": 1.682613730430603,
      "rewards/rejected": -2.795743465423584,
      "step": 5070
    },
    {
      "epoch": 4.064,
      "grad_norm": 13.896095275878906,
      "learning_rate": 3.701936324508537e-06,
      "logits/chosen": 1.1272010803222656,
      "logits/rejected": 0.9481062293052673,
      "logps/chosen": -211.05064392089844,
      "logps/rejected": -214.7148895263672,
      "loss": 0.3481,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.8524333834648132,
      "rewards/margins": 1.6547110080718994,
      "rewards/rejected": -2.5071444511413574,
      "step": 5080
    },
    {
      "epoch": 4.072,
      "grad_norm": 11.393054008483887,
      "learning_rate": 3.6958101182676725e-06,
      "logits/chosen": 0.9813786745071411,
      "logits/rejected": 0.962406575679779,
      "logps/chosen": -205.95632934570312,
      "logps/rejected": -231.1307373046875,
      "loss": 0.3552,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.020094633102417,
      "rewards/margins": 1.620416283607483,
      "rewards/rejected": -2.6405110359191895,
      "step": 5090
    },
    {
      "epoch": 4.08,
      "grad_norm": 16.073274612426758,
      "learning_rate": 3.6896745868592924e-06,
      "logits/chosen": 0.980717658996582,
      "logits/rejected": 0.9125275611877441,
      "logps/chosen": -199.70022583007812,
      "logps/rejected": -237.94412231445312,
      "loss": 0.3716,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2604044675827026,
      "rewards/margins": 1.5706150531768799,
      "rewards/rejected": -2.831019639968872,
      "step": 5100
    },
    {
      "epoch": 4.088,
      "grad_norm": 15.651266098022461,
      "learning_rate": 3.683529778129504e-06,
      "logits/chosen": 1.0326038599014282,
      "logits/rejected": 0.9012963175773621,
      "logps/chosen": -218.704833984375,
      "logps/rejected": -238.8866424560547,
      "loss": 0.3246,
      "rewards/accuracies": 0.9000000357627869,
      "rewards/chosen": -1.407049536705017,
      "rewards/margins": 1.6295727491378784,
      "rewards/rejected": -3.0366222858428955,
      "step": 5110
    },
    {
      "epoch": 4.096,
      "grad_norm": 9.559797286987305,
      "learning_rate": 3.677375739996759e-06,
      "logits/chosen": 0.9783565402030945,
      "logits/rejected": 0.9558612704277039,
      "logps/chosen": -231.9866485595703,
      "logps/rejected": -237.6753387451172,
      "loss": 0.3427,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.2938991785049438,
      "rewards/margins": 1.4221700429916382,
      "rewards/rejected": -2.716069221496582,
      "step": 5120
    },
    {
      "epoch": 4.104,
      "grad_norm": 20.652666091918945,
      "learning_rate": 3.6712125204514836e-06,
      "logits/chosen": 1.0956971645355225,
      "logits/rejected": 1.1476175785064697,
      "logps/chosen": -198.1278533935547,
      "logps/rejected": -215.04112243652344,
      "loss": 0.3917,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.28689682483673096,
      "rewards/margins": 1.569340467453003,
      "rewards/rejected": -1.8562372922897339,
      "step": 5130
    },
    {
      "epoch": 4.112,
      "grad_norm": 20.57211685180664,
      "learning_rate": 3.6650401675557025e-06,
      "logits/chosen": 1.0987768173217773,
      "logits/rejected": 0.8933493494987488,
      "logps/chosen": -203.9111785888672,
      "logps/rejected": -229.03076171875,
      "loss": 0.3811,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.243438720703125,
      "rewards/margins": 1.4107178449630737,
      "rewards/rejected": -2.654156446456909,
      "step": 5140
    },
    {
      "epoch": 4.12,
      "grad_norm": 5.7946271896362305,
      "learning_rate": 3.658858729442662e-06,
      "logits/chosen": 0.9748809933662415,
      "logits/rejected": 0.8560425043106079,
      "logps/chosen": -199.7461395263672,
      "logps/rejected": -234.7223663330078,
      "loss": 0.3288,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.376451015472412,
      "rewards/margins": 1.6137140989303589,
      "rewards/rejected": -2.9901649951934814,
      "step": 5150
    },
    {
      "epoch": 4.128,
      "grad_norm": 6.650130271911621,
      "learning_rate": 3.65266825431646e-06,
      "logits/chosen": 0.9371269345283508,
      "logits/rejected": 1.002604603767395,
      "logps/chosen": -216.43150329589844,
      "logps/rejected": -230.34336853027344,
      "loss": 0.3251,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1472088098526,
      "rewards/margins": 1.6432514190673828,
      "rewards/rejected": -2.7904601097106934,
      "step": 5160
    },
    {
      "epoch": 4.136,
      "grad_norm": 24.08129119873047,
      "learning_rate": 3.646468790451663e-06,
      "logits/chosen": 0.8831321597099304,
      "logits/rejected": 0.776918888092041,
      "logps/chosen": -213.083740234375,
      "logps/rejected": -230.79202270507812,
      "loss": 0.4005,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.3220943212509155,
      "rewards/margins": 1.3914340734481812,
      "rewards/rejected": -2.7135283946990967,
      "step": 5170
    },
    {
      "epoch": 4.144,
      "grad_norm": 7.815832138061523,
      "learning_rate": 3.6402603861929374e-06,
      "logits/chosen": 1.0589165687561035,
      "logits/rejected": 0.9391776919364929,
      "logps/chosen": -221.99449157714844,
      "logps/rejected": -242.86216735839844,
      "loss": 0.2609,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.1631277799606323,
      "rewards/margins": 1.766058325767517,
      "rewards/rejected": -2.9291858673095703,
      "step": 5180
    },
    {
      "epoch": 4.152,
      "grad_norm": 13.720664978027344,
      "learning_rate": 3.6340430899546656e-06,
      "logits/chosen": 0.9697431921958923,
      "logits/rejected": 0.9235960841178894,
      "logps/chosen": -223.24876403808594,
      "logps/rejected": -240.31167602539062,
      "loss": 0.4369,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.2503145933151245,
      "rewards/margins": 1.2561858892440796,
      "rewards/rejected": -2.506500482559204,
      "step": 5190
    },
    {
      "epoch": 4.16,
      "grad_norm": 16.428617477416992,
      "learning_rate": 3.6278169502205734e-06,
      "logits/chosen": 1.0490871667861938,
      "logits/rejected": 0.9422445297241211,
      "logps/chosen": -211.8967742919922,
      "logps/rejected": -263.45782470703125,
      "loss": 0.3235,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.041167140007019,
      "rewards/margins": 1.970610499382019,
      "rewards/rejected": -3.011777639389038,
      "step": 5200
    },
    {
      "epoch": 4.168,
      "grad_norm": 28.398910522460938,
      "learning_rate": 3.6215820155433486e-06,
      "logits/chosen": 0.9427713751792908,
      "logits/rejected": 0.9313327670097351,
      "logps/chosen": -243.6337890625,
      "logps/rejected": -252.7070770263672,
      "loss": 0.3734,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9518672227859497,
      "rewards/margins": 1.5367608070373535,
      "rewards/rejected": -2.4886281490325928,
      "step": 5210
    },
    {
      "epoch": 4.176,
      "grad_norm": 14.895367622375488,
      "learning_rate": 3.615338334544265e-06,
      "logits/chosen": 0.9828926920890808,
      "logits/rejected": 0.9496780633926392,
      "logps/chosen": -203.6242218017578,
      "logps/rejected": -224.4546661376953,
      "loss": 0.3542,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.7610888481140137,
      "rewards/margins": 1.4806041717529297,
      "rewards/rejected": -2.2416930198669434,
      "step": 5220
    },
    {
      "epoch": 4.184,
      "grad_norm": 19.1573543548584,
      "learning_rate": 3.6090859559128e-06,
      "logits/chosen": 0.9076266288757324,
      "logits/rejected": 0.9045122265815735,
      "logps/chosen": -212.6093292236328,
      "logps/rejected": -223.61474609375,
      "loss": 0.3464,
      "rewards/accuracies": 0.8375000357627869,
      "rewards/chosen": -0.48652344942092896,
      "rewards/margins": 1.6024211645126343,
      "rewards/rejected": -2.088944435119629,
      "step": 5230
    },
    {
      "epoch": 4.192,
      "grad_norm": 7.1050310134887695,
      "learning_rate": 3.6028249284062593e-06,
      "logits/chosen": 0.9402229189872742,
      "logits/rejected": 0.9130407571792603,
      "logps/chosen": -215.83328247070312,
      "logps/rejected": -237.27294921875,
      "loss": 0.3177,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.442810297012329,
      "rewards/margins": 1.6023463010787964,
      "rewards/rejected": -3.045156478881836,
      "step": 5240
    },
    {
      "epoch": 4.2,
      "grad_norm": 13.212199211120605,
      "learning_rate": 3.5965553008493924e-06,
      "logits/chosen": 0.9931579828262329,
      "logits/rejected": 0.8333091139793396,
      "logps/chosen": -216.7156219482422,
      "logps/rejected": -227.3023681640625,
      "loss": 0.4671,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.2548701763153076,
      "rewards/margins": 1.3508440256118774,
      "rewards/rejected": -2.6057140827178955,
      "step": 5250
    },
    {
      "epoch": 4.208,
      "grad_norm": 15.673922538757324,
      "learning_rate": 3.590277122134015e-06,
      "logits/chosen": 0.925972580909729,
      "logits/rejected": 0.9485855102539062,
      "logps/chosen": -203.7695770263672,
      "logps/rejected": -233.1253204345703,
      "loss": 0.2825,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.1053215265274048,
      "rewards/margins": 1.8811248540878296,
      "rewards/rejected": -2.9864463806152344,
      "step": 5260
    },
    {
      "epoch": 4.216,
      "grad_norm": 18.827274322509766,
      "learning_rate": 3.5839904412186254e-06,
      "logits/chosen": 0.959101676940918,
      "logits/rejected": 1.012556552886963,
      "logps/chosen": -203.8087615966797,
      "logps/rejected": -251.2672119140625,
      "loss": 0.3625,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9612110257148743,
      "rewards/margins": 1.7676079273223877,
      "rewards/rejected": -2.7288191318511963,
      "step": 5270
    },
    {
      "epoch": 4.224,
      "grad_norm": 10.582806587219238,
      "learning_rate": 3.577695307128024e-06,
      "logits/chosen": 0.8980388641357422,
      "logits/rejected": 0.7367168664932251,
      "logps/chosen": -219.5625,
      "logps/rejected": -244.0664520263672,
      "loss": 0.2701,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.9930900931358337,
      "rewards/margins": 2.069688558578491,
      "rewards/rejected": -3.0627784729003906,
      "step": 5280
    },
    {
      "epoch": 4.232,
      "grad_norm": 17.180286407470703,
      "learning_rate": 3.571391768952932e-06,
      "logits/chosen": 0.8705675005912781,
      "logits/rejected": 0.8819023966789246,
      "logps/chosen": -189.57054138183594,
      "logps/rejected": -228.0927734375,
      "loss": 0.3455,
      "rewards/accuracies": 0.8375000357627869,
      "rewards/chosen": -0.9885832667350769,
      "rewards/margins": 1.6450799703598022,
      "rewards/rejected": -2.6336631774902344,
      "step": 5290
    },
    {
      "epoch": 4.24,
      "grad_norm": 13.662627220153809,
      "learning_rate": 3.5650798758496053e-06,
      "logits/chosen": 0.880375325679779,
      "logits/rejected": 0.8753491640090942,
      "logps/chosen": -213.9600067138672,
      "logps/rejected": -244.53857421875,
      "loss": 0.3318,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.6102064847946167,
      "rewards/margins": 1.6889526844024658,
      "rewards/rejected": -3.299158811569214,
      "step": 5300
    },
    {
      "epoch": 4.248,
      "grad_norm": 19.561792373657227,
      "learning_rate": 3.558759677039455e-06,
      "logits/chosen": 0.8705887794494629,
      "logits/rejected": 0.8816524744033813,
      "logps/chosen": -213.3289031982422,
      "logps/rejected": -243.65737915039062,
      "loss": 0.3461,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.5360292196273804,
      "rewards/margins": 1.6422207355499268,
      "rewards/rejected": -3.1782500743865967,
      "step": 5310
    },
    {
      "epoch": 4.256,
      "grad_norm": 16.225622177124023,
      "learning_rate": 3.552431221808661e-06,
      "logits/chosen": 0.988519012928009,
      "logits/rejected": 0.824196457862854,
      "logps/chosen": -195.06642150878906,
      "logps/rejected": -229.2406768798828,
      "loss": 0.3154,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.1573065519332886,
      "rewards/margins": 1.9272657632827759,
      "rewards/rejected": -3.0845723152160645,
      "step": 5320
    },
    {
      "epoch": 4.264,
      "grad_norm": 11.897854804992676,
      "learning_rate": 3.5460945595077874e-06,
      "logits/chosen": 0.9042286276817322,
      "logits/rejected": 0.8848081827163696,
      "logps/chosen": -203.17344665527344,
      "logps/rejected": -256.4823303222656,
      "loss": 0.3054,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.392894983291626,
      "rewards/margins": 1.7944774627685547,
      "rewards/rejected": -3.1873722076416016,
      "step": 5330
    },
    {
      "epoch": 4.272,
      "grad_norm": 13.320012092590332,
      "learning_rate": 3.539749739551401e-06,
      "logits/chosen": 0.9267520904541016,
      "logits/rejected": 0.9451652765274048,
      "logps/chosen": -191.97021484375,
      "logps/rejected": -214.5209503173828,
      "loss": 0.2868,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.6399614810943604,
      "rewards/margins": 1.7933428287506104,
      "rewards/rejected": -2.4333043098449707,
      "step": 5340
    },
    {
      "epoch": 4.28,
      "grad_norm": 10.956954002380371,
      "learning_rate": 3.533396811417682e-06,
      "logits/chosen": 0.9412531852722168,
      "logits/rejected": 0.9464859366416931,
      "logps/chosen": -192.187255859375,
      "logps/rejected": -215.4243621826172,
      "loss": 0.3242,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8624544143676758,
      "rewards/margins": 1.6475841999053955,
      "rewards/rejected": -2.5100386142730713,
      "step": 5350
    },
    {
      "epoch": 4.288,
      "grad_norm": 18.71451187133789,
      "learning_rate": 3.527035824648039e-06,
      "logits/chosen": 0.8915054202079773,
      "logits/rejected": 0.7794849276542664,
      "logps/chosen": -192.20079040527344,
      "logps/rejected": -232.98291015625,
      "loss": 0.3807,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.0030672550201416,
      "rewards/margins": 1.811819076538086,
      "rewards/rejected": -2.8148863315582275,
      "step": 5360
    },
    {
      "epoch": 4.296,
      "grad_norm": 12.68542194366455,
      "learning_rate": 3.520666828846726e-06,
      "logits/chosen": 0.7684556841850281,
      "logits/rejected": 0.7673687934875488,
      "logps/chosen": -211.497314453125,
      "logps/rejected": -240.1795654296875,
      "loss": 0.3209,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.4810975790023804,
      "rewards/margins": 1.700387954711914,
      "rewards/rejected": -3.181485652923584,
      "step": 5370
    },
    {
      "epoch": 4.304,
      "grad_norm": 6.8565144538879395,
      "learning_rate": 3.5142898736804516e-06,
      "logits/chosen": 0.9116408228874207,
      "logits/rejected": 0.796613335609436,
      "logps/chosen": -211.6884765625,
      "logps/rejected": -243.06777954101562,
      "loss": 0.3276,
      "rewards/accuracies": 0.9000000357627869,
      "rewards/chosen": -1.3549237251281738,
      "rewards/margins": 1.6243526935577393,
      "rewards/rejected": -2.979276418685913,
      "step": 5380
    },
    {
      "epoch": 4.312,
      "grad_norm": 9.291360855102539,
      "learning_rate": 3.5079050088779927e-06,
      "logits/chosen": 0.8643763661384583,
      "logits/rejected": 0.8873327374458313,
      "logps/chosen": -180.9557342529297,
      "logps/rejected": -216.88780212402344,
      "loss": 0.3089,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.6550284624099731,
      "rewards/margins": 1.8085215091705322,
      "rewards/rejected": -2.463550090789795,
      "step": 5390
    },
    {
      "epoch": 4.32,
      "grad_norm": 10.52054500579834,
      "learning_rate": 3.501512284229807e-06,
      "logits/chosen": 0.8750426173210144,
      "logits/rejected": 0.7605211734771729,
      "logps/chosen": -212.87950134277344,
      "logps/rejected": -263.2989196777344,
      "loss": 0.2426,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.4775450229644775,
      "rewards/margins": 2.0750772953033447,
      "rewards/rejected": -3.552621841430664,
      "step": 5400
    },
    {
      "epoch": 4.328,
      "grad_norm": 16.574983596801758,
      "learning_rate": 3.4951117495876473e-06,
      "logits/chosen": 0.9197494387626648,
      "logits/rejected": 0.8915640115737915,
      "logps/chosen": -215.3748016357422,
      "logps/rejected": -229.8962860107422,
      "loss": 0.3404,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.7154279947280884,
      "rewards/margins": 1.5610477924346924,
      "rewards/rejected": -3.276475667953491,
      "step": 5410
    },
    {
      "epoch": 4.336,
      "grad_norm": 15.055716514587402,
      "learning_rate": 3.4887034548641673e-06,
      "logits/chosen": 0.9449960589408875,
      "logits/rejected": 0.9040482640266418,
      "logps/chosen": -217.7838897705078,
      "logps/rejected": -268.0353088378906,
      "loss": 0.3165,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.7793148756027222,
      "rewards/margins": 1.969291090965271,
      "rewards/rejected": -3.748605728149414,
      "step": 5420
    },
    {
      "epoch": 4.344,
      "grad_norm": 7.452909469604492,
      "learning_rate": 3.482287450032536e-06,
      "logits/chosen": 0.8897709250450134,
      "logits/rejected": 0.8994399905204773,
      "logps/chosen": -212.1781768798828,
      "logps/rejected": -253.2353973388672,
      "loss": 0.3063,
      "rewards/accuracies": 0.9000000357627869,
      "rewards/chosen": -1.805542230606079,
      "rewards/margins": 1.7644466161727905,
      "rewards/rejected": -3.569988965988159,
      "step": 5430
    },
    {
      "epoch": 4.352,
      "grad_norm": 19.223543167114258,
      "learning_rate": 3.47586378512605e-06,
      "logits/chosen": 1.0354408025741577,
      "logits/rejected": 0.9919587969779968,
      "logps/chosen": -216.69546508789062,
      "logps/rejected": -250.68069458007812,
      "loss": 0.3631,
      "rewards/accuracies": 0.8375000357627869,
      "rewards/chosen": -1.8229974508285522,
      "rewards/margins": 1.5450613498687744,
      "rewards/rejected": -3.368058919906616,
      "step": 5440
    },
    {
      "epoch": 4.36,
      "grad_norm": 13.662184715270996,
      "learning_rate": 3.4694325102377356e-06,
      "logits/chosen": 0.972303569316864,
      "logits/rejected": 0.8980138897895813,
      "logps/chosen": -206.581787109375,
      "logps/rejected": -222.6265106201172,
      "loss": 0.4966,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6232937574386597,
      "rewards/margins": 1.1546478271484375,
      "rewards/rejected": -2.7779414653778076,
      "step": 5450
    },
    {
      "epoch": 4.368,
      "grad_norm": 6.83968448638916,
      "learning_rate": 3.462993675519968e-06,
      "logits/chosen": 0.8973142504692078,
      "logits/rejected": 1.0178273916244507,
      "logps/chosen": -226.5254364013672,
      "logps/rejected": -241.5300750732422,
      "loss": 0.3852,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.1246910095214844,
      "rewards/margins": 1.604926347732544,
      "rewards/rejected": -3.7296173572540283,
      "step": 5460
    },
    {
      "epoch": 4.376,
      "grad_norm": 13.287477493286133,
      "learning_rate": 3.4565473311840735e-06,
      "logits/chosen": 1.0390936136245728,
      "logits/rejected": 0.9516915678977966,
      "logps/chosen": -214.5906982421875,
      "logps/rejected": -220.36778259277344,
      "loss": 0.2884,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.473663330078125,
      "rewards/margins": 1.6746243238449097,
      "rewards/rejected": -3.148287534713745,
      "step": 5470
    },
    {
      "epoch": 4.384,
      "grad_norm": 8.79075813293457,
      "learning_rate": 3.4500935274999414e-06,
      "logits/chosen": 0.8991445899009705,
      "logits/rejected": 0.7987154126167297,
      "logps/chosen": -221.56704711914062,
      "logps/rejected": -226.0802764892578,
      "loss": 0.3229,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.3047336339950562,
      "rewards/margins": 1.384783148765564,
      "rewards/rejected": -2.68951678276062,
      "step": 5480
    },
    {
      "epoch": 4.392,
      "grad_norm": 18.390661239624023,
      "learning_rate": 3.443632314795627e-06,
      "logits/chosen": 1.0096405744552612,
      "logits/rejected": 0.9392326474189758,
      "logps/chosen": -209.8367156982422,
      "logps/rejected": -223.6701202392578,
      "loss": 0.3204,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4388588666915894,
      "rewards/margins": 1.779291033744812,
      "rewards/rejected": -3.2181496620178223,
      "step": 5490
    },
    {
      "epoch": 4.4,
      "grad_norm": 14.75433349609375,
      "learning_rate": 3.4371637434569664e-06,
      "logits/chosen": 0.9798237085342407,
      "logits/rejected": 0.8885357975959778,
      "logps/chosen": -215.8732147216797,
      "logps/rejected": -250.0683135986328,
      "loss": 0.2836,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.3270418643951416,
      "rewards/margins": 1.8547767400741577,
      "rewards/rejected": -3.181818723678589,
      "step": 5500
    },
    {
      "epoch": 4.408,
      "grad_norm": 7.223197937011719,
      "learning_rate": 3.430687863927178e-06,
      "logits/chosen": 0.9878398776054382,
      "logits/rejected": 0.907991886138916,
      "logps/chosen": -220.220947265625,
      "logps/rejected": -232.602294921875,
      "loss": 0.3854,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.4777723550796509,
      "rewards/margins": 1.486155390739441,
      "rewards/rejected": -2.963927745819092,
      "step": 5510
    },
    {
      "epoch": 4.416,
      "grad_norm": 18.754655838012695,
      "learning_rate": 3.4242047267064714e-06,
      "logits/chosen": 1.0766704082489014,
      "logits/rejected": 1.0072535276412964,
      "logps/chosen": -218.5634307861328,
      "logps/rejected": -240.5037841796875,
      "loss": 0.3645,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.6403402090072632,
      "rewards/margins": 1.6216999292373657,
      "rewards/rejected": -3.26203989982605,
      "step": 5520
    },
    {
      "epoch": 4.424,
      "grad_norm": 16.398393630981445,
      "learning_rate": 3.4177143823516523e-06,
      "logits/chosen": 0.9290235638618469,
      "logits/rejected": 0.8646947145462036,
      "logps/chosen": -221.277099609375,
      "logps/rejected": -243.0144805908203,
      "loss": 0.3467,
      "rewards/accuracies": 0.8375000357627869,
      "rewards/chosen": -1.3182934522628784,
      "rewards/margins": 1.7082767486572266,
      "rewards/rejected": -3.0265700817108154,
      "step": 5530
    },
    {
      "epoch": 4.432,
      "grad_norm": 10.032676696777344,
      "learning_rate": 3.4112168814757307e-06,
      "logits/chosen": 0.9694162607192993,
      "logits/rejected": 0.9398545622825623,
      "logps/chosen": -205.362060546875,
      "logps/rejected": -224.15017700195312,
      "loss": 0.2903,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.2131686210632324,
      "rewards/margins": 1.873663306236267,
      "rewards/rejected": -3.086832046508789,
      "step": 5540
    },
    {
      "epoch": 4.44,
      "grad_norm": 21.09775733947754,
      "learning_rate": 3.4047122747475227e-06,
      "logits/chosen": 0.9186191558837891,
      "logits/rejected": 0.763767421245575,
      "logps/chosen": -226.12075805664062,
      "logps/rejected": -264.490966796875,
      "loss": 0.3921,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.8982375860214233,
      "rewards/margins": 1.5824882984161377,
      "rewards/rejected": -3.4807260036468506,
      "step": 5550
    },
    {
      "epoch": 4.448,
      "grad_norm": 25.094377517700195,
      "learning_rate": 3.3982006128912587e-06,
      "logits/chosen": 0.8753210306167603,
      "logits/rejected": 0.8916776776313782,
      "logps/chosen": -205.23233032226562,
      "logps/rejected": -227.34988403320312,
      "loss": 0.433,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.5061285495758057,
      "rewards/margins": 1.4716495275497437,
      "rewards/rejected": -2.9777779579162598,
      "step": 5560
    },
    {
      "epoch": 4.456,
      "grad_norm": 19.3261661529541,
      "learning_rate": 3.391681946686186e-06,
      "logits/chosen": 0.8373166918754578,
      "logits/rejected": 0.819493293762207,
      "logps/chosen": -207.85977172851562,
      "logps/rejected": -245.46836853027344,
      "loss": 0.3845,
      "rewards/accuracies": 0.9000000357627869,
      "rewards/chosen": -1.96077561378479,
      "rewards/margins": 1.5056307315826416,
      "rewards/rejected": -3.4664063453674316,
      "step": 5570
    },
    {
      "epoch": 4.464,
      "grad_norm": 17.59246253967285,
      "learning_rate": 3.385156326966173e-06,
      "logits/chosen": 0.9824124574661255,
      "logits/rejected": 0.9056622385978699,
      "logps/chosen": -212.71084594726562,
      "logps/rejected": -223.310791015625,
      "loss": 0.3712,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.4070976972579956,
      "rewards/margins": 1.7284908294677734,
      "rewards/rejected": -3.1355881690979004,
      "step": 5580
    },
    {
      "epoch": 4.4719999999999995,
      "grad_norm": 10.986981391906738,
      "learning_rate": 3.3786238046193125e-06,
      "logits/chosen": 0.9522453546524048,
      "logits/rejected": 0.8866806030273438,
      "logps/chosen": -206.303955078125,
      "logps/rejected": -239.8437042236328,
      "loss": 0.4034,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.8517950773239136,
      "rewards/margins": 1.48299241065979,
      "rewards/rejected": -3.3347878456115723,
      "step": 5590
    },
    {
      "epoch": 4.48,
      "grad_norm": 10.576725006103516,
      "learning_rate": 3.372084430587528e-06,
      "logits/chosen": 0.8120132684707642,
      "logits/rejected": 0.8881155252456665,
      "logps/chosen": -224.1233367919922,
      "logps/rejected": -252.2123260498047,
      "loss": 0.3466,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8974634408950806,
      "rewards/margins": 1.5087515115737915,
      "rewards/rejected": -3.406214952468872,
      "step": 5600
    },
    {
      "epoch": 4.4879999999999995,
      "grad_norm": 7.61484432220459,
      "learning_rate": 3.365538255866169e-06,
      "logits/chosen": 0.9328517913818359,
      "logits/rejected": 0.832955539226532,
      "logps/chosen": -224.76219177246094,
      "logps/rejected": -263.3474426269531,
      "loss": 0.3206,
      "rewards/accuracies": 0.9000000357627869,
      "rewards/chosen": -1.6335548162460327,
      "rewards/margins": 2.0621376037597656,
      "rewards/rejected": -3.695692539215088,
      "step": 5610
    },
    {
      "epoch": 4.496,
      "grad_norm": 7.165886878967285,
      "learning_rate": 3.3589853315036227e-06,
      "logits/chosen": 0.9069485068321228,
      "logits/rejected": 0.8654775619506836,
      "logps/chosen": -231.8223876953125,
      "logps/rejected": -247.6749267578125,
      "loss": 0.3769,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.6844438314437866,
      "rewards/margins": 1.5118721723556519,
      "rewards/rejected": -3.1963162422180176,
      "step": 5620
    },
    {
      "epoch": 4.504,
      "grad_norm": 12.06303596496582,
      "learning_rate": 3.3524257086009105e-06,
      "logits/chosen": 0.8854274153709412,
      "logits/rejected": 0.9237664341926575,
      "logps/chosen": -219.48692321777344,
      "logps/rejected": -252.7774200439453,
      "loss": 0.2831,
      "rewards/accuracies": 0.9000000357627869,
      "rewards/chosen": -1.6034126281738281,
      "rewards/margins": 1.9384158849716187,
      "rewards/rejected": -3.541828155517578,
      "step": 5630
    },
    {
      "epoch": 4.5120000000000005,
      "grad_norm": 10.347089767456055,
      "learning_rate": 3.3458594383112868e-06,
      "logits/chosen": 0.9743103981018066,
      "logits/rejected": 0.820307731628418,
      "logps/chosen": -192.3817138671875,
      "logps/rejected": -223.5673065185547,
      "loss": 0.3259,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.055615782737732,
      "rewards/margins": 1.9324188232421875,
      "rewards/rejected": -2.98803448677063,
      "step": 5640
    },
    {
      "epoch": 4.52,
      "grad_norm": 21.758241653442383,
      "learning_rate": 3.339286571839848e-06,
      "logits/chosen": 0.8464789390563965,
      "logits/rejected": 0.8443576693534851,
      "logps/chosen": -203.8209991455078,
      "logps/rejected": -246.0260772705078,
      "loss": 0.3542,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.057892084121704,
      "rewards/margins": 1.6661615371704102,
      "rewards/rejected": -2.7240536212921143,
      "step": 5650
    },
    {
      "epoch": 4.5280000000000005,
      "grad_norm": 15.415779113769531,
      "learning_rate": 3.332707160443128e-06,
      "logits/chosen": 0.9635658264160156,
      "logits/rejected": 0.8357729911804199,
      "logps/chosen": -222.7996826171875,
      "logps/rejected": -249.92446899414062,
      "loss": 0.2987,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3085647821426392,
      "rewards/margins": 1.8709415197372437,
      "rewards/rejected": -3.179506301879883,
      "step": 5660
    },
    {
      "epoch": 4.536,
      "grad_norm": 22.9267578125,
      "learning_rate": 3.3261212554286977e-06,
      "logits/chosen": 0.8950627446174622,
      "logits/rejected": 0.8704226613044739,
      "logps/chosen": -221.8727569580078,
      "logps/rejected": -254.3755645751953,
      "loss": 0.4041,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.043503761291504,
      "rewards/margins": 1.4210357666015625,
      "rewards/rejected": -3.4645392894744873,
      "step": 5670
    },
    {
      "epoch": 4.5440000000000005,
      "grad_norm": 31.6077880859375,
      "learning_rate": 3.319528908154766e-06,
      "logits/chosen": 0.8877006769180298,
      "logits/rejected": 0.8138026595115662,
      "logps/chosen": -252.089111328125,
      "logps/rejected": -247.1710968017578,
      "loss": 0.427,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.8856754302978516,
      "rewards/margins": 1.4032905101776123,
      "rewards/rejected": -3.288965940475464,
      "step": 5680
    },
    {
      "epoch": 4.552,
      "grad_norm": 19.946836471557617,
      "learning_rate": 3.3129301700297834e-06,
      "logits/chosen": 1.060017704963684,
      "logits/rejected": 1.0112991333007812,
      "logps/chosen": -202.8851776123047,
      "logps/rejected": -234.48912048339844,
      "loss": 0.3442,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.6744213104248047,
      "rewards/margins": 1.476469874382019,
      "rewards/rejected": -3.1508913040161133,
      "step": 5690
    },
    {
      "epoch": 4.5600000000000005,
      "grad_norm": 9.337148666381836,
      "learning_rate": 3.306325092512034e-06,
      "logits/chosen": 0.9761505126953125,
      "logits/rejected": 0.9596638083457947,
      "logps/chosen": -208.33743286132812,
      "logps/rejected": -234.8901824951172,
      "loss": 0.3815,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.9948211908340454,
      "rewards/margins": 1.56894850730896,
      "rewards/rejected": -3.563769578933716,
      "step": 5700
    },
    {
      "epoch": 4.568,
      "grad_norm": 13.035239219665527,
      "learning_rate": 3.2997137271092396e-06,
      "logits/chosen": 0.963702380657196,
      "logits/rejected": 0.872109055519104,
      "logps/chosen": -234.18614196777344,
      "logps/rejected": -254.7106475830078,
      "loss": 0.3564,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.163374900817871,
      "rewards/margins": 1.6218290328979492,
      "rewards/rejected": -3.7852039337158203,
      "step": 5710
    },
    {
      "epoch": 4.576,
      "grad_norm": 27.116188049316406,
      "learning_rate": 3.293096125378156e-06,
      "logits/chosen": 0.92717045545578,
      "logits/rejected": 0.908390462398529,
      "logps/chosen": -210.4567413330078,
      "logps/rejected": -253.227294921875,
      "loss": 0.3333,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5652707815170288,
      "rewards/margins": 1.8512943983078003,
      "rewards/rejected": -3.41656494140625,
      "step": 5720
    },
    {
      "epoch": 4.584,
      "grad_norm": 19.045427322387695,
      "learning_rate": 3.2864723389241697e-06,
      "logits/chosen": 0.9135765433311462,
      "logits/rejected": 0.859271228313446,
      "logps/chosen": -222.8280487060547,
      "logps/rejected": -242.25540161132812,
      "loss": 0.3374,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.3668630123138428,
      "rewards/margins": 1.7538166046142578,
      "rewards/rejected": -3.1206796169281006,
      "step": 5730
    },
    {
      "epoch": 4.592,
      "grad_norm": 20.58856773376465,
      "learning_rate": 3.279842419400899e-06,
      "logits/chosen": 0.8949125409126282,
      "logits/rejected": 0.9091690182685852,
      "logps/chosen": -224.7333984375,
      "logps/rejected": -265.9559631347656,
      "loss": 0.4016,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.5353691577911377,
      "rewards/margins": 1.6605640649795532,
      "rewards/rejected": -3.1959331035614014,
      "step": 5740
    },
    {
      "epoch": 4.6,
      "grad_norm": 15.529658317565918,
      "learning_rate": 3.2732064185097885e-06,
      "logits/chosen": 1.0308302640914917,
      "logits/rejected": 0.9736013412475586,
      "logps/chosen": -207.3667449951172,
      "logps/rejected": -235.87290954589844,
      "loss": 0.3166,
      "rewards/accuracies": 0.9000000357627869,
      "rewards/chosen": -1.106737732887268,
      "rewards/margins": 1.6635363101959229,
      "rewards/rejected": -2.7702741622924805,
      "step": 5750
    },
    {
      "epoch": 4.608,
      "grad_norm": 13.790717124938965,
      "learning_rate": 3.2665643879997054e-06,
      "logits/chosen": 1.0450878143310547,
      "logits/rejected": 1.0168274641036987,
      "logps/chosen": -201.68594360351562,
      "logps/rejected": -224.7779541015625,
      "loss": 0.4748,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.329674243927002,
      "rewards/margins": 1.2952015399932861,
      "rewards/rejected": -2.624875783920288,
      "step": 5760
    },
    {
      "epoch": 4.616,
      "grad_norm": 19.334339141845703,
      "learning_rate": 3.259916379666538e-06,
      "logits/chosen": 0.9460241198539734,
      "logits/rejected": 0.902341365814209,
      "logps/chosen": -206.3498077392578,
      "logps/rejected": -234.02978515625,
      "loss": 0.3343,
      "rewards/accuracies": 0.9000000357627869,
      "rewards/chosen": -0.8321787714958191,
      "rewards/margins": 1.6867774724960327,
      "rewards/rejected": -2.518956422805786,
      "step": 5770
    },
    {
      "epoch": 4.624,
      "grad_norm": 11.740504264831543,
      "learning_rate": 3.2532624453527904e-06,
      "logits/chosen": 0.765145480632782,
      "logits/rejected": 0.8052225112915039,
      "logps/chosen": -215.4130401611328,
      "logps/rejected": -230.92808532714844,
      "loss": 0.3438,
      "rewards/accuracies": 0.8375000357627869,
      "rewards/chosen": -0.9054020047187805,
      "rewards/margins": 1.7861785888671875,
      "rewards/rejected": -2.6915805339813232,
      "step": 5780
    },
    {
      "epoch": 4.632,
      "grad_norm": 23.415552139282227,
      "learning_rate": 3.2466026369471804e-06,
      "logits/chosen": 0.8790031671524048,
      "logits/rejected": 0.8607792854309082,
      "logps/chosen": -205.34423828125,
      "logps/rejected": -228.1710968017578,
      "loss": 0.4049,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.2571852207183838,
      "rewards/margins": 1.416163682937622,
      "rewards/rejected": -2.6733486652374268,
      "step": 5790
    },
    {
      "epoch": 4.64,
      "grad_norm": 17.48906707763672,
      "learning_rate": 3.2399370063842297e-06,
      "logits/chosen": 0.865296483039856,
      "logits/rejected": 0.8317980170249939,
      "logps/chosen": -233.6914520263672,
      "logps/rejected": -257.34161376953125,
      "loss": 0.4092,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.9758745431900024,
      "rewards/margins": 1.5804134607315063,
      "rewards/rejected": -3.556288242340088,
      "step": 5800
    },
    {
      "epoch": 4.648,
      "grad_norm": 15.079739570617676,
      "learning_rate": 3.2332656056438663e-06,
      "logits/chosen": 0.9580942392349243,
      "logits/rejected": 0.8425354957580566,
      "logps/chosen": -206.0723419189453,
      "logps/rejected": -235.4949493408203,
      "loss": 0.4092,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.753932237625122,
      "rewards/margins": 1.4880059957504272,
      "rewards/rejected": -3.241938352584839,
      "step": 5810
    },
    {
      "epoch": 4.656,
      "grad_norm": 27.08865737915039,
      "learning_rate": 3.226588486751012e-06,
      "logits/chosen": 1.0157322883605957,
      "logits/rejected": 0.9247297644615173,
      "logps/chosen": -198.93539428710938,
      "logps/rejected": -235.2766876220703,
      "loss": 0.37,
      "rewards/accuracies": 0.8375000357627869,
      "rewards/chosen": -1.8956657648086548,
      "rewards/margins": 1.5729421377182007,
      "rewards/rejected": -3.4686081409454346,
      "step": 5820
    },
    {
      "epoch": 4.664,
      "grad_norm": 13.346570014953613,
      "learning_rate": 3.2199057017751822e-06,
      "logits/chosen": 1.0085042715072632,
      "logits/rejected": 0.9250786900520325,
      "logps/chosen": -216.81689453125,
      "logps/rejected": -223.42991638183594,
      "loss": 0.3252,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.240125060081482,
      "rewards/margins": 1.7455374002456665,
      "rewards/rejected": -2.9856624603271484,
      "step": 5830
    },
    {
      "epoch": 4.672,
      "grad_norm": 13.748871803283691,
      "learning_rate": 3.2132173028300756e-06,
      "logits/chosen": 1.0077823400497437,
      "logits/rejected": 0.9494333267211914,
      "logps/chosen": -220.22952270507812,
      "logps/rejected": -239.5417938232422,
      "loss": 0.4106,
      "rewards/accuracies": 0.8375000357627869,
      "rewards/chosen": -1.271531343460083,
      "rewards/margins": 1.3013349771499634,
      "rewards/rejected": -2.572866201400757,
      "step": 5840
    },
    {
      "epoch": 4.68,
      "grad_norm": 11.705482482910156,
      "learning_rate": 3.2065233420731717e-06,
      "logits/chosen": 0.9431636929512024,
      "logits/rejected": 0.945948600769043,
      "logps/chosen": -211.1238555908203,
      "logps/rejected": -250.669189453125,
      "loss": 0.3788,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.2726339101791382,
      "rewards/margins": 1.5259406566619873,
      "rewards/rejected": -2.798574447631836,
      "step": 5850
    },
    {
      "epoch": 4.688,
      "grad_norm": 19.19478416442871,
      "learning_rate": 3.1998238717053202e-06,
      "logits/chosen": 1.0462414026260376,
      "logits/rejected": 0.9334519505500793,
      "logps/chosen": -204.07395935058594,
      "logps/rejected": -229.44419860839844,
      "loss": 0.3411,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.3902432918548584,
      "rewards/margins": 1.6846592426300049,
      "rewards/rejected": -3.0749025344848633,
      "step": 5860
    },
    {
      "epoch": 4.696,
      "grad_norm": 16.826845169067383,
      "learning_rate": 3.1931189439703383e-06,
      "logits/chosen": 0.9771106839179993,
      "logits/rejected": 0.8640087246894836,
      "logps/chosen": -211.6616668701172,
      "logps/rejected": -238.89736938476562,
      "loss": 0.3763,
      "rewards/accuracies": 0.8375000357627869,
      "rewards/chosen": -1.612330436706543,
      "rewards/margins": 1.4654382467269897,
      "rewards/rejected": -3.0777688026428223,
      "step": 5870
    },
    {
      "epoch": 4.704,
      "grad_norm": 18.69582176208496,
      "learning_rate": 3.186408611154597e-06,
      "logits/chosen": 0.8807933926582336,
      "logits/rejected": 0.8436470031738281,
      "logps/chosen": -199.52122497558594,
      "logps/rejected": -235.47007751464844,
      "loss": 0.3434,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.1381405591964722,
      "rewards/margins": 1.7940791845321655,
      "rewards/rejected": -2.9322197437286377,
      "step": 5880
    },
    {
      "epoch": 4.712,
      "grad_norm": 13.82207202911377,
      "learning_rate": 3.1796929255866223e-06,
      "logits/chosen": 0.908224880695343,
      "logits/rejected": 0.8031226992607117,
      "logps/chosen": -226.13247680664062,
      "logps/rejected": -268.6899108886719,
      "loss": 0.2751,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.7007683515548706,
      "rewards/margins": 2.1727986335754395,
      "rewards/rejected": -3.8735668659210205,
      "step": 5890
    },
    {
      "epoch": 4.72,
      "grad_norm": 17.216737747192383,
      "learning_rate": 3.1729719396366765e-06,
      "logits/chosen": 0.8856326341629028,
      "logits/rejected": 0.8279878497123718,
      "logps/chosen": -211.522216796875,
      "logps/rejected": -239.03599548339844,
      "loss": 0.3723,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.6209534406661987,
      "rewards/margins": 1.6722420454025269,
      "rewards/rejected": -3.2931954860687256,
      "step": 5900
    },
    {
      "epoch": 4.728,
      "grad_norm": 11.951440811157227,
      "learning_rate": 3.1662457057163603e-06,
      "logits/chosen": 0.9104948043823242,
      "logits/rejected": 0.80853271484375,
      "logps/chosen": -216.1103057861328,
      "logps/rejected": -228.3678741455078,
      "loss": 0.3353,
      "rewards/accuracies": 0.9000000357627869,
      "rewards/chosen": -1.3812806606292725,
      "rewards/margins": 1.7015780210494995,
      "rewards/rejected": -3.0828585624694824,
      "step": 5910
    },
    {
      "epoch": 4.736,
      "grad_norm": 18.6155948638916,
      "learning_rate": 3.1595142762781966e-06,
      "logits/chosen": 1.020864486694336,
      "logits/rejected": 0.9599675536155701,
      "logps/chosen": -213.35317993164062,
      "logps/rejected": -227.1990203857422,
      "loss": 0.3485,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.3319700956344604,
      "rewards/margins": 1.7788978815078735,
      "rewards/rejected": -3.110867738723755,
      "step": 5920
    },
    {
      "epoch": 4.744,
      "grad_norm": 15.233264923095703,
      "learning_rate": 3.1527777038152237e-06,
      "logits/chosen": 0.99336177110672,
      "logits/rejected": 0.9045869708061218,
      "logps/chosen": -225.9204559326172,
      "logps/rejected": -246.9542999267578,
      "loss": 0.2477,
      "rewards/accuracies": 0.9000000357627869,
      "rewards/chosen": -1.912535548210144,
      "rewards/margins": 2.0346450805664062,
      "rewards/rejected": -3.947180986404419,
      "step": 5930
    },
    {
      "epoch": 4.752,
      "grad_norm": 8.648079872131348,
      "learning_rate": 3.1460360408605866e-06,
      "logits/chosen": 0.778138279914856,
      "logits/rejected": 0.8619017004966736,
      "logps/chosen": -206.7869873046875,
      "logps/rejected": -250.70361328125,
      "loss": 0.2783,
      "rewards/accuracies": 0.9000000357627869,
      "rewards/chosen": -1.4929323196411133,
      "rewards/margins": 1.780416488647461,
      "rewards/rejected": -3.2733490467071533,
      "step": 5940
    },
    {
      "epoch": 4.76,
      "grad_norm": 13.389790534973145,
      "learning_rate": 3.1392893399871294e-06,
      "logits/chosen": 0.8986393213272095,
      "logits/rejected": 0.8389616012573242,
      "logps/chosen": -233.67100524902344,
      "logps/rejected": -222.7420196533203,
      "loss": 0.427,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -1.2816507816314697,
      "rewards/margins": 1.3874958753585815,
      "rewards/rejected": -2.6691465377807617,
      "step": 5950
    },
    {
      "epoch": 4.768,
      "grad_norm": 13.679242134094238,
      "learning_rate": 3.132537653806978e-06,
      "logits/chosen": 0.9529990553855896,
      "logits/rejected": 0.9898123741149902,
      "logps/chosen": -206.0071258544922,
      "logps/rejected": -235.917236328125,
      "loss": 0.489,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -1.56574547290802,
      "rewards/margins": 1.2233296632766724,
      "rewards/rejected": -2.7890751361846924,
      "step": 5960
    },
    {
      "epoch": 4.776,
      "grad_norm": 25.058631896972656,
      "learning_rate": 3.1257810349711388e-06,
      "logits/chosen": 1.1274060010910034,
      "logits/rejected": 0.9430057406425476,
      "logps/chosen": -229.041259765625,
      "logps/rejected": -240.43399047851562,
      "loss": 0.3475,
      "rewards/accuracies": 0.8375000357627869,
      "rewards/chosen": -1.2477760314941406,
      "rewards/margins": 1.6116218566894531,
      "rewards/rejected": -2.8593976497650146,
      "step": 5970
    },
    {
      "epoch": 4.784,
      "grad_norm": 26.500873565673828,
      "learning_rate": 3.1190195361690833e-06,
      "logits/chosen": 1.0664939880371094,
      "logits/rejected": 0.9565810561180115,
      "logps/chosen": -206.330078125,
      "logps/rejected": -240.9081573486328,
      "loss": 0.3886,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.549399971961975,
      "rewards/margins": 1.5753132104873657,
      "rewards/rejected": -3.1247129440307617,
      "step": 5980
    },
    {
      "epoch": 4.792,
      "grad_norm": 8.312376976013184,
      "learning_rate": 3.1122532101283366e-06,
      "logits/chosen": 0.9260608553886414,
      "logits/rejected": 0.8606094717979431,
      "logps/chosen": -221.0975341796875,
      "logps/rejected": -267.6355285644531,
      "loss": 0.3165,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.3144680261611938,
      "rewards/margins": 1.7448300123214722,
      "rewards/rejected": -3.059298038482666,
      "step": 5990
    },
    {
      "epoch": 4.8,
      "grad_norm": 9.584151268005371,
      "learning_rate": 3.1054821096140675e-06,
      "logits/chosen": 1.0778309106826782,
      "logits/rejected": 0.8670400977134705,
      "logps/chosen": -201.90811157226562,
      "logps/rejected": -216.34109497070312,
      "loss": 0.2893,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.7194448709487915,
      "rewards/margins": 1.7727628946304321,
      "rewards/rejected": -2.4922077655792236,
      "step": 6000
    },
    {
      "epoch": 4.808,
      "grad_norm": 10.907683372497559,
      "learning_rate": 3.0987062874286805e-06,
      "logits/chosen": 0.895865261554718,
      "logits/rejected": 0.9385225176811218,
      "logps/chosen": -206.30361938476562,
      "logps/rejected": -236.09593200683594,
      "loss": 0.2717,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7000702023506165,
      "rewards/margins": 2.0624783039093018,
      "rewards/rejected": -2.7625484466552734,
      "step": 6010
    },
    {
      "epoch": 4.816,
      "grad_norm": 20.199901580810547,
      "learning_rate": 3.0919257964113962e-06,
      "logits/chosen": 0.9721388220787048,
      "logits/rejected": 0.9084480404853821,
      "logps/chosen": -205.85423278808594,
      "logps/rejected": -226.93325805664062,
      "loss": 0.4159,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.0990513563156128,
      "rewards/margins": 1.5490108728408813,
      "rewards/rejected": -2.648062229156494,
      "step": 6020
    },
    {
      "epoch": 4.824,
      "grad_norm": 13.252838134765625,
      "learning_rate": 3.085140689437846e-06,
      "logits/chosen": 1.1252657175064087,
      "logits/rejected": 1.0671288967132568,
      "logps/chosen": -192.517333984375,
      "logps/rejected": -223.1140594482422,
      "loss": 0.285,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.5657386779785156,
      "rewards/margins": 2.0705223083496094,
      "rewards/rejected": -2.636261224746704,
      "step": 6030
    },
    {
      "epoch": 4.832,
      "grad_norm": 16.6810359954834,
      "learning_rate": 3.0783510194196577e-06,
      "logits/chosen": 1.0133423805236816,
      "logits/rejected": 0.9432527422904968,
      "logps/chosen": -212.550537109375,
      "logps/rejected": -248.9680633544922,
      "loss": 0.3551,
      "rewards/accuracies": 0.8375000357627869,
      "rewards/chosen": -1.3131626844406128,
      "rewards/margins": 1.684877634048462,
      "rewards/rejected": -2.998040199279785,
      "step": 6040
    },
    {
      "epoch": 4.84,
      "grad_norm": 12.429515838623047,
      "learning_rate": 3.0715568393040405e-06,
      "logits/chosen": 0.997822105884552,
      "logits/rejected": 0.9970841407775879,
      "logps/chosen": -186.12039184570312,
      "logps/rejected": -226.0194854736328,
      "loss": 0.2044,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5419926047325134,
      "rewards/margins": 2.209963321685791,
      "rewards/rejected": -2.75195574760437,
      "step": 6050
    },
    {
      "epoch": 4.848,
      "grad_norm": 10.648442268371582,
      "learning_rate": 3.0647582020733773e-06,
      "logits/chosen": 0.8079277873039246,
      "logits/rejected": 0.8897015452384949,
      "logps/chosen": -228.3728485107422,
      "logps/rejected": -263.2770690917969,
      "loss": 0.2747,
      "rewards/accuracies": 0.9000000357627869,
      "rewards/chosen": -1.4080426692962646,
      "rewards/margins": 2.117032051086426,
      "rewards/rejected": -3.5250747203826904,
      "step": 6060
    },
    {
      "epoch": 4.856,
      "grad_norm": 34.41978073120117,
      "learning_rate": 3.0579551607448064e-06,
      "logits/chosen": 0.9691902995109558,
      "logits/rejected": 0.8717222213745117,
      "logps/chosen": -229.96450805664062,
      "logps/rejected": -267.5755920410156,
      "loss": 0.3608,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.6460784673690796,
      "rewards/margins": 1.8111892938613892,
      "rewards/rejected": -3.4572677612304688,
      "step": 6070
    },
    {
      "epoch": 4.864,
      "grad_norm": 10.431986808776855,
      "learning_rate": 3.051147768369811e-06,
      "logits/chosen": 0.8802177309989929,
      "logits/rejected": 0.8142924308776855,
      "logps/chosen": -208.6748046875,
      "logps/rejected": -240.06729125976562,
      "loss": 0.2808,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.5522571802139282,
      "rewards/margins": 2.0125110149383545,
      "rewards/rejected": -3.5647683143615723,
      "step": 6080
    },
    {
      "epoch": 4.872,
      "grad_norm": 15.201266288757324,
      "learning_rate": 3.0443360780338034e-06,
      "logits/chosen": 0.8416620492935181,
      "logits/rejected": 0.8180571794509888,
      "logps/chosen": -210.61656188964844,
      "logps/rejected": -238.749267578125,
      "loss": 0.4195,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -1.1236116886138916,
      "rewards/margins": 1.5696989297866821,
      "rewards/rejected": -2.693310499191284,
      "step": 6090
    },
    {
      "epoch": 4.88,
      "grad_norm": 13.592103004455566,
      "learning_rate": 3.0375201428557135e-06,
      "logits/chosen": 0.9775605201721191,
      "logits/rejected": 0.9477388262748718,
      "logps/chosen": -211.87046813964844,
      "logps/rejected": -223.9567413330078,
      "loss": 0.3774,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.3752092123031616,
      "rewards/margins": 1.6053650379180908,
      "rewards/rejected": -2.980574369430542,
      "step": 6100
    },
    {
      "epoch": 4.888,
      "grad_norm": 18.705928802490234,
      "learning_rate": 3.0307000159875733e-06,
      "logits/chosen": 1.0056161880493164,
      "logits/rejected": 1.0067530870437622,
      "logps/chosen": -217.2322235107422,
      "logps/rejected": -247.19590759277344,
      "loss": 0.3999,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.5061392784118652,
      "rewards/margins": 1.4345601797103882,
      "rewards/rejected": -2.940699338912964,
      "step": 6110
    },
    {
      "epoch": 4.896,
      "grad_norm": 25.736547470092773,
      "learning_rate": 3.0238757506141013e-06,
      "logits/chosen": 0.9335775375366211,
      "logits/rejected": 0.8118970990180969,
      "logps/chosen": -203.6862030029297,
      "logps/rejected": -248.3121337890625,
      "loss": 0.3255,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.9543191194534302,
      "rewards/margins": 2.1874382495880127,
      "rewards/rejected": -3.1417574882507324,
      "step": 6120
    },
    {
      "epoch": 4.904,
      "grad_norm": 36.35403823852539,
      "learning_rate": 3.0170473999522914e-06,
      "logits/chosen": 0.7437816262245178,
      "logits/rejected": 0.7528760433197021,
      "logps/chosen": -203.16519165039062,
      "logps/rejected": -233.81167602539062,
      "loss": 0.3915,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3550660610198975,
      "rewards/margins": 1.738660454750061,
      "rewards/rejected": -3.093726396560669,
      "step": 6130
    },
    {
      "epoch": 4.912,
      "grad_norm": 21.394716262817383,
      "learning_rate": 3.010215017250993e-06,
      "logits/chosen": 0.8748787045478821,
      "logits/rejected": 0.8038078546524048,
      "logps/chosen": -190.148681640625,
      "logps/rejected": -229.1069793701172,
      "loss": 0.3333,
      "rewards/accuracies": 0.8375000357627869,
      "rewards/chosen": -1.0847818851470947,
      "rewards/margins": 1.952123999595642,
      "rewards/rejected": -3.0369060039520264,
      "step": 6140
    },
    {
      "epoch": 4.92,
      "grad_norm": 10.547325134277344,
      "learning_rate": 3.0033786557904982e-06,
      "logits/chosen": 0.9673800468444824,
      "logits/rejected": 0.8164333701133728,
      "logps/chosen": -215.92520141601562,
      "logps/rejected": -247.7021942138672,
      "loss": 0.385,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.3397104740142822,
      "rewards/margins": 1.9476795196533203,
      "rewards/rejected": -3.2873897552490234,
      "step": 6150
    },
    {
      "epoch": 4.928,
      "grad_norm": 8.210875511169434,
      "learning_rate": 2.996538368882127e-06,
      "logits/chosen": 1.024520754814148,
      "logits/rejected": 0.8471878170967102,
      "logps/chosen": -226.50840759277344,
      "logps/rejected": -246.0098114013672,
      "loss": 0.2765,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8774771690368652,
      "rewards/margins": 1.9889500141143799,
      "rewards/rejected": -2.866427183151245,
      "step": 6160
    },
    {
      "epoch": 4.936,
      "grad_norm": 11.18367862701416,
      "learning_rate": 2.9896942098678124e-06,
      "logits/chosen": 0.9735992550849915,
      "logits/rejected": 0.981691300868988,
      "logps/chosen": -192.818359375,
      "logps/rejected": -243.27273559570312,
      "loss": 0.3407,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.7151866555213928,
      "rewards/margins": 1.898476481437683,
      "rewards/rejected": -2.6136631965637207,
      "step": 6170
    },
    {
      "epoch": 4.944,
      "grad_norm": 22.406173706054688,
      "learning_rate": 2.982846232119679e-06,
      "logits/chosen": 1.0046285390853882,
      "logits/rejected": 0.9979419708251953,
      "logps/chosen": -216.0959014892578,
      "logps/rejected": -244.8262176513672,
      "loss": 0.324,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.145372986793518,
      "rewards/margins": 1.6784679889678955,
      "rewards/rejected": -2.823841094970703,
      "step": 6180
    },
    {
      "epoch": 4.952,
      "grad_norm": 28.244096755981445,
      "learning_rate": 2.975994489039634e-06,
      "logits/chosen": 0.9947078824043274,
      "logits/rejected": 0.91530841588974,
      "logps/chosen": -208.3612823486328,
      "logps/rejected": -234.4157257080078,
      "loss": 0.3708,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.6478198766708374,
      "rewards/margins": 1.7686901092529297,
      "rewards/rejected": -3.4165101051330566,
      "step": 6190
    },
    {
      "epoch": 4.96,
      "grad_norm": 18.052692413330078,
      "learning_rate": 2.9691390340589467e-06,
      "logits/chosen": 1.0676629543304443,
      "logits/rejected": 0.9271091818809509,
      "logps/chosen": -199.14598083496094,
      "logps/rejected": -222.794921875,
      "loss": 0.2456,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8989327549934387,
      "rewards/margins": 2.075803518295288,
      "rewards/rejected": -2.974736213684082,
      "step": 6200
    },
    {
      "epoch": 4.968,
      "grad_norm": 10.866455078125,
      "learning_rate": 2.9622799206378306e-06,
      "logits/chosen": 1.0321582555770874,
      "logits/rejected": 0.9904826283454895,
      "logps/chosen": -201.66383361816406,
      "logps/rejected": -241.1051788330078,
      "loss": 0.3023,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.0962049961090088,
      "rewards/margins": 1.7505791187286377,
      "rewards/rejected": -2.8467838764190674,
      "step": 6210
    },
    {
      "epoch": 4.976,
      "grad_norm": 14.606306076049805,
      "learning_rate": 2.955417202265032e-06,
      "logits/chosen": 1.0466164350509644,
      "logits/rejected": 0.9499098062515259,
      "logps/chosen": -206.0836639404297,
      "logps/rejected": -229.9392852783203,
      "loss": 0.3209,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.0380445718765259,
      "rewards/margins": 1.6832345724105835,
      "rewards/rejected": -2.7212791442871094,
      "step": 6220
    },
    {
      "epoch": 4.984,
      "grad_norm": 27.189558029174805,
      "learning_rate": 2.948550932457407e-06,
      "logits/chosen": 0.9810224771499634,
      "logits/rejected": 0.8776025176048279,
      "logps/chosen": -220.196044921875,
      "logps/rejected": -241.2982940673828,
      "loss": 0.344,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.4501808881759644,
      "rewards/margins": 1.7209481000900269,
      "rewards/rejected": -3.171128988265991,
      "step": 6230
    },
    {
      "epoch": 4.992,
      "grad_norm": 13.859283447265625,
      "learning_rate": 2.9416811647595052e-06,
      "logits/chosen": 1.027130365371704,
      "logits/rejected": 0.9805015921592712,
      "logps/chosen": -207.3785858154297,
      "logps/rejected": -245.68896484375,
      "loss": 0.2176,
      "rewards/accuracies": 0.9625000357627869,
      "rewards/chosen": -1.786203384399414,
      "rewards/margins": 2.202639579772949,
      "rewards/rejected": -3.9888432025909424,
      "step": 6240
    },
    {
      "epoch": 5.0,
      "grad_norm": 27.184709548950195,
      "learning_rate": 2.9348079527431565e-06,
      "logits/chosen": 0.9774474501609802,
      "logits/rejected": 0.830977737903595,
      "logps/chosen": -207.9765625,
      "logps/rejected": -246.947021484375,
      "loss": 0.2871,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.9358631372451782,
      "rewards/margins": 2.2705037593841553,
      "rewards/rejected": -4.206367492675781,
      "step": 6250
    },
    {
      "epoch": 5.008,
      "grad_norm": 9.864638328552246,
      "learning_rate": 2.927931350007048e-06,
      "logits/chosen": 0.9456281661987305,
      "logits/rejected": 0.8694238662719727,
      "logps/chosen": -221.7121124267578,
      "logps/rejected": -256.9190979003906,
      "loss": 0.2611,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.0195796489715576,
      "rewards/margins": 2.1393039226531982,
      "rewards/rejected": -4.158883571624756,
      "step": 6260
    },
    {
      "epoch": 5.016,
      "grad_norm": 8.061872482299805,
      "learning_rate": 2.9210514101763116e-06,
      "logits/chosen": 0.9081168174743652,
      "logits/rejected": 0.8007159233093262,
      "logps/chosen": -208.5377197265625,
      "logps/rejected": -260.0176696777344,
      "loss": 0.2097,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.110689878463745,
      "rewards/margins": 2.2806320190429688,
      "rewards/rejected": -4.391321659088135,
      "step": 6270
    },
    {
      "epoch": 5.024,
      "grad_norm": 13.147801399230957,
      "learning_rate": 2.9141681869020973e-06,
      "logits/chosen": 0.8848690390586853,
      "logits/rejected": 0.9038190245628357,
      "logps/chosen": -234.75083923339844,
      "logps/rejected": -270.18408203125,
      "loss": 0.2374,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.0579333305358887,
      "rewards/margins": 2.2107951641082764,
      "rewards/rejected": -4.268728733062744,
      "step": 6280
    },
    {
      "epoch": 5.032,
      "grad_norm": 22.0900936126709,
      "learning_rate": 2.907281733861164e-06,
      "logits/chosen": 0.8925933241844177,
      "logits/rejected": 0.8973062634468079,
      "logps/chosen": -206.32763671875,
      "logps/rejected": -228.326904296875,
      "loss": 0.3404,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.4018090963363647,
      "rewards/margins": 1.735507607460022,
      "rewards/rejected": -3.1373164653778076,
      "step": 6290
    },
    {
      "epoch": 5.04,
      "grad_norm": 8.617568016052246,
      "learning_rate": 2.900392104755455e-06,
      "logits/chosen": 0.8251405954360962,
      "logits/rejected": 0.7466452121734619,
      "logps/chosen": -231.0389862060547,
      "logps/rejected": -279.979248046875,
      "loss": 0.261,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.3339290618896484,
      "rewards/margins": 2.2454123497009277,
      "rewards/rejected": -4.579341411590576,
      "step": 6300
    },
    {
      "epoch": 5.048,
      "grad_norm": 10.349010467529297,
      "learning_rate": 2.8934993533116827e-06,
      "logits/chosen": 0.8101547360420227,
      "logits/rejected": 0.8393245935440063,
      "logps/chosen": -218.5235137939453,
      "logps/rejected": -276.7334899902344,
      "loss": 0.2239,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.8703062534332275,
      "rewards/margins": 2.4195773601531982,
      "rewards/rejected": -4.289883613586426,
      "step": 6310
    },
    {
      "epoch": 5.056,
      "grad_norm": 6.812648296356201,
      "learning_rate": 2.8866035332809083e-06,
      "logits/chosen": 0.9793133735656738,
      "logits/rejected": 0.9104774594306946,
      "logps/chosen": -209.8477783203125,
      "logps/rejected": -235.04031372070312,
      "loss": 0.2581,
      "rewards/accuracies": 0.9000000357627869,
      "rewards/chosen": -1.3933300971984863,
      "rewards/margins": 2.483262777328491,
      "rewards/rejected": -3.8765931129455566,
      "step": 6320
    },
    {
      "epoch": 5.064,
      "grad_norm": 13.762728691101074,
      "learning_rate": 2.879704698438121e-06,
      "logits/chosen": 0.9087037444114685,
      "logits/rejected": 0.890673816204071,
      "logps/chosen": -208.9591064453125,
      "logps/rejected": -263.70452880859375,
      "loss": 0.2147,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.8164809942245483,
      "rewards/margins": 2.4334583282470703,
      "rewards/rejected": -4.24993896484375,
      "step": 6330
    },
    {
      "epoch": 5.072,
      "grad_norm": 27.724454879760742,
      "learning_rate": 2.8728029025818206e-06,
      "logits/chosen": 0.9317230582237244,
      "logits/rejected": 0.7435727119445801,
      "logps/chosen": -212.3452911376953,
      "logps/rejected": -262.7144470214844,
      "loss": 0.2844,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.7303894758224487,
      "rewards/margins": 2.263822317123413,
      "rewards/rejected": -3.9942119121551514,
      "step": 6340
    },
    {
      "epoch": 5.08,
      "grad_norm": 23.111251831054688,
      "learning_rate": 2.865898199533597e-06,
      "logits/chosen": 0.9201246500015259,
      "logits/rejected": 0.8896446228027344,
      "logps/chosen": -209.3739013671875,
      "logps/rejected": -240.9625244140625,
      "loss": 0.2962,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.9742488861083984,
      "rewards/margins": 1.8627338409423828,
      "rewards/rejected": -3.8369832038879395,
      "step": 6350
    },
    {
      "epoch": 5.088,
      "grad_norm": 24.608367919921875,
      "learning_rate": 2.8589906431377133e-06,
      "logits/chosen": 1.0965173244476318,
      "logits/rejected": 0.9428312182426453,
      "logps/chosen": -223.93748474121094,
      "logps/rejected": -240.9108428955078,
      "loss": 0.2826,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.867902398109436,
      "rewards/margins": 2.019596815109253,
      "rewards/rejected": -3.8874988555908203,
      "step": 6360
    },
    {
      "epoch": 5.096,
      "grad_norm": 14.740994453430176,
      "learning_rate": 2.8520802872606803e-06,
      "logits/chosen": 0.9774947166442871,
      "logits/rejected": 0.9799107909202576,
      "logps/chosen": -204.7219696044922,
      "logps/rejected": -231.45262145996094,
      "loss": 0.3067,
      "rewards/accuracies": 0.9000000357627869,
      "rewards/chosen": -1.5889536142349243,
      "rewards/margins": 2.0691580772399902,
      "rewards/rejected": -3.658111572265625,
      "step": 6370
    },
    {
      "epoch": 5.104,
      "grad_norm": 9.468008995056152,
      "learning_rate": 2.8451671857908414e-06,
      "logits/chosen": 0.9921843409538269,
      "logits/rejected": 0.9002591371536255,
      "logps/chosen": -198.7250213623047,
      "logps/rejected": -225.64834594726562,
      "loss": 0.2472,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.3202179670333862,
      "rewards/margins": 2.2721245288848877,
      "rewards/rejected": -3.5923423767089844,
      "step": 6380
    },
    {
      "epoch": 5.112,
      "grad_norm": 14.54666805267334,
      "learning_rate": 2.8382513926379508e-06,
      "logits/chosen": 0.9203723073005676,
      "logits/rejected": 0.8950970768928528,
      "logps/chosen": -196.84657287597656,
      "logps/rejected": -230.43809509277344,
      "loss": 0.2501,
      "rewards/accuracies": 0.9625000357627869,
      "rewards/chosen": -1.5130068063735962,
      "rewards/margins": 2.144577741622925,
      "rewards/rejected": -3.6575844287872314,
      "step": 6390
    },
    {
      "epoch": 5.12,
      "grad_norm": 27.838401794433594,
      "learning_rate": 2.831332961732754e-06,
      "logits/chosen": 1.0219676494598389,
      "logits/rejected": 0.9032443165779114,
      "logps/chosen": -212.4551544189453,
      "logps/rejected": -241.2015838623047,
      "loss": 0.2542,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.7502228021621704,
      "rewards/margins": 2.144639015197754,
      "rewards/rejected": -3.894861936569214,
      "step": 6400
    },
    {
      "epoch": 5.128,
      "grad_norm": 23.901247024536133,
      "learning_rate": 2.8244119470265628e-06,
      "logits/chosen": 0.9221467971801758,
      "logits/rejected": 0.844575822353363,
      "logps/chosen": -216.648681640625,
      "logps/rejected": -226.3651123046875,
      "loss": 0.3867,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8910839557647705,
      "rewards/margins": 2.042074203491211,
      "rewards/rejected": -3.9331586360931396,
      "step": 6410
    },
    {
      "epoch": 5.136,
      "grad_norm": 28.365636825561523,
      "learning_rate": 2.817488402490841e-06,
      "logits/chosen": 0.9146007895469666,
      "logits/rejected": 0.9036192297935486,
      "logps/chosen": -217.9712371826172,
      "logps/rejected": -246.8628387451172,
      "loss": 0.2528,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.148552417755127,
      "rewards/margins": 2.226809024810791,
      "rewards/rejected": -4.375361919403076,
      "step": 6420
    },
    {
      "epoch": 5.144,
      "grad_norm": 11.069653511047363,
      "learning_rate": 2.8105623821167804e-06,
      "logits/chosen": 0.9730092883110046,
      "logits/rejected": 0.8130871057510376,
      "logps/chosen": -223.188720703125,
      "logps/rejected": -259.5501403808594,
      "loss": 0.2751,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.180257558822632,
      "rewards/margins": 2.317899465560913,
      "rewards/rejected": -4.498157024383545,
      "step": 6430
    },
    {
      "epoch": 5.152,
      "grad_norm": 4.842264652252197,
      "learning_rate": 2.8036339399148783e-06,
      "logits/chosen": 0.8939194083213806,
      "logits/rejected": 0.8858678936958313,
      "logps/chosen": -210.72207641601562,
      "logps/rejected": -263.359375,
      "loss": 0.2706,
      "rewards/accuracies": 0.9000000357627869,
      "rewards/chosen": -1.6614614725112915,
      "rewards/margins": 2.150581121444702,
      "rewards/rejected": -3.812042713165283,
      "step": 6440
    },
    {
      "epoch": 5.16,
      "grad_norm": 15.502528190612793,
      "learning_rate": 2.796703129914519e-06,
      "logits/chosen": 0.9737855792045593,
      "logits/rejected": 0.8153099417686462,
      "logps/chosen": -200.01934814453125,
      "logps/rejected": -228.4278106689453,
      "loss": 0.2503,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.3171756267547607,
      "rewards/margins": 2.079169988632202,
      "rewards/rejected": -3.396345615386963,
      "step": 6450
    },
    {
      "epoch": 5.168,
      "grad_norm": 13.45697021484375,
      "learning_rate": 2.7897700061635517e-06,
      "logits/chosen": 0.9658388495445251,
      "logits/rejected": 0.8410293459892273,
      "logps/chosen": -197.5312957763672,
      "logps/rejected": -234.99929809570312,
      "loss": 0.2299,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.5396157503128052,
      "rewards/margins": 2.2552897930145264,
      "rewards/rejected": -3.794905424118042,
      "step": 6460
    },
    {
      "epoch": 5.176,
      "grad_norm": 21.632190704345703,
      "learning_rate": 2.7828346227278676e-06,
      "logits/chosen": 0.8654441833496094,
      "logits/rejected": 0.8274869322776794,
      "logps/chosen": -220.19541931152344,
      "logps/rejected": -281.8578796386719,
      "loss": 0.3081,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.2533280849456787,
      "rewards/margins": 2.436194658279419,
      "rewards/rejected": -4.689522743225098,
      "step": 6470
    },
    {
      "epoch": 5.184,
      "grad_norm": 23.60213851928711,
      "learning_rate": 2.7758970336909795e-06,
      "logits/chosen": 0.9322023391723633,
      "logits/rejected": 0.8540251851081848,
      "logps/chosen": -226.8634796142578,
      "logps/rejected": -249.7890625,
      "loss": 0.3004,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.6639741659164429,
      "rewards/margins": 2.1177260875701904,
      "rewards/rejected": -3.781700611114502,
      "step": 6480
    },
    {
      "epoch": 5.192,
      "grad_norm": 8.637389183044434,
      "learning_rate": 2.768957293153602e-06,
      "logits/chosen": 1.001732587814331,
      "logits/rejected": 1.0790340900421143,
      "logps/chosen": -212.6641845703125,
      "logps/rejected": -248.3900604248047,
      "loss": 0.2027,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.9972444772720337,
      "rewards/margins": 2.5673165321350098,
      "rewards/rejected": -4.564560890197754,
      "step": 6490
    },
    {
      "epoch": 5.2,
      "grad_norm": 17.306949615478516,
      "learning_rate": 2.7620154552332236e-06,
      "logits/chosen": 1.0369716882705688,
      "logits/rejected": 0.8845782279968262,
      "logps/chosen": -210.2711944580078,
      "logps/rejected": -248.135009765625,
      "loss": 0.2221,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.953507423400879,
      "rewards/margins": 2.378849506378174,
      "rewards/rejected": -4.332356929779053,
      "step": 6500
    },
    {
      "epoch": 5.208,
      "grad_norm": 20.987321853637695,
      "learning_rate": 2.755071574063692e-06,
      "logits/chosen": 0.9889179468154907,
      "logits/rejected": 0.9481157660484314,
      "logps/chosen": -209.94509887695312,
      "logps/rejected": -241.16763305664062,
      "loss": 0.192,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.9956191778182983,
      "rewards/margins": 2.235707998275757,
      "rewards/rejected": -4.231327056884766,
      "step": 6510
    },
    {
      "epoch": 5.216,
      "grad_norm": 22.150367736816406,
      "learning_rate": 2.7481257037947873e-06,
      "logits/chosen": 1.0156019926071167,
      "logits/rejected": 0.8769356608390808,
      "logps/chosen": -219.45596313476562,
      "logps/rejected": -244.5421905517578,
      "loss": 0.2559,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.335376262664795,
      "rewards/margins": 2.1717984676361084,
      "rewards/rejected": -3.5071747303009033,
      "step": 6520
    },
    {
      "epoch": 5.224,
      "grad_norm": 24.29526138305664,
      "learning_rate": 2.741177898591801e-06,
      "logits/chosen": 1.0322328805923462,
      "logits/rejected": 0.9097422957420349,
      "logps/chosen": -215.4120635986328,
      "logps/rejected": -245.8870849609375,
      "loss": 0.2705,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.2170920372009277,
      "rewards/margins": 2.0834217071533203,
      "rewards/rejected": -4.300513744354248,
      "step": 6530
    },
    {
      "epoch": 5.232,
      "grad_norm": 11.549744606018066,
      "learning_rate": 2.7342282126351145e-06,
      "logits/chosen": 1.0487924814224243,
      "logits/rejected": 0.9894416928291321,
      "logps/chosen": -229.5109405517578,
      "logps/rejected": -263.1656494140625,
      "loss": 0.2757,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.495918035507202,
      "rewards/margins": 2.1933634281158447,
      "rewards/rejected": -4.689281463623047,
      "step": 6540
    },
    {
      "epoch": 5.24,
      "grad_norm": 11.867500305175781,
      "learning_rate": 2.727276700119774e-06,
      "logits/chosen": 1.0762747526168823,
      "logits/rejected": 0.9830428957939148,
      "logps/chosen": -216.1927490234375,
      "logps/rejected": -268.2547302246094,
      "loss": 0.3535,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.7991743087768555,
      "rewards/margins": 2.384692430496216,
      "rewards/rejected": -5.183866500854492,
      "step": 6550
    },
    {
      "epoch": 5.248,
      "grad_norm": 12.038691520690918,
      "learning_rate": 2.720323415255071e-06,
      "logits/chosen": 1.0917648077011108,
      "logits/rejected": 1.0066547393798828,
      "logps/chosen": -216.46788024902344,
      "logps/rejected": -242.0109405517578,
      "loss": 0.2506,
      "rewards/accuracies": 0.9000000357627869,
      "rewards/chosen": -2.0974338054656982,
      "rewards/margins": 2.069669723510742,
      "rewards/rejected": -4.1671037673950195,
      "step": 6560
    },
    {
      "epoch": 5.256,
      "grad_norm": 16.80321502685547,
      "learning_rate": 2.713368412264118e-06,
      "logits/chosen": 0.9983375668525696,
      "logits/rejected": 0.8874215483665466,
      "logps/chosen": -205.94091796875,
      "logps/rejected": -261.5921936035156,
      "loss": 0.2188,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.691676139831543,
      "rewards/margins": 2.6781671047210693,
      "rewards/rejected": -4.369843482971191,
      "step": 6570
    },
    {
      "epoch": 5.264,
      "grad_norm": 45.896324157714844,
      "learning_rate": 2.7064117453834245e-06,
      "logits/chosen": 1.0483051538467407,
      "logits/rejected": 0.9006147384643555,
      "logps/chosen": -196.8784637451172,
      "logps/rejected": -240.83731079101562,
      "loss": 0.2838,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.3554176092147827,
      "rewards/margins": 2.4722535610198975,
      "rewards/rejected": -3.8276710510253906,
      "step": 6580
    },
    {
      "epoch": 5.272,
      "grad_norm": 52.870967864990234,
      "learning_rate": 2.699453468862477e-06,
      "logits/chosen": 0.8586319088935852,
      "logits/rejected": 0.8957145810127258,
      "logps/chosen": -202.1999053955078,
      "logps/rejected": -247.98497009277344,
      "loss": 0.3508,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.5604242086410522,
      "rewards/margins": 2.2796757221221924,
      "rewards/rejected": -3.840100049972534,
      "step": 6590
    },
    {
      "epoch": 5.28,
      "grad_norm": 15.097871780395508,
      "learning_rate": 2.6924936369633126e-06,
      "logits/chosen": 0.9903114438056946,
      "logits/rejected": 0.8972232937812805,
      "logps/chosen": -220.2637176513672,
      "logps/rejected": -246.101318359375,
      "loss": 0.2577,
      "rewards/accuracies": 0.9000000357627869,
      "rewards/chosen": -1.400626540184021,
      "rewards/margins": 2.2727909088134766,
      "rewards/rejected": -3.673417806625366,
      "step": 6600
    },
    {
      "epoch": 5.288,
      "grad_norm": 30.0377140045166,
      "learning_rate": 2.6855323039601e-06,
      "logits/chosen": 0.9595162272453308,
      "logits/rejected": 0.9324502944946289,
      "logps/chosen": -217.57749938964844,
      "logps/rejected": -249.253662109375,
      "loss": 0.2962,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.08235502243042,
      "rewards/margins": 2.116223096847534,
      "rewards/rejected": -4.198577880859375,
      "step": 6610
    },
    {
      "epoch": 5.296,
      "grad_norm": 35.225372314453125,
      "learning_rate": 2.678569524138711e-06,
      "logits/chosen": 1.0795904397964478,
      "logits/rejected": 0.9872816205024719,
      "logps/chosen": -217.6615753173828,
      "logps/rejected": -247.40611267089844,
      "loss": 0.2785,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.6414073705673218,
      "rewards/margins": 2.288874387741089,
      "rewards/rejected": -3.9302818775177,
      "step": 6620
    },
    {
      "epoch": 5.304,
      "grad_norm": 19.5015926361084,
      "learning_rate": 2.671605351796302e-06,
      "logits/chosen": 1.0191433429718018,
      "logits/rejected": 1.0426959991455078,
      "logps/chosen": -215.02542114257812,
      "logps/rejected": -250.69419860839844,
      "loss": 0.2692,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.4523265361785889,
      "rewards/margins": 2.202432632446289,
      "rewards/rejected": -3.654759168624878,
      "step": 6630
    },
    {
      "epoch": 5.312,
      "grad_norm": 13.902155876159668,
      "learning_rate": 2.664639841240888e-06,
      "logits/chosen": 1.0611361265182495,
      "logits/rejected": 1.0298746824264526,
      "logps/chosen": -210.43655395507812,
      "logps/rejected": -242.15591430664062,
      "loss": 0.2278,
      "rewards/accuracies": 0.9625000357627869,
      "rewards/chosen": -1.0366039276123047,
      "rewards/margins": 2.3139262199401855,
      "rewards/rejected": -3.3505303859710693,
      "step": 6640
    },
    {
      "epoch": 5.32,
      "grad_norm": 19.404386520385742,
      "learning_rate": 2.6576730467909202e-06,
      "logits/chosen": 0.9389854669570923,
      "logits/rejected": 0.8830668330192566,
      "logps/chosen": -211.813720703125,
      "logps/rejected": -255.224365234375,
      "loss": 0.2227,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.772042155265808,
      "rewards/margins": 2.3123362064361572,
      "rewards/rejected": -4.084378719329834,
      "step": 6650
    },
    {
      "epoch": 5.328,
      "grad_norm": 19.177539825439453,
      "learning_rate": 2.6507050227748595e-06,
      "logits/chosen": 0.9689971804618835,
      "logits/rejected": 0.8664621710777283,
      "logps/chosen": -218.7672882080078,
      "logps/rejected": -242.08872985839844,
      "loss": 0.2906,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.9733574390411377,
      "rewards/margins": 2.0987534523010254,
      "rewards/rejected": -4.072110652923584,
      "step": 6660
    },
    {
      "epoch": 5.336,
      "grad_norm": 19.155794143676758,
      "learning_rate": 2.6437358235307574e-06,
      "logits/chosen": 0.9650457501411438,
      "logits/rejected": 0.9216805696487427,
      "logps/chosen": -223.25161743164062,
      "logps/rejected": -267.2557373046875,
      "loss": 0.2284,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.293266773223877,
      "rewards/margins": 2.462886095046997,
      "rewards/rejected": -4.756152629852295,
      "step": 6670
    },
    {
      "epoch": 5.344,
      "grad_norm": 39.01654815673828,
      "learning_rate": 2.6367655034058302e-06,
      "logits/chosen": 0.9830673336982727,
      "logits/rejected": 0.9171612858772278,
      "logps/chosen": -213.6517791748047,
      "logps/rejected": -247.17857360839844,
      "loss": 0.1726,
      "rewards/accuracies": 0.9625000357627869,
      "rewards/chosen": -1.5426089763641357,
      "rewards/margins": 2.6942059993743896,
      "rewards/rejected": -4.236814975738525,
      "step": 6680
    },
    {
      "epoch": 5.352,
      "grad_norm": 18.91624641418457,
      "learning_rate": 2.629794116756035e-06,
      "logits/chosen": 1.1095460653305054,
      "logits/rejected": 1.073819875717163,
      "logps/chosen": -196.78050231933594,
      "logps/rejected": -221.8975067138672,
      "loss": 0.2172,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.370259165763855,
      "rewards/margins": 2.258117198944092,
      "rewards/rejected": -3.628376007080078,
      "step": 6690
    },
    {
      "epoch": 5.36,
      "grad_norm": 10.549464225769043,
      "learning_rate": 2.6228217179456433e-06,
      "logits/chosen": 0.9206059575080872,
      "logits/rejected": 0.8385736346244812,
      "logps/chosen": -229.06982421875,
      "logps/rejected": -266.5553894042969,
      "loss": 0.2318,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.381563663482666,
      "rewards/margins": 2.3572521209716797,
      "rewards/rejected": -4.7388153076171875,
      "step": 6700
    },
    {
      "epoch": 5.368,
      "grad_norm": 18.678146362304688,
      "learning_rate": 2.6158483613468227e-06,
      "logits/chosen": 0.8933264017105103,
      "logits/rejected": 0.9182813763618469,
      "logps/chosen": -246.1497039794922,
      "logps/rejected": -265.3963317871094,
      "loss": 0.2916,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.833988666534424,
      "rewards/margins": 2.2055633068084717,
      "rewards/rejected": -5.039552211761475,
      "step": 6710
    },
    {
      "epoch": 5.376,
      "grad_norm": 15.417664527893066,
      "learning_rate": 2.60887410133921e-06,
      "logits/chosen": 1.0603265762329102,
      "logits/rejected": 0.9646215438842773,
      "logps/chosen": -213.02725219726562,
      "logps/rejected": -248.04296875,
      "loss": 0.1764,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.2449893951416016,
      "rewards/margins": 2.7219924926757812,
      "rewards/rejected": -4.966982364654541,
      "step": 6720
    },
    {
      "epoch": 5.384,
      "grad_norm": 16.097373962402344,
      "learning_rate": 2.6018989923094827e-06,
      "logits/chosen": 1.0397377014160156,
      "logits/rejected": 1.0830873250961304,
      "logps/chosen": -216.1459197998047,
      "logps/rejected": -231.951904296875,
      "loss": 0.2511,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.4918582439422607,
      "rewards/margins": 2.454404354095459,
      "rewards/rejected": -4.946262359619141,
      "step": 6730
    },
    {
      "epoch": 5.392,
      "grad_norm": 15.579238891601562,
      "learning_rate": 2.594923088650946e-06,
      "logits/chosen": 1.000435709953308,
      "logits/rejected": 1.0281956195831299,
      "logps/chosen": -225.9973907470703,
      "logps/rejected": -261.2825012207031,
      "loss": 0.2612,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.4454987049102783,
      "rewards/margins": 2.452462911605835,
      "rewards/rejected": -3.897961378097534,
      "step": 6740
    },
    {
      "epoch": 5.4,
      "grad_norm": 23.80678367614746,
      "learning_rate": 2.5879464447630947e-06,
      "logits/chosen": 0.9257308840751648,
      "logits/rejected": 0.9335107207298279,
      "logps/chosen": -205.0893096923828,
      "logps/rejected": -268.3067932128906,
      "loss": 0.2616,
      "rewards/accuracies": 0.9000000357627869,
      "rewards/chosen": -1.4560121297836304,
      "rewards/margins": 2.8784759044647217,
      "rewards/rejected": -4.3344879150390625,
      "step": 6750
    },
    {
      "epoch": 5.408,
      "grad_norm": 14.877538681030273,
      "learning_rate": 2.5809691150512013e-06,
      "logits/chosen": 1.0292404890060425,
      "logits/rejected": 0.8708595633506775,
      "logps/chosen": -230.5414581298828,
      "logps/rejected": -267.4679870605469,
      "loss": 0.2617,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.2800981998443604,
      "rewards/margins": 2.2236454486846924,
      "rewards/rejected": -4.503743648529053,
      "step": 6760
    },
    {
      "epoch": 5.416,
      "grad_norm": 30.09552764892578,
      "learning_rate": 2.573991153925883e-06,
      "logits/chosen": 0.9744608998298645,
      "logits/rejected": 0.9204347729682922,
      "logps/chosen": -230.64749145507812,
      "logps/rejected": -238.489990234375,
      "loss": 0.2917,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.851356863975525,
      "rewards/margins": 2.2804553508758545,
      "rewards/rejected": -4.131812572479248,
      "step": 6770
    },
    {
      "epoch": 5.424,
      "grad_norm": 9.124749183654785,
      "learning_rate": 2.5670126158026843e-06,
      "logits/chosen": 0.9641228914260864,
      "logits/rejected": 0.9122328758239746,
      "logps/chosen": -209.06993103027344,
      "logps/rejected": -239.84507751464844,
      "loss": 0.1824,
      "rewards/accuracies": 0.9625000357627869,
      "rewards/chosen": -1.1467351913452148,
      "rewards/margins": 2.435342788696289,
      "rewards/rejected": -3.582077741622925,
      "step": 6780
    },
    {
      "epoch": 5.432,
      "grad_norm": 13.851131439208984,
      "learning_rate": 2.5600335551016447e-06,
      "logits/chosen": 1.2253344058990479,
      "logits/rejected": 1.1556156873703003,
      "logps/chosen": -200.3877410888672,
      "logps/rejected": -242.9773712158203,
      "loss": 0.2018,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.4336289167404175,
      "rewards/margins": 2.616694450378418,
      "rewards/rejected": -4.050323486328125,
      "step": 6790
    },
    {
      "epoch": 5.44,
      "grad_norm": 15.476871490478516,
      "learning_rate": 2.553054026246884e-06,
      "logits/chosen": 1.0492868423461914,
      "logits/rejected": 0.955453097820282,
      "logps/chosen": -229.8250732421875,
      "logps/rejected": -254.77793884277344,
      "loss": 0.2157,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.6499685049057007,
      "rewards/margins": 2.502528429031372,
      "rewards/rejected": -4.152497291564941,
      "step": 6800
    },
    {
      "epoch": 5.448,
      "grad_norm": 17.754188537597656,
      "learning_rate": 2.546074083666169e-06,
      "logits/chosen": 1.1146057844161987,
      "logits/rejected": 1.0327948331832886,
      "logps/chosen": -196.68882751464844,
      "logps/rejected": -219.73692321777344,
      "loss": 0.2566,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.2834064960479736,
      "rewards/margins": 2.399442434310913,
      "rewards/rejected": -3.6828484535217285,
      "step": 6810
    },
    {
      "epoch": 5.456,
      "grad_norm": 14.777560234069824,
      "learning_rate": 2.539093781790494e-06,
      "logits/chosen": 1.1234928369522095,
      "logits/rejected": 0.8710097670555115,
      "logps/chosen": -229.3218231201172,
      "logps/rejected": -263.53204345703125,
      "loss": 0.3007,
      "rewards/accuracies": 0.9000000357627869,
      "rewards/chosen": -2.53562593460083,
      "rewards/margins": 2.2426531314849854,
      "rewards/rejected": -4.778278827667236,
      "step": 6820
    },
    {
      "epoch": 5.464,
      "grad_norm": 11.484394073486328,
      "learning_rate": 2.5321131750536548e-06,
      "logits/chosen": 1.0820449590682983,
      "logits/rejected": 1.0439467430114746,
      "logps/chosen": -210.1967315673828,
      "logps/rejected": -252.2407684326172,
      "loss": 0.2437,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.7417097091674805,
      "rewards/margins": 2.493793487548828,
      "rewards/rejected": -4.235503673553467,
      "step": 6830
    },
    {
      "epoch": 5.4719999999999995,
      "grad_norm": 7.133698463439941,
      "learning_rate": 2.525132317891827e-06,
      "logits/chosen": 0.9405183792114258,
      "logits/rejected": 0.7907823920249939,
      "logps/chosen": -223.71621704101562,
      "logps/rejected": -250.6181182861328,
      "loss": 0.309,
      "rewards/accuracies": 0.9000000357627869,
      "rewards/chosen": -2.273303747177124,
      "rewards/margins": 2.0772006511688232,
      "rewards/rejected": -4.350503444671631,
      "step": 6840
    },
    {
      "epoch": 5.48,
      "grad_norm": 36.052452087402344,
      "learning_rate": 2.518151264743135e-06,
      "logits/chosen": 1.0576660633087158,
      "logits/rejected": 0.9024376273155212,
      "logps/chosen": -219.67166137695312,
      "logps/rejected": -256.7233581542969,
      "loss": 0.2635,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.318809986114502,
      "rewards/margins": 2.248960494995117,
      "rewards/rejected": -4.567770481109619,
      "step": 6850
    },
    {
      "epoch": 5.4879999999999995,
      "grad_norm": 22.618186950683594,
      "learning_rate": 2.5111700700472346e-06,
      "logits/chosen": 1.0195873975753784,
      "logits/rejected": 0.735422670841217,
      "logps/chosen": -220.07212829589844,
      "logps/rejected": -274.3692932128906,
      "loss": 0.3001,
      "rewards/accuracies": 0.9000000357627869,
      "rewards/chosen": -2.25004243850708,
      "rewards/margins": 2.4187867641448975,
      "rewards/rejected": -4.668828964233398,
      "step": 6860
    },
    {
      "epoch": 5.496,
      "grad_norm": 14.920022964477539,
      "learning_rate": 2.5041887882448845e-06,
      "logits/chosen": 0.9542160034179688,
      "logits/rejected": 0.9047889709472656,
      "logps/chosen": -217.98045349121094,
      "logps/rejected": -259.5525817871094,
      "loss": 0.2509,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.3205878734588623,
      "rewards/margins": 2.485825777053833,
      "rewards/rejected": -4.806413173675537,
      "step": 6870
    },
    {
      "epoch": 5.504,
      "grad_norm": 27.746845245361328,
      "learning_rate": 2.4972074737775215e-06,
      "logits/chosen": 0.8094510436058044,
      "logits/rejected": 0.820503830909729,
      "logps/chosen": -235.88784790039062,
      "logps/rejected": -269.9020690917969,
      "loss": 0.2144,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.0258710384368896,
      "rewards/margins": 2.428048610687256,
      "rewards/rejected": -5.453919410705566,
      "step": 6880
    },
    {
      "epoch": 5.5120000000000005,
      "grad_norm": 38.07632064819336,
      "learning_rate": 2.490226181086838e-06,
      "logits/chosen": 1.0112783908843994,
      "logits/rejected": 0.9033260345458984,
      "logps/chosen": -205.05882263183594,
      "logps/rejected": -243.261962890625,
      "loss": 0.2399,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.0925614833831787,
      "rewards/margins": 2.7473719120025635,
      "rewards/rejected": -4.839933395385742,
      "step": 6890
    },
    {
      "epoch": 5.52,
      "grad_norm": 35.062713623046875,
      "learning_rate": 2.4832449646143605e-06,
      "logits/chosen": 1.0022151470184326,
      "logits/rejected": 0.9039661288261414,
      "logps/chosen": -211.7960968017578,
      "logps/rejected": -265.2066345214844,
      "loss": 0.3377,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.1607472896575928,
      "rewards/margins": 2.2616782188415527,
      "rewards/rejected": -4.422425746917725,
      "step": 6900
    },
    {
      "epoch": 5.5280000000000005,
      "grad_norm": 21.59208869934082,
      "learning_rate": 2.4762638788010123e-06,
      "logits/chosen": 0.8795046210289001,
      "logits/rejected": 0.8476438522338867,
      "logps/chosen": -229.28501892089844,
      "logps/rejected": -292.383544921875,
      "loss": 0.283,
      "rewards/accuracies": 0.9000000357627869,
      "rewards/chosen": -2.4365899562835693,
      "rewards/margins": 2.430114984512329,
      "rewards/rejected": -4.86670446395874,
      "step": 6910
    },
    {
      "epoch": 5.536,
      "grad_norm": 33.235862731933594,
      "learning_rate": 2.4692829780867066e-06,
      "logits/chosen": 0.9138635993003845,
      "logits/rejected": 0.8161695599555969,
      "logps/chosen": -233.41311645507812,
      "logps/rejected": -261.68975830078125,
      "loss": 0.2933,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.5699150562286377,
      "rewards/margins": 2.5199315547943115,
      "rewards/rejected": -4.089846611022949,
      "step": 6920
    },
    {
      "epoch": 5.5440000000000005,
      "grad_norm": 28.37091064453125,
      "learning_rate": 2.4623023169099074e-06,
      "logits/chosen": 1.0121078491210938,
      "logits/rejected": 0.8701547980308533,
      "logps/chosen": -226.9220733642578,
      "logps/rejected": -260.5491638183594,
      "loss": 0.2636,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.4063045978546143,
      "rewards/margins": 2.338066577911377,
      "rewards/rejected": -4.744370937347412,
      "step": 6930
    },
    {
      "epoch": 5.552,
      "grad_norm": 17.56963348388672,
      "learning_rate": 2.4553219497072144e-06,
      "logits/chosen": 0.9864139556884766,
      "logits/rejected": 0.9286383986473083,
      "logps/chosen": -224.4748992919922,
      "logps/rejected": -247.63027954101562,
      "loss": 0.2675,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.1812076568603516,
      "rewards/margins": 2.474334716796875,
      "rewards/rejected": -4.655541896820068,
      "step": 6940
    },
    {
      "epoch": 5.5600000000000005,
      "grad_norm": 37.597293853759766,
      "learning_rate": 2.4483419309129315e-06,
      "logits/chosen": 0.8857519030570984,
      "logits/rejected": 0.9429466128349304,
      "logps/chosen": -202.9930877685547,
      "logps/rejected": -244.86172485351562,
      "loss": 0.2741,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.7118009328842163,
      "rewards/margins": 2.3302266597747803,
      "rewards/rejected": -4.042027473449707,
      "step": 6950
    },
    {
      "epoch": 5.568,
      "grad_norm": 22.82415008544922,
      "learning_rate": 2.441362314958649e-06,
      "logits/chosen": 1.015781283378601,
      "logits/rejected": 0.8939239382743835,
      "logps/chosen": -197.0980987548828,
      "logps/rejected": -247.9208526611328,
      "loss": 0.2189,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.4986218214035034,
      "rewards/margins": 2.6701772212982178,
      "rewards/rejected": -4.168798923492432,
      "step": 6960
    },
    {
      "epoch": 5.576,
      "grad_norm": 30.6041259765625,
      "learning_rate": 2.4343831562728135e-06,
      "logits/chosen": 0.8862255215644836,
      "logits/rejected": 0.8734444975852966,
      "logps/chosen": -238.130859375,
      "logps/rejected": -281.5000915527344,
      "loss": 0.3029,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.5411267280578613,
      "rewards/margins": 2.136220693588257,
      "rewards/rejected": -4.677347660064697,
      "step": 6970
    },
    {
      "epoch": 5.584,
      "grad_norm": 14.699413299560547,
      "learning_rate": 2.4274045092803056e-06,
      "logits/chosen": 0.9806970953941345,
      "logits/rejected": 0.8635124564170837,
      "logps/chosen": -230.31900024414062,
      "logps/rejected": -276.25421142578125,
      "loss": 0.241,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.257481098175049,
      "rewards/margins": 2.4393088817596436,
      "rewards/rejected": -4.696789741516113,
      "step": 6980
    },
    {
      "epoch": 5.592,
      "grad_norm": 26.546295166015625,
      "learning_rate": 2.4204264284020182e-06,
      "logits/chosen": 1.0598602294921875,
      "logits/rejected": 1.0161948204040527,
      "logps/chosen": -208.1858367919922,
      "logps/rejected": -228.87876892089844,
      "loss": 0.3018,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.463453769683838,
      "rewards/margins": 2.1223933696746826,
      "rewards/rejected": -3.5858471393585205,
      "step": 6990
    },
    {
      "epoch": 5.6,
      "grad_norm": 13.010027885437012,
      "learning_rate": 2.4134489680544263e-06,
      "logits/chosen": 1.0219472646713257,
      "logits/rejected": 0.916050910949707,
      "logps/chosen": -237.35302734375,
      "logps/rejected": -259.0421447753906,
      "loss": 0.1984,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.8679497241973877,
      "rewards/margins": 2.485231399536133,
      "rewards/rejected": -4.3531813621521,
      "step": 7000
    }
  ],
  "logging_steps": 10,
  "max_steps": 12500,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 10,
  "save_steps": 1000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 1.0023474241339392e+18,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}