{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.9871794871794872,
  "eval_steps": 500,
  "global_step": 699,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.021367521367521368,
      "grad_norm": 16.125,
      "learning_rate": 1.7556090538745385e-06,
      "logits/chosen": -3.5723183155059814,
      "logits/rejected": -3.5035064220428467,
      "logps/chosen": -41.11209487915039,
      "logps/rejected": -79.84024047851562,
      "loss": 0.693,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.00025137339252978563,
      "rewards/margins": 0.0003745006979443133,
      "rewards/rejected": -0.0006258742650970817,
      "step": 5
    },
    {
      "epoch": 0.042735042735042736,
      "grad_norm": 16.625,
      "learning_rate": 3.950120371217711e-06,
      "logits/chosen": -3.5866427421569824,
      "logits/rejected": -3.506498336791992,
      "logps/chosen": -40.02031707763672,
      "logps/rejected": -78.2303237915039,
      "loss": 0.6804,
      "rewards/accuracies": 0.8083333969116211,
      "rewards/chosen": -0.0030814227648079395,
      "rewards/margins": 0.025840703397989273,
      "rewards/rejected": -0.028922125697135925,
      "step": 10
    },
    {
      "epoch": 0.0641025641025641,
      "grad_norm": 14.75,
      "learning_rate": 6.144631688560886e-06,
      "logits/chosen": -3.5990729331970215,
      "logits/rejected": -3.507888078689575,
      "logps/chosen": -39.841556549072266,
      "logps/rejected": -81.43814086914062,
      "loss": 0.6304,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.007047838531434536,
      "rewards/margins": 0.131792351603508,
      "rewards/rejected": -0.1388401836156845,
      "step": 15
    },
    {
      "epoch": 0.08547008547008547,
      "grad_norm": 13.6875,
      "learning_rate": 8.339143005904057e-06,
      "logits/chosen": -3.5527968406677246,
      "logits/rejected": -3.4919159412384033,
      "logps/chosen": -39.75408935546875,
      "logps/rejected": -80.72636413574219,
      "loss": 0.5433,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.014718407765030861,
      "rewards/margins": 0.33949846029281616,
      "rewards/rejected": -0.35421687364578247,
      "step": 20
    },
    {
      "epoch": 0.10683760683760683,
      "grad_norm": 11.1875,
      "learning_rate": 1.0533654323247232e-05,
      "logits/chosen": -3.5133519172668457,
      "logits/rejected": -3.472460985183716,
      "logps/chosen": -40.00214385986328,
      "logps/rejected": -85.55389404296875,
      "loss": 0.396,
      "rewards/accuracies": 0.98333340883255,
      "rewards/chosen": -0.041852615773677826,
      "rewards/margins": 0.7774692177772522,
      "rewards/rejected": -0.8193218111991882,
      "step": 25
    },
    {
      "epoch": 0.1282051282051282,
      "grad_norm": 7.59375,
      "learning_rate": 1.2728165640590407e-05,
      "logits/chosen": -3.4411749839782715,
      "logits/rejected": -3.445669651031494,
      "logps/chosen": -42.452301025390625,
      "logps/rejected": -93.04366302490234,
      "loss": 0.259,
      "rewards/accuracies": 0.9958332777023315,
      "rewards/chosen": -0.13290514051914215,
      "rewards/margins": 1.4268956184387207,
      "rewards/rejected": -1.5598008632659912,
      "step": 30
    },
    {
      "epoch": 0.14957264957264957,
      "grad_norm": 4.25,
      "learning_rate": 1.4922676957933578e-05,
      "logits/chosen": -3.2215240001678467,
      "logits/rejected": -3.2669529914855957,
      "logps/chosen": -44.15301513671875,
      "logps/rejected": -108.67581939697266,
      "loss": 0.1196,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4359213709831238,
      "rewards/margins": 2.7526631355285645,
      "rewards/rejected": -3.1885838508605957,
      "step": 35
    },
    {
      "epoch": 0.17094017094017094,
      "grad_norm": 2.046875,
      "learning_rate": 1.5360556888469565e-05,
      "logits/chosen": -2.778581142425537,
      "logits/rejected": -2.812112331390381,
      "logps/chosen": -52.46855926513672,
      "logps/rejected": -143.13113403320312,
      "loss": 0.0425,
      "rewards/accuracies": 0.9916666746139526,
      "rewards/chosen": -1.2584294080734253,
      "rewards/margins": 5.229200839996338,
      "rewards/rejected": -6.4876298904418945,
      "step": 40
    },
    {
      "epoch": 0.19230769230769232,
      "grad_norm": 1.6875,
      "learning_rate": 1.535640428282884e-05,
      "logits/chosen": -2.3370110988616943,
      "logits/rejected": -2.2969162464141846,
      "logps/chosen": -51.32811737060547,
      "logps/rejected": -161.91485595703125,
      "loss": 0.0166,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -1.2713468074798584,
      "rewards/margins": 7.2617645263671875,
      "rewards/rejected": -8.533111572265625,
      "step": 45
    },
    {
      "epoch": 0.21367521367521367,
      "grad_norm": 9.8125,
      "learning_rate": 1.5349059809872097e-05,
      "logits/chosen": -1.9922136068344116,
      "logits/rejected": -1.9316717386245728,
      "logps/chosen": -57.087364196777344,
      "logps/rejected": -180.17538452148438,
      "loss": 0.0198,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -1.6679397821426392,
      "rewards/margins": 8.57298755645752,
      "rewards/rejected": -10.240926742553711,
      "step": 50
    },
    {
      "epoch": 0.23504273504273504,
      "grad_norm": 0.71484375,
      "learning_rate": 1.5338527542732884e-05,
      "logits/chosen": -1.988929033279419,
      "logits/rejected": -1.921478271484375,
      "logps/chosen": -58.5947380065918,
      "logps/rejected": -196.42152404785156,
      "loss": 0.0097,
      "rewards/accuracies": 0.9958332777023315,
      "rewards/chosen": -1.8699321746826172,
      "rewards/margins": 9.876224517822266,
      "rewards/rejected": -11.746156692504883,
      "step": 55
    },
    {
      "epoch": 0.2564102564102564,
      "grad_norm": 3.0625,
      "learning_rate": 1.532481332244717e-05,
      "logits/chosen": -1.9385712146759033,
      "logits/rejected": -1.8249061107635498,
      "logps/chosen": -57.23170852661133,
      "logps/rejected": -198.87722778320312,
      "loss": 0.0046,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.809206247329712,
      "rewards/margins": 10.275887489318848,
      "rewards/rejected": -12.08509349822998,
      "step": 60
    },
    {
      "epoch": 0.2777777777777778,
      "grad_norm": 8.625,
      "learning_rate": 1.5307924754713968e-05,
      "logits/chosen": -1.939802885055542,
      "logits/rejected": -1.8658767938613892,
      "logps/chosen": -59.283775329589844,
      "logps/rejected": -196.01666259765625,
      "loss": 0.0104,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -1.8633003234863281,
      "rewards/margins": 9.977038383483887,
      "rewards/rejected": -11.840339660644531,
      "step": 65
    },
    {
      "epoch": 0.29914529914529914,
      "grad_norm": 0.32421875,
      "learning_rate": 1.528787120567736e-05,
      "logits/chosen": -2.0353214740753174,
      "logits/rejected": -1.9403060674667358,
      "logps/chosen": -53.66817092895508,
      "logps/rejected": -192.67019653320312,
      "loss": 0.0032,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.436279058456421,
      "rewards/margins": 10.137629508972168,
      "rewards/rejected": -11.573908805847168,
      "step": 70
    },
    {
      "epoch": 0.32051282051282054,
      "grad_norm": 0.33203125,
      "learning_rate": 1.526466379673215e-05,
      "logits/chosen": -2.051652669906616,
      "logits/rejected": -1.9418226480484009,
      "logps/chosen": -55.503456115722656,
      "logps/rejected": -201.16795349121094,
      "loss": 0.006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6412267684936523,
      "rewards/margins": 10.790639877319336,
      "rewards/rejected": -12.431867599487305,
      "step": 75
    },
    {
      "epoch": 0.3418803418803419,
      "grad_norm": 6.375,
      "learning_rate": 1.5238315398356126e-05,
      "logits/chosen": -2.026411533355713,
      "logits/rejected": -1.8895785808563232,
      "logps/chosen": -58.959983825683594,
      "logps/rejected": -205.6489715576172,
      "loss": 0.0054,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.9467172622680664,
      "rewards/margins": 11.003195762634277,
      "rewards/rejected": -12.949913024902344,
      "step": 80
    },
    {
      "epoch": 0.36324786324786323,
      "grad_norm": 0.032958984375,
      "learning_rate": 1.5208840622972272e-05,
      "logits/chosen": -2.0241971015930176,
      "logits/rejected": -1.8603122234344482,
      "logps/chosen": -59.0450553894043,
      "logps/rejected": -208.4755401611328,
      "loss": 0.023,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -1.7697633504867554,
      "rewards/margins": 11.253450393676758,
      "rewards/rejected": -13.023214340209961,
      "step": 85
    },
    {
      "epoch": 0.38461538461538464,
      "grad_norm": 1.203125,
      "learning_rate": 1.5176255816844948e-05,
      "logits/chosen": -1.9871257543563843,
      "logits/rejected": -1.795332670211792,
      "logps/chosen": -50.853111267089844,
      "logps/rejected": -205.7463836669922,
      "loss": 0.0063,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1833254098892212,
      "rewards/margins": 11.639432907104492,
      "rewards/rejected": -12.82275676727295,
      "step": 90
    },
    {
      "epoch": 0.405982905982906,
      "grad_norm": 0.59765625,
      "learning_rate": 1.5140579051014502e-05,
      "logits/chosen": -1.9575443267822266,
      "logits/rejected": -1.730380654335022,
      "logps/chosen": -55.5305290222168,
      "logps/rejected": -221.403564453125,
      "loss": 0.0028,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.54853355884552,
      "rewards/margins": 12.512056350708008,
      "rewards/rejected": -14.060590744018555,
      "step": 95
    },
    {
      "epoch": 0.42735042735042733,
      "grad_norm": 0.107421875,
      "learning_rate": 1.5101830111275334e-05,
      "logits/chosen": -1.9054193496704102,
      "logits/rejected": -1.7034189701080322,
      "logps/chosen": -59.12446975708008,
      "logps/rejected": -213.0598907470703,
      "loss": 0.0117,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -1.9219242334365845,
      "rewards/margins": 11.585506439208984,
      "rewards/rejected": -13.507431030273438,
      "step": 100
    },
    {
      "epoch": 0.44871794871794873,
      "grad_norm": 3.484375,
      "learning_rate": 1.5060030487203004e-05,
      "logits/chosen": -1.84870183467865,
      "logits/rejected": -1.5906012058258057,
      "logps/chosen": -61.794334411621094,
      "logps/rejected": -229.24472045898438,
      "loss": 0.0058,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -2.169433116912842,
      "rewards/margins": 12.846372604370117,
      "rewards/rejected": -15.015806198120117,
      "step": 105
    },
    {
      "epoch": 0.4700854700854701,
      "grad_norm": 0.0888671875,
      "learning_rate": 1.501520336023643e-05,
      "logits/chosen": -1.8036330938339233,
      "logits/rejected": -1.4892421960830688,
      "logps/chosen": -63.50853729248047,
      "logps/rejected": -225.1219940185547,
      "loss": 0.0011,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.3367650508880615,
      "rewards/margins": 12.470510482788086,
      "rewards/rejected": -14.807273864746094,
      "step": 110
    },
    {
      "epoch": 0.49145299145299143,
      "grad_norm": 2.203125,
      "learning_rate": 1.4967373590821828e-05,
      "logits/chosen": -1.77732253074646,
      "logits/rejected": -1.4749488830566406,
      "logps/chosen": -69.56774139404297,
      "logps/rejected": -229.52078247070312,
      "loss": 0.0192,
      "rewards/accuracies": 0.9916666746139526,
      "rewards/chosen": -2.8969478607177734,
      "rewards/margins": 12.228403091430664,
      "rewards/rejected": -15.125350952148438,
      "step": 115
    },
    {
      "epoch": 0.5128205128205128,
      "grad_norm": 0.08935546875,
      "learning_rate": 1.491656770462546e-05,
      "logits/chosen": -1.6314146518707275,
      "logits/rejected": -1.2666215896606445,
      "logps/chosen": -61.828941345214844,
      "logps/rejected": -231.4860076904297,
      "loss": 0.0026,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.1970794200897217,
      "rewards/margins": 13.084007263183594,
      "rewards/rejected": -15.281087875366211,
      "step": 120
    },
    {
      "epoch": 0.5341880341880342,
      "grad_norm": 9.5625,
      "learning_rate": 1.4862813877822923e-05,
      "logits/chosen": -1.695638656616211,
      "logits/rejected": -1.3275301456451416,
      "logps/chosen": -59.98823165893555,
      "logps/rejected": -228.39599609375,
      "loss": 0.0141,
      "rewards/accuracies": 0.9916666746139526,
      "rewards/chosen": -1.9660316705703735,
      "rewards/margins": 13.168081283569336,
      "rewards/rejected": -15.134114265441895,
      "step": 125
    },
    {
      "epoch": 0.5555555555555556,
      "grad_norm": 0.2021484375,
      "learning_rate": 1.4806141921473063e-05,
      "logits/chosen": -1.7461655139923096,
      "logits/rejected": -1.3385170698165894,
      "logps/chosen": -57.775604248046875,
      "logps/rejected": -232.9607391357422,
      "loss": 0.006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.762024164199829,
      "rewards/margins": 13.665751457214355,
      "rewards/rejected": -15.427775382995605,
      "step": 130
    },
    {
      "epoch": 0.5769230769230769,
      "grad_norm": 0.10791015625,
      "learning_rate": 1.4746583264985202e-05,
      "logits/chosen": -1.7675241231918335,
      "logits/rejected": -1.349183201789856,
      "logps/chosen": -59.05058670043945,
      "logps/rejected": -236.23599243164062,
      "loss": 0.0032,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.866996169090271,
      "rewards/margins": 13.846403121948242,
      "rewards/rejected": -15.713399887084961,
      "step": 135
    },
    {
      "epoch": 0.5982905982905983,
      "grad_norm": 0.029052734375,
      "learning_rate": 1.468417093868888e-05,
      "logits/chosen": -1.7844130992889404,
      "logits/rejected": -1.393346905708313,
      "logps/chosen": -55.1547966003418,
      "logps/rejected": -230.82485961914062,
      "loss": 0.0039,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -1.6007912158966064,
      "rewards/margins": 13.634336471557617,
      "rewards/rejected": -15.235128402709961,
      "step": 140
    },
    {
      "epoch": 0.6196581196581197,
      "grad_norm": 0.10888671875,
      "learning_rate": 1.4618939555515721e-05,
      "logits/chosen": -1.736817717552185,
      "logits/rejected": -1.3590004444122314,
      "logps/chosen": -57.30821990966797,
      "logps/rejected": -228.35568237304688,
      "loss": 0.0029,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.7177435159683228,
      "rewards/margins": 13.364781379699707,
      "rewards/rejected": -15.08252239227295,
      "step": 145
    },
    {
      "epoch": 0.6410256410256411,
      "grad_norm": 0.0126953125,
      "learning_rate": 1.455092529180363e-05,
      "logits/chosen": -1.767240285873413,
      "logits/rejected": -1.3727061748504639,
      "logps/chosen": -54.07440185546875,
      "logps/rejected": -230.85598754882812,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.538040041923523,
      "rewards/margins": 13.841748237609863,
      "rewards/rejected": -15.379789352416992,
      "step": 150
    },
    {
      "epoch": 0.6623931623931624,
      "grad_norm": 0.296875,
      "learning_rate": 1.4480165867233946e-05,
      "logits/chosen": -1.7429969310760498,
      "logits/rejected": -1.3700876235961914,
      "logps/chosen": -58.81968307495117,
      "logps/rejected": -237.78775024414062,
      "loss": 0.005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.9107345342636108,
      "rewards/margins": 13.975476264953613,
      "rewards/rejected": -15.886209487915039,
      "step": 155
    },
    {
      "epoch": 0.6837606837606838,
      "grad_norm": 3.71875,
      "learning_rate": 1.440670052391267e-05,
      "logits/chosen": -1.7467654943466187,
      "logits/rejected": -1.333251714706421,
      "logps/chosen": -55.909812927246094,
      "logps/rejected": -228.398681640625,
      "loss": 0.006,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -1.737717866897583,
      "rewards/margins": 13.500417709350586,
      "rewards/rejected": -15.238134384155273,
      "step": 160
    },
    {
      "epoch": 0.7051282051282052,
      "grad_norm": 2.0625,
      "learning_rate": 1.4330570004607398e-05,
      "logits/chosen": -1.8055540323257446,
      "logits/rejected": -1.365281343460083,
      "logps/chosen": -56.362159729003906,
      "logps/rejected": -241.47616577148438,
      "loss": 0.0033,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -1.7291532754898071,
      "rewards/margins": 14.622825622558594,
      "rewards/rejected": -16.351978302001953,
      "step": 165
    },
    {
      "epoch": 0.7264957264957265,
      "grad_norm": 0.71875,
      "learning_rate": 1.4251816530151986e-05,
      "logits/chosen": -1.7655203342437744,
      "logits/rejected": -1.3694696426391602,
      "logps/chosen": -59.91992950439453,
      "logps/rejected": -240.9622039794922,
      "loss": 0.0035,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.9651933908462524,
      "rewards/margins": 14.319523811340332,
      "rewards/rejected": -16.284717559814453,
      "step": 170
    },
    {
      "epoch": 0.7478632478632479,
      "grad_norm": 0.228515625,
      "learning_rate": 1.4170483776031526e-05,
      "logits/chosen": -1.705065369606018,
      "logits/rejected": -1.2570855617523193,
      "logps/chosen": -59.033119201660156,
      "logps/rejected": -244.8512725830078,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.9400125741958618,
      "rewards/margins": 14.696588516235352,
      "rewards/rejected": -16.636600494384766,
      "step": 175
    },
    {
      "epoch": 0.7692307692307693,
      "grad_norm": 0.048095703125,
      "learning_rate": 1.4086616848160574e-05,
      "logits/chosen": -1.6960264444351196,
      "logits/rejected": -1.2389576435089111,
      "logps/chosen": -62.79465866088867,
      "logps/rejected": -241.3025360107422,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.3021397590637207,
      "rewards/margins": 14.0509614944458,
      "rewards/rejected": -16.353099822998047,
      "step": 180
    },
    {
      "epoch": 0.7905982905982906,
      "grad_norm": 0.087890625,
      "learning_rate": 1.4000262257868096e-05,
      "logits/chosen": -1.665123701095581,
      "logits/rejected": -1.1488758325576782,
      "logps/chosen": -61.19834518432617,
      "logps/rejected": -244.506103515625,
      "loss": 0.0027,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.189235210418701,
      "rewards/margins": 14.604774475097656,
      "rewards/rejected": -16.794010162353516,
      "step": 185
    },
    {
      "epoch": 0.811965811965812,
      "grad_norm": 0.921875,
      "learning_rate": 1.3911467896102994e-05,
      "logits/chosen": -1.6246988773345947,
      "logits/rejected": -1.1719400882720947,
      "logps/chosen": -58.097412109375,
      "logps/rejected": -242.1889190673828,
      "loss": 0.0049,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -1.8747377395629883,
      "rewards/margins": 14.691726684570312,
      "rewards/rejected": -16.566465377807617,
      "step": 190
    },
    {
      "epoch": 0.8333333333333334,
      "grad_norm": 0.216796875,
      "learning_rate": 1.3820283006874503e-05,
      "logits/chosen": -1.6263649463653564,
      "logits/rejected": -1.1733753681182861,
      "logps/chosen": -64.73831939697266,
      "logps/rejected": -246.49752807617188,
      "loss": 0.0091,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -2.265516996383667,
      "rewards/margins": 14.604884147644043,
      "rewards/rejected": -16.87040138244629,
      "step": 195
    },
    {
      "epoch": 0.8547008547008547,
      "grad_norm": 0.1875,
      "learning_rate": 1.372675815994221e-05,
      "logits/chosen": -1.5483165979385376,
      "logits/rejected": -1.0878021717071533,
      "logps/chosen": -52.65502166748047,
      "logps/rejected": -248.62643432617188,
      "loss": 0.0008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5096328258514404,
      "rewards/margins": 15.69578742980957,
      "rewards/rejected": -17.205419540405273,
      "step": 200
    },
    {
      "epoch": 0.8760683760683761,
      "grad_norm": 9.5,
      "learning_rate": 1.3630945222770829e-05,
      "logits/chosen": -1.5609447956085205,
      "logits/rejected": -1.0415008068084717,
      "logps/chosen": -58.0806884765625,
      "logps/rejected": -255.28414916992188,
      "loss": 0.0182,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.7902355194091797,
      "rewards/margins": 15.908655166625977,
      "rewards/rejected": -17.69889259338379,
      "step": 205
    },
    {
      "epoch": 0.8974358974358975,
      "grad_norm": 1.6484375,
      "learning_rate": 1.3532897331765301e-05,
      "logits/chosen": -1.5497121810913086,
      "logits/rejected": -1.0044095516204834,
      "logps/chosen": -59.89592742919922,
      "logps/rejected": -257.2408752441406,
      "loss": 0.0162,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -1.991355538368225,
      "rewards/margins": 15.942468643188477,
      "rewards/rejected": -17.933826446533203,
      "step": 210
    },
    {
      "epoch": 0.9188034188034188,
      "grad_norm": 0.1884765625,
      "learning_rate": 1.3432668862802134e-05,
      "logits/chosen": -1.4721449613571167,
      "logits/rejected": -0.9380281567573547,
      "logps/chosen": -59.00041961669922,
      "logps/rejected": -255.982421875,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.8558237552642822,
      "rewards/margins": 15.840028762817383,
      "rewards/rejected": -17.695852279663086,
      "step": 215
    },
    {
      "epoch": 0.9401709401709402,
      "grad_norm": 0.1318359375,
      "learning_rate": 1.3330315401073371e-05,
      "logits/chosen": -1.4841619729995728,
      "logits/rejected": -0.9569031596183777,
      "logps/chosen": -60.60542678833008,
      "logps/rejected": -249.15353393554688,
      "loss": 0.0015,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.9671761989593506,
      "rewards/margins": 15.184603691101074,
      "rewards/rejected": -17.151779174804688,
      "step": 220
    },
    {
      "epoch": 0.9615384615384616,
      "grad_norm": 9.8125,
      "learning_rate": 1.3225893710259887e-05,
      "logits/chosen": -1.1225636005401611,
      "logits/rejected": -0.5225231051445007,
      "logps/chosen": -63.623390197753906,
      "logps/rejected": -252.66281127929688,
      "loss": 0.0095,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -2.4430124759674072,
      "rewards/margins": 15.111320495605469,
      "rewards/rejected": -17.554332733154297,
      "step": 225
    },
    {
      "epoch": 0.9829059829059829,
      "grad_norm": 12.875,
      "learning_rate": 1.3119461701051105e-05,
      "logits/chosen": -1.2154924869537354,
      "logits/rejected": -0.7296081781387329,
      "logps/chosen": -65.45369720458984,
      "logps/rejected": -248.72622680664062,
      "loss": 0.0116,
      "rewards/accuracies": 0.9916666746139526,
      "rewards/chosen": -2.7806317806243896,
      "rewards/margins": 14.623510360717773,
      "rewards/rejected": -17.40414047241211,
      "step": 230
    },
    {
      "epoch": 0.9957264957264957,
      "eval_logits/chosen": -1.554829716682434,
      "eval_logits/rejected": -0.9467828869819641,
      "eval_logps/chosen": -59.39779281616211,
      "eval_logps/rejected": -258.1591491699219,
      "eval_loss": 0.0002499581314623356,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -1.995406150817871,
      "eval_rewards/margins": 16.018821716308594,
      "eval_rewards/rejected": -18.01422882080078,
      "eval_runtime": 9.7352,
      "eval_samples_per_second": 20.544,
      "eval_steps_per_second": 20.544,
      "step": 233
    },
    {
      "epoch": 1.0042735042735043,
      "grad_norm": 0.030029296875,
      "learning_rate": 1.3011078399028605e-05,
      "logits/chosen": -1.5108505487442017,
      "logits/rejected": -0.9396005868911743,
      "logps/chosen": -60.448936462402344,
      "logps/rejected": -262.9476013183594,
      "loss": 0.0035,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -2.0872457027435303,
      "rewards/margins": 16.35396957397461,
      "rewards/rejected": -18.44121742248535,
      "step": 235
    },
    {
      "epoch": 1.0256410256410255,
      "grad_norm": 0.1025390625,
      "learning_rate": 1.2900803911931431e-05,
      "logits/chosen": -1.5595149993896484,
      "logits/rejected": -0.9638253450393677,
      "logps/chosen": -56.65764617919922,
      "logps/rejected": -258.0482482910156,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6903537511825562,
      "rewards/margins": 16.261157989501953,
      "rewards/rejected": -17.95151138305664,
      "step": 240
    },
    {
      "epoch": 1.047008547008547,
      "grad_norm": 0.022705078125,
      "learning_rate": 1.2788699396321252e-05,
      "logits/chosen": -1.520930290222168,
      "logits/rejected": -0.9344191551208496,
      "logps/chosen": -54.35309982299805,
      "logps/rejected": -249.8235321044922,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5775222778320312,
      "rewards/margins": 15.838638305664062,
      "rewards/rejected": -17.416160583496094,
      "step": 245
    },
    {
      "epoch": 1.0683760683760684,
      "grad_norm": 0.00031280517578125,
      "learning_rate": 1.2674827023665853e-05,
      "logits/chosen": -1.5657716989517212,
      "logits/rejected": -0.9992179870605469,
      "logps/chosen": -57.8154411315918,
      "logps/rejected": -263.98992919921875,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.8247381448745728,
      "rewards/margins": 16.810840606689453,
      "rewards/rejected": -18.635578155517578,
      "step": 250
    },
    {
      "epoch": 1.0897435897435896,
      "grad_norm": 0.1005859375,
      "learning_rate": 1.255924994585978e-05,
      "logits/chosen": -1.532013177871704,
      "logits/rejected": -0.9263051152229309,
      "logps/chosen": -60.24767303466797,
      "logps/rejected": -260.04595947265625,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.0023863315582275,
      "rewards/margins": 16.219375610351562,
      "rewards/rejected": -18.221763610839844,
      "step": 255
    },
    {
      "epoch": 1.1111111111111112,
      "grad_norm": 0.0054931640625,
      "learning_rate": 1.2442032260201255e-05,
      "logits/chosen": -1.5195871591567993,
      "logits/rejected": -0.9636222124099731,
      "logps/chosen": -59.14739990234375,
      "logps/rejected": -256.98883056640625,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.9601608514785767,
      "rewards/margins": 15.914080619812012,
      "rewards/rejected": -17.874242782592773,
      "step": 260
    },
    {
      "epoch": 1.1324786324786325,
      "grad_norm": 0.283203125,
      "learning_rate": 1.2323238973844796e-05,
      "logits/chosen": -1.5553245544433594,
      "logits/rejected": -0.9653908014297485,
      "logps/chosen": -59.334739685058594,
      "logps/rejected": -266.408935546875,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.896758794784546,
      "rewards/margins": 16.70488739013672,
      "rewards/rejected": -18.601646423339844,
      "step": 265
    },
    {
      "epoch": 1.1538461538461537,
      "grad_norm": 0.0145263671875,
      "learning_rate": 1.2202935967749212e-05,
      "logits/chosen": -1.5117428302764893,
      "logits/rejected": -0.9256662130355835,
      "logps/chosen": -62.41327667236328,
      "logps/rejected": -261.29559326171875,
      "loss": 0.0006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.1235432624816895,
      "rewards/margins": 16.034034729003906,
      "rewards/rejected": -18.15757942199707,
      "step": 270
    },
    {
      "epoch": 1.1752136752136753,
      "grad_norm": 0.16796875,
      "learning_rate": 1.2081189960141038e-05,
      "logits/chosen": -1.5283492803573608,
      "logits/rejected": -0.9296523332595825,
      "logps/chosen": -57.735595703125,
      "logps/rejected": -260.36627197265625,
      "loss": 0.0049,
      "rewards/accuracies": 0.9958332777023315,
      "rewards/chosen": -1.738776445388794,
      "rewards/margins": 16.458282470703125,
      "rewards/rejected": -18.19706153869629,
      "step": 275
    },
    {
      "epoch": 1.1965811965811965,
      "grad_norm": 0.51171875,
      "learning_rate": 1.1958068469513604e-05,
      "logits/chosen": -1.5641071796417236,
      "logits/rejected": -0.977310299873352,
      "logps/chosen": -59.65924072265625,
      "logps/rejected": -269.66253662109375,
      "loss": 0.0029,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.9156348705291748,
      "rewards/margins": 17.137256622314453,
      "rewards/rejected": -19.05289077758789,
      "step": 280
    },
    {
      "epoch": 1.217948717948718,
      "grad_norm": 0.09033203125,
      "learning_rate": 1.1833639777182316e-05,
      "logits/chosen": -1.5035396814346313,
      "logits/rejected": -0.8975561857223511,
      "logps/chosen": -54.24858856201172,
      "logps/rejected": -262.6181945800781,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5752283334732056,
      "rewards/margins": 16.97848129272461,
      "rewards/rejected": -18.5537109375,
      "step": 285
    },
    {
      "epoch": 1.2393162393162394,
      "grad_norm": 0.025390625,
      "learning_rate": 1.170797288941685e-05,
      "logits/chosen": -1.4596306085586548,
      "logits/rejected": -0.8935238718986511,
      "logps/chosen": -60.20256805419922,
      "logps/rejected": -270.41259765625,
      "loss": 0.001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.0154104232788086,
      "rewards/margins": 16.990856170654297,
      "rewards/rejected": -19.00626564025879,
      "step": 290
    },
    {
      "epoch": 1.2606837606837606,
      "grad_norm": 0.443359375,
      "learning_rate": 1.1581137499171342e-05,
      "logits/chosen": -1.478342890739441,
      "logits/rejected": -0.8745730519294739,
      "logps/chosen": -61.747718811035156,
      "logps/rejected": -260.48358154296875,
      "loss": 0.0006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.1923012733459473,
      "rewards/margins": 16.068017959594727,
      "rewards/rejected": -18.260318756103516,
      "step": 295
    },
    {
      "epoch": 1.282051282051282,
      "grad_norm": 0.0164794921875,
      "learning_rate": 1.145320394743371e-05,
      "logits/chosen": -1.439082384109497,
      "logits/rejected": -0.8525334596633911,
      "logps/chosen": -59.48680877685547,
      "logps/rejected": -256.1888122558594,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.9997012615203857,
      "rewards/margins": 16.03377914428711,
      "rewards/rejected": -18.033479690551758,
      "step": 300
    },
    {
      "epoch": 1.3034188034188035,
      "grad_norm": 0.004180908203125,
      "learning_rate": 1.1324243184215622e-05,
      "logits/chosen": -1.415179967880249,
      "logits/rejected": -0.811850368976593,
      "logps/chosen": -60.232139587402344,
      "logps/rejected": -271.9323425292969,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.1755707263946533,
      "rewards/margins": 17.213939666748047,
      "rewards/rejected": -19.389511108398438,
      "step": 305
    },
    {
      "epoch": 1.3247863247863247,
      "grad_norm": 0.0079345703125,
      "learning_rate": 1.1194326729204686e-05,
      "logits/chosen": -1.4362504482269287,
      "logits/rejected": -0.8141258358955383,
      "logps/chosen": -60.56780242919922,
      "logps/rejected": -269.37628173828125,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.1700046062469482,
      "rewards/margins": 17.103158950805664,
      "rewards/rejected": -19.273160934448242,
      "step": 310
    },
    {
      "epoch": 1.3461538461538463,
      "grad_norm": 0.0264892578125,
      "learning_rate": 1.1063526632100717e-05,
      "logits/chosen": -1.4126843214035034,
      "logits/rejected": -0.8487581014633179,
      "logps/chosen": -63.435699462890625,
      "logps/rejected": -263.54461669921875,
      "loss": 0.0035,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -2.3221802711486816,
      "rewards/margins": 16.341337203979492,
      "rewards/rejected": -18.66351890563965,
      "step": 315
    },
    {
      "epoch": 1.3675213675213675,
      "grad_norm": 0.01202392578125,
      "learning_rate": 1.0931915432658055e-05,
      "logits/chosen": -1.3911561965942383,
      "logits/rejected": -0.7923445701599121,
      "logps/chosen": -63.30759811401367,
      "logps/rejected": -268.752685546875,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.365898609161377,
      "rewards/margins": 16.713449478149414,
      "rewards/rejected": -19.079347610473633,
      "step": 320
    },
    {
      "epoch": 1.3888888888888888,
      "grad_norm": 0.01446533203125,
      "learning_rate": 1.0799566120456133e-05,
      "logits/chosen": -1.3831051588058472,
      "logits/rejected": -0.7522771954536438,
      "logps/chosen": -64.623046875,
      "logps/rejected": -268.64984130859375,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.4689812660217285,
      "rewards/margins": 16.626056671142578,
      "rewards/rejected": -19.09503746032715,
      "step": 325
    },
    {
      "epoch": 1.4102564102564101,
      "grad_norm": 0.01055908203125,
      "learning_rate": 1.066655209442054e-05,
      "logits/chosen": -1.4041249752044678,
      "logits/rejected": -0.7933310866355896,
      "logps/chosen": -65.8058853149414,
      "logps/rejected": -272.6845397949219,
      "loss": 0.0021,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.579711675643921,
      "rewards/margins": 16.771991729736328,
      "rewards/rejected": -19.351699829101562,
      "step": 330
    },
    {
      "epoch": 1.4316239316239316,
      "grad_norm": 0.0155029296875,
      "learning_rate": 1.0532947122117101e-05,
      "logits/chosen": -1.3893976211547852,
      "logits/rejected": -0.7593523263931274,
      "logps/chosen": -65.74365234375,
      "logps/rejected": -271.69537353515625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.5910518169403076,
      "rewards/margins": 16.82119369506836,
      "rewards/rejected": -19.41224479675293,
      "step": 335
    },
    {
      "epoch": 1.452991452991453,
      "grad_norm": 0.0167236328125,
      "learning_rate": 1.0398825298841499e-05,
      "logits/chosen": -1.3964606523513794,
      "logits/rejected": -0.8304880261421204,
      "logps/chosen": -69.43986511230469,
      "logps/rejected": -275.15960693359375,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.830099582672119,
      "rewards/margins": 16.80275535583496,
      "rewards/rejected": -19.632854461669922,
      "step": 340
    },
    {
      "epoch": 1.4743589743589745,
      "grad_norm": 0.0263671875,
      "learning_rate": 1.0264261006527144e-05,
      "logits/chosen": -1.4159036874771118,
      "logits/rejected": -0.8477792739868164,
      "logps/chosen": -66.49871063232422,
      "logps/rejected": -265.2044372558594,
      "loss": 0.004,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -2.641693592071533,
      "rewards/margins": 16.258649826049805,
      "rewards/rejected": -18.900344848632812,
      "step": 345
    },
    {
      "epoch": 1.4957264957264957,
      "grad_norm": 0.4140625,
      "learning_rate": 1.0129328872494075e-05,
      "logits/chosen": -1.4811489582061768,
      "logits/rejected": -0.9099611043930054,
      "logps/chosen": -66.62818908691406,
      "logps/rejected": -271.5672912597656,
      "loss": 0.003,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -2.69557523727417,
      "rewards/margins": 16.735515594482422,
      "rewards/rejected": -19.431089401245117,
      "step": 350
    },
    {
      "epoch": 1.517094017094017,
      "grad_norm": 0.134765625,
      "learning_rate": 9.994103728061786e-06,
      "logits/chosen": -1.434282660484314,
      "logits/rejected": -0.8590477705001831,
      "logps/chosen": -68.98523712158203,
      "logps/rejected": -271.1317138671875,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.879164934158325,
      "rewards/margins": 16.430065155029297,
      "rewards/rejected": -19.30923080444336,
      "step": 355
    },
    {
      "epoch": 1.5384615384615383,
      "grad_norm": 0.00104522705078125,
      "learning_rate": 9.858660567048902e-06,
      "logits/chosen": -1.4451745748519897,
      "logits/rejected": -0.8094388842582703,
      "logps/chosen": -71.38107299804688,
      "logps/rejected": -284.3163757324219,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.1282448768615723,
      "rewards/margins": 17.431547164916992,
      "rewards/rejected": -20.559795379638672,
      "step": 360
    },
    {
      "epoch": 1.5598290598290598,
      "grad_norm": 0.06591796875,
      "learning_rate": 9.72307450418274e-06,
      "logits/chosen": -1.4680107831954956,
      "logits/rejected": -0.8780597448348999,
      "logps/chosen": -66.4084701538086,
      "logps/rejected": -269.15948486328125,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.635270357131958,
      "rewards/margins": 16.57504653930664,
      "rewards/rejected": -19.210315704345703,
      "step": 365
    },
    {
      "epoch": 1.5811965811965814,
      "grad_norm": 1.921875,
      "learning_rate": 9.587420733441835e-06,
      "logits/chosen": -1.484803557395935,
      "logits/rejected": -0.9129883646965027,
      "logps/chosen": -63.17878341674805,
      "logps/rejected": -272.344482421875,
      "loss": 0.0006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.336841106414795,
      "rewards/margins": 17.08325958251953,
      "rewards/rejected": -19.420101165771484,
      "step": 370
    },
    {
      "epoch": 1.6025641025641026,
      "grad_norm": 0.000766754150390625,
      "learning_rate": 9.45177448635447e-06,
      "logits/chosen": -1.4960134029388428,
      "logits/rejected": -0.8223481178283691,
      "logps/chosen": -63.178985595703125,
      "logps/rejected": -274.9540100097656,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.362715482711792,
      "rewards/margins": 17.505586624145508,
      "rewards/rejected": -19.868301391601562,
      "step": 375
    },
    {
      "epoch": 1.623931623931624,
      "grad_norm": 0.046630859375,
      "learning_rate": 9.316210990276434e-06,
      "logits/chosen": -1.425362467765808,
      "logits/rejected": -0.851107120513916,
      "logps/chosen": -62.67730712890625,
      "logps/rejected": -263.357666015625,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.2725672721862793,
      "rewards/margins": 16.371471405029297,
      "rewards/rejected": -18.644039154052734,
      "step": 380
    },
    {
      "epoch": 1.6452991452991452,
      "grad_norm": 0.0045166015625,
      "learning_rate": 9.18080542667105e-06,
      "logits/chosen": -1.4633301496505737,
      "logits/rejected": -0.8226639628410339,
      "logps/chosen": -63.28871536254883,
      "logps/rejected": -280.1250915527344,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.2893106937408447,
      "rewards/margins": 17.715099334716797,
      "rewards/rejected": -20.004411697387695,
      "step": 385
    },
    {
      "epoch": 1.6666666666666665,
      "grad_norm": 0.004486083984375,
      "learning_rate": 9.045632889414686e-06,
      "logits/chosen": -1.451174020767212,
      "logits/rejected": -0.8335480690002441,
      "logps/chosen": -61.83076095581055,
      "logps/rejected": -272.6547546386719,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.2272868156433105,
      "rewards/margins": 17.232776641845703,
      "rewards/rejected": -19.460063934326172,
      "step": 390
    },
    {
      "epoch": 1.688034188034188,
      "grad_norm": 0.01153564453125,
      "learning_rate": 8.910768343150828e-06,
      "logits/chosen": -1.4321696758270264,
      "logits/rejected": -0.8404842615127563,
      "logps/chosen": -64.93434143066406,
      "logps/rejected": -273.5433044433594,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.5962464809417725,
      "rewards/margins": 17.047863006591797,
      "rewards/rejected": -19.64410972595215,
      "step": 395
    },
    {
      "epoch": 1.7094017094017095,
      "grad_norm": 0.006378173828125,
      "learning_rate": 8.77628658171581e-06,
      "logits/chosen": -1.4662706851959229,
      "logits/rejected": -0.8858024477958679,
      "logps/chosen": -63.3156852722168,
      "logps/rejected": -268.3216552734375,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.346360683441162,
      "rewards/margins": 16.638729095458984,
      "rewards/rejected": -18.98509407043457,
      "step": 400
    },
    {
      "epoch": 1.7307692307692308,
      "grad_norm": 0.033203125,
      "learning_rate": 8.642262186659298e-06,
      "logits/chosen": -1.4258873462677002,
      "logits/rejected": -0.8420296907424927,
      "logps/chosen": -64.47160339355469,
      "logps/rejected": -271.00213623046875,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.43169903755188,
      "rewards/margins": 16.784656524658203,
      "rewards/rejected": -19.21635627746582,
      "step": 405
    },
    {
      "epoch": 1.7521367521367521,
      "grad_norm": 0.023681640625,
      "learning_rate": 8.508769485882487e-06,
      "logits/chosen": -1.4450222253799438,
      "logits/rejected": -0.8718585968017578,
      "logps/chosen": -69.8866195678711,
      "logps/rejected": -272.61322021484375,
      "loss": 0.0022,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.834794759750366,
      "rewards/margins": 16.60897445678711,
      "rewards/rejected": -19.443769454956055,
      "step": 410
    },
    {
      "epoch": 1.7735042735042734,
      "grad_norm": 0.703125,
      "learning_rate": 8.375882512416969e-06,
      "logits/chosen": -1.4202231168746948,
      "logits/rejected": -0.7998637557029724,
      "logps/chosen": -67.1268310546875,
      "logps/rejected": -275.6379089355469,
      "loss": 0.0006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.6624515056610107,
      "rewards/margins": 17.090303421020508,
      "rewards/rejected": -19.752756118774414,
      "step": 415
    },
    {
      "epoch": 1.7948717948717947,
      "grad_norm": 0.0458984375,
      "learning_rate": 8.243674963367137e-06,
      "logits/chosen": -1.420807957649231,
      "logits/rejected": -0.8041313886642456,
      "logps/chosen": -68.99574279785156,
      "logps/rejected": -272.53936767578125,
      "loss": 0.0033,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -2.8813891410827637,
      "rewards/margins": 16.42134666442871,
      "rewards/rejected": -19.302734375,
      "step": 420
    },
    {
      "epoch": 1.8162393162393162,
      "grad_norm": 0.000850677490234375,
      "learning_rate": 8.11222015903888e-06,
      "logits/chosen": -1.4625945091247559,
      "logits/rejected": -0.8097749948501587,
      "logps/chosen": -67.76467895507812,
      "logps/rejected": -278.23931884765625,
      "loss": 0.0032,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -2.830801010131836,
      "rewards/margins": 17.239028930664062,
      "rewards/rejected": -20.0698299407959,
      "step": 425
    },
    {
      "epoch": 1.8376068376068377,
      "grad_norm": 0.03515625,
      "learning_rate": 7.981591002277265e-06,
      "logits/chosen": -1.4442012310028076,
      "logits/rejected": -0.8179518580436707,
      "logps/chosen": -69.31373596191406,
      "logps/rejected": -273.0030822753906,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.9278433322906494,
      "rewards/margins": 16.736169815063477,
      "rewards/rejected": -19.66401481628418,
      "step": 430
    },
    {
      "epoch": 1.858974358974359,
      "grad_norm": 0.01116943359375,
      "learning_rate": 7.851859938035712e-06,
      "logits/chosen": -1.4216701984405518,
      "logits/rejected": -0.7942286729812622,
      "logps/chosen": -71.70731353759766,
      "logps/rejected": -282.1054382324219,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.118105888366699,
      "rewards/margins": 17.2371768951416,
      "rewards/rejected": -20.355281829833984,
      "step": 435
    },
    {
      "epoch": 1.8803418803418803,
      "grad_norm": 0.0279541015625,
      "learning_rate": 7.723098913199118e-06,
      "logits/chosen": -1.4586832523345947,
      "logits/rejected": -0.8305650949478149,
      "logps/chosen": -67.39842224121094,
      "logps/rejected": -269.7090148925781,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.8383772373199463,
      "rewards/margins": 16.61200714111328,
      "rewards/rejected": -19.450382232666016,
      "step": 440
    },
    {
      "epoch": 1.9017094017094016,
      "grad_norm": 0.0419921875,
      "learning_rate": 7.595379336683204e-06,
      "logits/chosen": -1.433342456817627,
      "logits/rejected": -0.7530536651611328,
      "logps/chosen": -65.9086685180664,
      "logps/rejected": -276.2018127441406,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.7526674270629883,
      "rewards/margins": 17.266189575195312,
      "rewards/rejected": -20.018856048583984,
      "step": 445
    },
    {
      "epoch": 1.9230769230769231,
      "grad_norm": 0.00433349609375,
      "learning_rate": 7.468772039832218e-06,
      "logits/chosen": -1.4057344198226929,
      "logits/rejected": -0.7420554757118225,
      "logps/chosen": -61.406890869140625,
      "logps/rejected": -270.76568603515625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.308377742767334,
      "rewards/margins": 17.16427230834961,
      "rewards/rejected": -19.472646713256836,
      "step": 450
    },
    {
      "epoch": 1.9444444444444444,
      "grad_norm": 0.00634765625,
      "learning_rate": 7.3433472371369404e-06,
      "logits/chosen": -1.4746873378753662,
      "logits/rejected": -0.8472849726676941,
      "logps/chosen": -66.808837890625,
      "logps/rejected": -274.1222839355469,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.7300994396209717,
      "rewards/margins": 16.893524169921875,
      "rewards/rejected": -19.623624801635742,
      "step": 455
    },
    {
      "epoch": 1.965811965811966,
      "grad_norm": 0.00092315673828125,
      "learning_rate": 7.219174487294784e-06,
      "logits/chosen": -1.4693877696990967,
      "logits/rejected": -0.856170654296875,
      "logps/chosen": -64.80795288085938,
      "logps/rejected": -276.7772216796875,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.508439779281616,
      "rewards/margins": 17.357221603393555,
      "rewards/rejected": -19.865665435791016,
      "step": 460
    },
    {
      "epoch": 1.9871794871794872,
      "grad_norm": 0.0234375,
      "learning_rate": 7.0963226546336e-06,
      "logits/chosen": -1.475427269935608,
      "logits/rejected": -0.8358310461044312,
      "logps/chosen": -66.6759262084961,
      "logps/rejected": -272.0386962890625,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.7268948554992676,
      "rewards/margins": 16.760303497314453,
      "rewards/rejected": -19.487197875976562,
      "step": 465
    },
    {
      "epoch": 1.9914529914529915,
      "eval_logits/chosen": -1.4994289875030518,
      "eval_logits/rejected": -0.8360934257507324,
      "eval_logps/chosen": -67.28651428222656,
      "eval_logps/rejected": -275.5657043457031,
      "eval_loss": 0.00012571073602885008,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -2.7842776775360107,
      "eval_rewards/margins": 16.970603942871094,
      "eval_rewards/rejected": -19.754880905151367,
      "eval_runtime": 9.583,
      "eval_samples_per_second": 20.87,
      "eval_steps_per_second": 20.87,
      "step": 466
    },
    {
      "epoch": 2.0085470085470085,
      "grad_norm": 0.080078125,
      "learning_rate": 6.974859870920561e-06,
      "logits/chosen": -1.3924753665924072,
      "logits/rejected": -0.8071160316467285,
      "logps/chosen": -68.68719482421875,
      "logps/rejected": -270.14801025390625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.911695957183838,
      "rewards/margins": 16.461864471435547,
      "rewards/rejected": -19.37356185913086,
      "step": 470
    },
    {
      "epoch": 2.02991452991453,
      "grad_norm": 0.0012359619140625,
      "learning_rate": 6.8548534975773135e-06,
      "logits/chosen": -1.4488104581832886,
      "logits/rejected": -0.82325679063797,
      "logps/chosen": -69.13402557373047,
      "logps/rejected": -275.86407470703125,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.8428947925567627,
      "rewards/margins": 16.885896682739258,
      "rewards/rejected": -19.728792190551758,
      "step": 475
    },
    {
      "epoch": 2.051282051282051,
      "grad_norm": 0.279296875,
      "learning_rate": 6.736370088322359e-06,
      "logits/chosen": -1.4575817584991455,
      "logits/rejected": -0.8154885172843933,
      "logps/chosen": -66.7756576538086,
      "logps/rejected": -272.9770812988281,
      "loss": 0.0006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.7416322231292725,
      "rewards/margins": 16.960206985473633,
      "rewards/rejected": -19.701839447021484,
      "step": 480
    },
    {
      "epoch": 2.072649572649573,
      "grad_norm": 0.00994873046875,
      "learning_rate": 6.619475352261356e-06,
      "logits/chosen": -1.4201619625091553,
      "logits/rejected": -0.8027874827384949,
      "logps/chosen": -72.34085845947266,
      "logps/rejected": -276.4122009277344,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.0328619480133057,
      "rewards/margins": 16.768281936645508,
      "rewards/rejected": -19.801143646240234,
      "step": 485
    },
    {
      "epoch": 2.094017094017094,
      "grad_norm": 0.0030975341796875,
      "learning_rate": 6.504234117445857e-06,
      "logits/chosen": -1.4316365718841553,
      "logits/rejected": -0.8082255125045776,
      "logps/chosen": -67.41413879394531,
      "logps/rejected": -272.8454284667969,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.8370206356048584,
      "rewards/margins": 16.798355102539062,
      "rewards/rejected": -19.635372161865234,
      "step": 490
    },
    {
      "epoch": 2.1153846153846154,
      "grad_norm": 0.012939453125,
      "learning_rate": 6.39071029492065e-06,
      "logits/chosen": -1.4187885522842407,
      "logits/rejected": -0.7696577906608582,
      "logps/chosen": -66.7591781616211,
      "logps/rejected": -271.50927734375,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.810885190963745,
      "rewards/margins": 16.76914405822754,
      "rewards/rejected": -19.580028533935547,
      "step": 495
    },
    {
      "epoch": 2.1367521367521367,
      "grad_norm": 0.125,
      "learning_rate": 6.2789668432796535e-06,
      "logits/chosen": -1.4300968647003174,
      "logits/rejected": -0.8352675437927246,
      "logps/chosen": -67.97314453125,
      "logps/rejected": -273.3833312988281,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.8787360191345215,
      "rewards/margins": 16.89468002319336,
      "rewards/rejected": -19.773418426513672,
      "step": 500
    },
    {
      "epoch": 2.1367521367521367,
      "eval_logits/chosen": -1.4944664239883423,
      "eval_logits/rejected": -0.8317099213600159,
      "eval_logps/chosen": -67.55956268310547,
      "eval_logps/rejected": -276.07452392578125,
      "eval_loss": 0.0001331747480435297,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -2.811582565307617,
      "eval_rewards/margins": 16.994182586669922,
      "eval_rewards/rejected": -19.805763244628906,
      "eval_runtime": 9.5603,
      "eval_samples_per_second": 20.92,
      "eval_steps_per_second": 20.92,
      "step": 500
    },
    {
      "epoch": 2.158119658119658,
      "grad_norm": 0.016845703125,
      "learning_rate": 6.16906573375004e-06,
      "logits/chosen": -1.4502363204956055,
      "logits/rejected": -0.8533867001533508,
      "logps/chosen": -67.79924774169922,
      "logps/rejected": -271.75946044921875,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.8223788738250732,
      "rewards/margins": 16.64777374267578,
      "rewards/rejected": -19.47015380859375,
      "step": 505
    },
    {
      "epoch": 2.1794871794871793,
      "grad_norm": 0.05908203125,
      "learning_rate": 6.061067915823923e-06,
      "logits/chosen": -1.3864041566848755,
      "logits/rejected": -0.7836092114448547,
      "logps/chosen": -65.47071838378906,
      "logps/rejected": -273.322265625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.629401683807373,
      "rewards/margins": 16.939075469970703,
      "rewards/rejected": -19.568477630615234,
      "step": 510
    },
    {
      "epoch": 2.200854700854701,
      "grad_norm": 0.006103515625,
      "learning_rate": 5.955033283456711e-06,
      "logits/chosen": -1.4336755275726318,
      "logits/rejected": -0.778075098991394,
      "logps/chosen": -70.93939971923828,
      "logps/rejected": -283.32598876953125,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.026041030883789,
      "rewards/margins": 17.39080238342285,
      "rewards/rejected": -20.41684341430664,
      "step": 515
    },
    {
      "epoch": 2.2222222222222223,
      "grad_norm": 0.00128936767578125,
      "learning_rate": 5.8510206418507914e-06,
      "logits/chosen": -1.4767441749572754,
      "logits/rejected": -0.8630167245864868,
      "logps/chosen": -68.52049255371094,
      "logps/rejected": -288.3775939941406,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.804711103439331,
      "rewards/margins": 18.02071762084961,
      "rewards/rejected": -20.825428009033203,
      "step": 520
    },
    {
      "epoch": 2.2435897435897436,
      "grad_norm": 0.0078125,
      "learning_rate": 5.749087674843095e-06,
      "logits/chosen": -1.416854977607727,
      "logits/rejected": -0.8053888082504272,
      "logps/chosen": -63.85468673706055,
      "logps/rejected": -274.10955810546875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.541356086730957,
      "rewards/margins": 17.136423110961914,
      "rewards/rejected": -19.67778205871582,
      "step": 525
    },
    {
      "epoch": 2.264957264957265,
      "grad_norm": 0.024658203125,
      "learning_rate": 5.649290912914482e-06,
      "logits/chosen": -1.4132906198501587,
      "logits/rejected": -0.8062090873718262,
      "logps/chosen": -71.18649291992188,
      "logps/rejected": -287.32562255859375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.9816391468048096,
      "rewards/margins": 17.66859245300293,
      "rewards/rejected": -20.650232315063477,
      "step": 530
    },
    {
      "epoch": 2.286324786324786,
      "grad_norm": 0.031005859375,
      "learning_rate": 5.5516857018388144e-06,
      "logits/chosen": -1.4750016927719116,
      "logits/rejected": -0.8834052085876465,
      "logps/chosen": -66.85182189941406,
      "logps/rejected": -271.5949401855469,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.7011666297912598,
      "rewards/margins": 16.737390518188477,
      "rewards/rejected": -19.438556671142578,
      "step": 535
    },
    {
      "epoch": 2.3076923076923075,
      "grad_norm": 0.0098876953125,
      "learning_rate": 5.456326171989005e-06,
      "logits/chosen": -1.4350497722625732,
      "logits/rejected": -0.854056715965271,
      "logps/chosen": -64.83439636230469,
      "logps/rejected": -287.66961669921875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.5173258781433105,
      "rewards/margins": 18.059844970703125,
      "rewards/rejected": -20.57716941833496,
      "step": 540
    },
    {
      "epoch": 2.3290598290598292,
      "grad_norm": 0.0036163330078125,
      "learning_rate": 5.363265208317156e-06,
      "logits/chosen": -1.4079747200012207,
      "logits/rejected": -0.8292921781539917,
      "logps/chosen": -66.03783416748047,
      "logps/rejected": -271.29266357421875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.6700127124786377,
      "rewards/margins": 16.861469268798828,
      "rewards/rejected": -19.531482696533203,
      "step": 545
    },
    {
      "epoch": 2.3504273504273505,
      "grad_norm": 0.0035247802734375,
      "learning_rate": 5.272554421025347e-06,
      "logits/chosen": -1.4516395330429077,
      "logits/rejected": -0.8227602243423462,
      "logps/chosen": -68.09796142578125,
      "logps/rejected": -281.10687255859375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.847126007080078,
      "rewards/margins": 17.43533706665039,
      "rewards/rejected": -20.2824649810791,
      "step": 550
    },
    {
      "epoch": 2.371794871794872,
      "grad_norm": 0.04638671875,
      "learning_rate": 5.184244116943411e-06,
      "logits/chosen": -1.4369877576828003,
      "logits/rejected": -0.8126457333564758,
      "logps/chosen": -67.11319732666016,
      "logps/rejected": -276.0461730957031,
      "loss": 0.003,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -2.69486665725708,
      "rewards/margins": 17.134822845458984,
      "rewards/rejected": -19.829692840576172,
      "step": 555
    },
    {
      "epoch": 2.393162393162393,
      "grad_norm": 0.013427734375,
      "learning_rate": 5.098383271629512e-06,
      "logits/chosen": -1.4552416801452637,
      "logits/rejected": -0.8165750503540039,
      "logps/chosen": -67.05858612060547,
      "logps/rejected": -270.05596923828125,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.7700178623199463,
      "rewards/margins": 16.640972137451172,
      "rewards/rejected": -19.41098976135254,
      "step": 560
    },
    {
      "epoch": 2.4145299145299144,
      "grad_norm": 0.024658203125,
      "learning_rate": 5.015019502209056e-06,
      "logits/chosen": -1.4252703189849854,
      "logits/rejected": -0.7998120188713074,
      "logps/chosen": -65.51549530029297,
      "logps/rejected": -266.0838623046875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.6941094398498535,
      "rewards/margins": 16.32393455505371,
      "rewards/rejected": -19.018043518066406,
      "step": 565
    },
    {
      "epoch": 2.435897435897436,
      "grad_norm": 0.00130462646484375,
      "learning_rate": 4.934199040966955e-06,
      "logits/chosen": -1.473314642906189,
      "logits/rejected": -0.847906231880188,
      "logps/chosen": -66.51988220214844,
      "logps/rejected": -269.98150634765625,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.734851360321045,
      "rewards/margins": 16.677799224853516,
      "rewards/rejected": -19.412649154663086,
      "step": 570
    },
    {
      "epoch": 2.4572649572649574,
      "grad_norm": 0.0120849609375,
      "learning_rate": 4.855966709707881e-06,
      "logits/chosen": -1.4144232273101807,
      "logits/rejected": -0.8331915736198425,
      "logps/chosen": -70.74285888671875,
      "logps/rejected": -275.6701354980469,
      "loss": 0.003,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -3.0315637588500977,
      "rewards/margins": 16.809673309326172,
      "rewards/rejected": -19.841238021850586,
      "step": 575
    },
    {
      "epoch": 2.4786324786324787,
      "grad_norm": 0.035888671875,
      "learning_rate": 4.780365894898799e-06,
      "logits/chosen": -1.446194052696228,
      "logits/rejected": -0.8259027600288391,
      "logps/chosen": -67.32369995117188,
      "logps/rejected": -276.65728759765625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.730370044708252,
      "rewards/margins": 17.11151123046875,
      "rewards/rejected": -19.841882705688477,
      "step": 580
    },
    {
      "epoch": 2.5,
      "grad_norm": 0.00885009765625,
      "learning_rate": 4.7074385236074684e-06,
      "logits/chosen": -1.4675248861312866,
      "logits/rejected": -0.8278427124023438,
      "logps/chosen": -71.28508758544922,
      "logps/rejected": -284.6138000488281,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.97928786277771,
      "rewards/margins": 17.529958724975586,
      "rewards/rejected": -20.509244918823242,
      "step": 585
    },
    {
      "epoch": 2.5213675213675213,
      "grad_norm": 0.054931640625,
      "learning_rate": 4.63722504025034e-06,
      "logits/chosen": -1.4478528499603271,
      "logits/rejected": -0.8462594747543335,
      "logps/chosen": -64.45930480957031,
      "logps/rejected": -275.44036865234375,
      "loss": 0.0008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.4965498447418213,
      "rewards/margins": 17.28372573852539,
      "rewards/rejected": -19.780277252197266,
      "step": 590
    },
    {
      "epoch": 2.5427350427350426,
      "grad_norm": 0.021240234375,
      "learning_rate": 4.569764384162676e-06,
      "logits/chosen": -1.4624818563461304,
      "logits/rejected": -0.8267906308174133,
      "logps/chosen": -61.47455596923828,
      "logps/rejected": -273.9830017089844,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.310239791870117,
      "rewards/margins": 17.343975067138672,
      "rewards/rejected": -19.65421485900879,
      "step": 595
    },
    {
      "epoch": 2.564102564102564,
      "grad_norm": 0.031494140625,
      "learning_rate": 4.50509396800341e-06,
      "logits/chosen": -1.4217302799224854,
      "logits/rejected": -0.7868741750717163,
      "logps/chosen": -65.87348937988281,
      "logps/rejected": -277.1817321777344,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.619393825531006,
      "rewards/margins": 17.432384490966797,
      "rewards/rejected": -20.051776885986328,
      "step": 600
    },
    {
      "epoch": 2.5854700854700856,
      "grad_norm": 0.0732421875,
      "learning_rate": 4.443249657006627e-06,
      "logits/chosen": -1.4423940181732178,
      "logits/rejected": -0.7987694144248962,
      "logps/chosen": -63.83234786987305,
      "logps/rejected": -280.8373718261719,
      "loss": 0.0029,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -2.4912962913513184,
      "rewards/margins": 17.850605010986328,
      "rewards/rejected": -20.341899871826172,
      "step": 605
    },
    {
      "epoch": 2.606837606837607,
      "grad_norm": 0.01336669921875,
      "learning_rate": 4.384265749091266e-06,
      "logits/chosen": -1.391326665878296,
      "logits/rejected": -0.7768113613128662,
      "logps/chosen": -71.64138793945312,
      "logps/rejected": -278.91400146484375,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.0774004459381104,
      "rewards/margins": 17.042530059814453,
      "rewards/rejected": -20.119930267333984,
      "step": 610
    },
    {
      "epoch": 2.628205128205128,
      "grad_norm": 0.01239013671875,
      "learning_rate": 4.328174955840002e-06,
      "logits/chosen": -1.4435267448425293,
      "logits/rejected": -0.8103730082511902,
      "logps/chosen": -62.11974334716797,
      "logps/rejected": -273.298828125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.468442440032959,
      "rewards/margins": 17.258960723876953,
      "rewards/rejected": -19.727405548095703,
      "step": 615
    },
    {
      "epoch": 2.6495726495726495,
      "grad_norm": 0.004791259765625,
      "learning_rate": 4.275008384357902e-06,
      "logits/chosen": -1.4775636196136475,
      "logits/rejected": -0.8828374743461609,
      "logps/chosen": -65.2374267578125,
      "logps/rejected": -272.7725524902344,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.561352491378784,
      "rewards/margins": 17.067296981811523,
      "rewards/rejected": -19.62864875793457,
      "step": 620
    },
    {
      "epoch": 2.6709401709401708,
      "grad_norm": 0.005157470703125,
      "learning_rate": 4.224795520020898e-06,
      "logits/chosen": -1.418306827545166,
      "logits/rejected": -0.7850183248519897,
      "logps/chosen": -68.51617431640625,
      "logps/rejected": -272.66455078125,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.727930784225464,
      "rewards/margins": 16.756616592407227,
      "rewards/rejected": -19.484546661376953,
      "step": 625
    },
    {
      "epoch": 2.6923076923076925,
      "grad_norm": 0.042724609375,
      "learning_rate": 4.177564210123634e-06,
      "logits/chosen": -1.4454832077026367,
      "logits/rejected": -0.8340616226196289,
      "logps/chosen": -65.76726531982422,
      "logps/rejected": -282.2267150878906,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.610382556915283,
      "rewards/margins": 17.712736129760742,
      "rewards/rejected": -20.323118209838867,
      "step": 630
    },
    {
      "epoch": 2.713675213675214,
      "grad_norm": 0.039306640625,
      "learning_rate": 4.133340648435789e-06,
      "logits/chosen": -1.4548784494400024,
      "logits/rejected": -0.8171395063400269,
      "logps/chosen": -66.9704360961914,
      "logps/rejected": -277.67156982421875,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.7575552463531494,
      "rewards/margins": 17.253223419189453,
      "rewards/rejected": -20.01078224182129,
      "step": 635
    },
    {
      "epoch": 2.735042735042735,
      "grad_norm": 0.003448486328125,
      "learning_rate": 4.092149360675402e-06,
      "logits/chosen": -1.429237961769104,
      "logits/rejected": -0.825334906578064,
      "logps/chosen": -71.72950744628906,
      "logps/rejected": -282.79864501953125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.9618561267852783,
      "rewards/margins": 17.245059967041016,
      "rewards/rejected": -20.206918716430664,
      "step": 640
    },
    {
      "epoch": 2.7564102564102564,
      "grad_norm": 0.04833984375,
      "learning_rate": 4.054013190907282e-06,
      "logits/chosen": -1.3933217525482178,
      "logits/rejected": -0.7962290048599243,
      "logps/chosen": -62.87525177001953,
      "logps/rejected": -274.01885986328125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.444106101989746,
      "rewards/margins": 17.38211441040039,
      "rewards/rejected": -19.826221466064453,
      "step": 645
    },
    {
      "epoch": 2.7777777777777777,
      "grad_norm": 0.0032196044921875,
      "learning_rate": 4.018953288874035e-06,
      "logits/chosen": -1.4579218626022339,
      "logits/rejected": -0.8607872724533081,
      "logps/chosen": -67.1677474975586,
      "logps/rejected": -278.09576416015625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.7266063690185547,
      "rewards/margins": 17.331296920776367,
      "rewards/rejected": -20.057903289794922,
      "step": 650
    },
    {
      "epoch": 2.799145299145299,
      "grad_norm": 0.0028533935546875,
      "learning_rate": 3.9869890982667385e-06,
      "logits/chosen": -1.4375386238098145,
      "logits/rejected": -0.7617470026016235,
      "logps/chosen": -66.19978332519531,
      "logps/rejected": -279.52581787109375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.685457706451416,
      "rewards/margins": 17.297794342041016,
      "rewards/rejected": -19.983253479003906,
      "step": 655
    },
    {
      "epoch": 2.8205128205128203,
      "grad_norm": 0.01300048828125,
      "learning_rate": 3.9581383459417625e-06,
      "logits/chosen": -1.409898042678833,
      "logits/rejected": -0.7993089556694031,
      "logps/chosen": -71.67328643798828,
      "logps/rejected": -286.96978759765625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.136204242706299,
      "rewards/margins": 17.608535766601562,
      "rewards/rejected": -20.744739532470703,
      "step": 660
    },
    {
      "epoch": 2.841880341880342,
      "grad_norm": 0.029052734375,
      "learning_rate": 3.932417032089722e-06,
      "logits/chosen": -1.4576003551483154,
      "logits/rejected": -0.821110725402832,
      "logps/chosen": -68.1684799194336,
      "logps/rejected": -282.56744384765625,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.7884955406188965,
      "rewards/margins": 17.563434600830078,
      "rewards/rejected": -20.351930618286133,
      "step": 665
    },
    {
      "epoch": 2.8632478632478633,
      "grad_norm": 0.000885009765625,
      "learning_rate": 3.909839421362017e-06,
      "logits/chosen": -1.4005993604660034,
      "logits/rejected": -0.7887626886367798,
      "logps/chosen": -67.8237075805664,
      "logps/rejected": -278.14056396484375,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.829472064971924,
      "rewards/margins": 17.069793701171875,
      "rewards/rejected": -19.899267196655273,
      "step": 670
    },
    {
      "epoch": 2.8846153846153846,
      "grad_norm": 0.048828125,
      "learning_rate": 3.890418034959871e-06,
      "logits/chosen": -1.3995271921157837,
      "logits/rejected": -0.7455986738204956,
      "logps/chosen": -66.70311737060547,
      "logps/rejected": -276.72540283203125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.7374672889709473,
      "rewards/margins": 17.271923065185547,
      "rewards/rejected": -20.00939178466797,
      "step": 675
    },
    {
      "epoch": 2.905982905982906,
      "grad_norm": 0.045166015625,
      "learning_rate": 3.874163643690263e-06,
      "logits/chosen": -1.3939011096954346,
      "logits/rejected": -0.7581735849380493,
      "logps/chosen": -74.34012603759766,
      "logps/rejected": -280.92291259765625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.3185927867889404,
      "rewards/margins": 16.952451705932617,
      "rewards/rejected": -20.27104377746582,
      "step": 680
    },
    {
      "epoch": 2.9273504273504276,
      "grad_norm": 0.01312255859375,
      "learning_rate": 3.861085261992599e-06,
      "logits/chosen": -1.4342122077941895,
      "logits/rejected": -0.8670775294303894,
      "logps/chosen": -70.84014129638672,
      "logps/rejected": -280.0901794433594,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.0605998039245605,
      "rewards/margins": 17.057954788208008,
      "rewards/rejected": -20.118555068969727,
      "step": 685
    },
    {
      "epoch": 2.948717948717949,
      "grad_norm": 0.1376953125,
      "learning_rate": 3.851190142939442e-06,
      "logits/chosen": -1.435960054397583,
      "logits/rejected": -0.8204304575920105,
      "logps/chosen": -65.02848052978516,
      "logps/rejected": -277.8736877441406,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.4977402687072754,
      "rewards/margins": 17.434282302856445,
      "rewards/rejected": -19.932022094726562,
      "step": 690
    },
    {
      "epoch": 2.97008547008547,
      "grad_norm": 0.003662109375,
      "learning_rate": 3.844483774214069e-06,
      "logits/chosen": -1.4186488389968872,
      "logits/rejected": -0.7549436688423157,
      "logps/chosen": -64.54364776611328,
      "logps/rejected": -277.66680908203125,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.5480058193206787,
      "rewards/margins": 17.503028869628906,
      "rewards/rejected": -20.0510311126709,
      "step": 695
    },
    {
      "epoch": 2.9871794871794872,
      "eval_logits/chosen": -1.4855865240097046,
      "eval_logits/rejected": -0.8208981156349182,
      "eval_logps/chosen": -67.3394775390625,
      "eval_logps/rejected": -276.6026306152344,
      "eval_loss": 0.00011922841804334894,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -2.789573907852173,
      "eval_rewards/margins": 17.068998336791992,
      "eval_rewards/rejected": -19.858572006225586,
      "eval_runtime": 9.6169,
      "eval_samples_per_second": 20.797,
      "eval_steps_per_second": 20.797,
      "step": 699
    }
  ],
  "logging_steps": 5,
  "max_steps": 702,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 12,
  "trial_name": null,
  "trial_params": null
}