| { |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 10.294117647058824, |
| "eval_steps": 500, |
| "global_step": 700, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.74038696289062, |
| "epoch": 0.014705882352941176, |
| "grad_norm": 4.535776925340435, |
| "kl": 0.0, |
| "learning_rate": 9.992647058823528e-07, |
| "loss": -0.0, |
| "reward": 1.7768200635910034, |
| "reward_std": 0.06235586851835251, |
| "rewards/accuracy_reward": 0.7828295826911926, |
| "rewards/format_reward": 0.993990421295166, |
| "step": 1 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.58173370361328, |
| "epoch": 0.029411764705882353, |
| "grad_norm": 4.950891299350819, |
| "kl": 0.000789642333984375, |
| "learning_rate": 9.98529411764706e-07, |
| "loss": 0.0, |
| "reward": 1.797240972518921, |
| "reward_std": 0.06516173481941223, |
| "rewards/accuracy_reward": 0.8056544661521912, |
| "rewards/format_reward": 0.9915865659713745, |
| "step": 2 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.5252456665039, |
| "epoch": 0.04411764705882353, |
| "grad_norm": 1.8505526818416387, |
| "kl": 0.00093841552734375, |
| "learning_rate": 9.977941176470588e-07, |
| "loss": 0.0, |
| "reward": 1.7827500104904175, |
| "reward_std": 0.06552266329526901, |
| "rewards/accuracy_reward": 0.7887594699859619, |
| "rewards/format_reward": 0.993990421295166, |
| "step": 3 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.68870544433594, |
| "epoch": 0.058823529411764705, |
| "grad_norm": 1.3899081066234504, |
| "kl": 0.00102996826171875, |
| "learning_rate": 9.970588235294117e-07, |
| "loss": 0.0, |
| "reward": 1.7779340744018555, |
| "reward_std": 0.06615827977657318, |
| "rewards/accuracy_reward": 0.7851455807685852, |
| "rewards/format_reward": 0.9927884936332703, |
| "step": 4 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.76563262939453, |
| "epoch": 0.07352941176470588, |
| "grad_norm": 2.31071643885251, |
| "kl": 0.00112152099609375, |
| "learning_rate": 9.963235294117647e-07, |
| "loss": 0.0, |
| "reward": 1.790160894393921, |
| "reward_std": 0.06119891628623009, |
| "rewards/accuracy_reward": 0.7949684262275696, |
| "rewards/format_reward": 0.9951923489570618, |
| "step": 5 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.93389892578125, |
| "epoch": 0.08823529411764706, |
| "grad_norm": 5.892350008855451, |
| "kl": 0.001983642578125, |
| "learning_rate": 9.955882352941176e-07, |
| "loss": 0.0001, |
| "reward": 1.8021262884140015, |
| "reward_std": 0.06035131961107254, |
| "rewards/accuracy_reward": 0.8033280372619629, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 6 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.10938262939453, |
| "epoch": 0.10294117647058823, |
| "grad_norm": 1.4454822561506098, |
| "kl": 0.002685546875, |
| "learning_rate": 9.948529411764705e-07, |
| "loss": 0.0001, |
| "reward": 1.7747721672058105, |
| "reward_std": 0.06199384108185768, |
| "rewards/accuracy_reward": 0.7795796990394592, |
| "rewards/format_reward": 0.9951923489570618, |
| "step": 7 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.11418914794922, |
| "epoch": 0.11764705882352941, |
| "grad_norm": 2.5592127837818732, |
| "kl": 0.0029449462890625, |
| "learning_rate": 9.941176470588236e-07, |
| "loss": 0.0001, |
| "reward": 1.7927086353302002, |
| "reward_std": 0.05324796214699745, |
| "rewards/accuracy_reward": 0.793910562992096, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 8 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.32933044433594, |
| "epoch": 0.1323529411764706, |
| "grad_norm": 1.2070688702059713, |
| "kl": 0.003509521484375, |
| "learning_rate": 9.933823529411765e-07, |
| "loss": 0.0001, |
| "reward": 1.807510495185852, |
| "reward_std": 0.05738683044910431, |
| "rewards/accuracy_reward": 0.8099143505096436, |
| "rewards/format_reward": 0.9975962042808533, |
| "step": 9 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.88702392578125, |
| "epoch": 0.14705882352941177, |
| "grad_norm": 1.1236907795600153, |
| "kl": 0.004150390625, |
| "learning_rate": 9.926470588235293e-07, |
| "loss": 0.0002, |
| "reward": 1.7909963130950928, |
| "reward_std": 0.05266621708869934, |
| "rewards/accuracy_reward": 0.7921981811523438, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 10 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.37981414794922, |
| "epoch": 0.16176470588235295, |
| "grad_norm": 1.6831920052248819, |
| "kl": 0.005645751953125, |
| "learning_rate": 9.919117647058824e-07, |
| "loss": 0.0002, |
| "reward": 1.805780053138733, |
| "reward_std": 0.05035817250609398, |
| "rewards/accuracy_reward": 0.8069819808006287, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 11 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 100.015625, |
| "epoch": 0.17647058823529413, |
| "grad_norm": 1.7093381711239366, |
| "kl": 0.0076904296875, |
| "learning_rate": 9.911764705882353e-07, |
| "loss": 0.0003, |
| "reward": 1.829386830329895, |
| "reward_std": 0.04737670719623566, |
| "rewards/accuracy_reward": 0.8305887579917908, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 12 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 99.73798370361328, |
| "epoch": 0.19117647058823528, |
| "grad_norm": 1.5253862652760264, |
| "kl": 0.00909423828125, |
| "learning_rate": 9.904411764705882e-07, |
| "loss": 0.0004, |
| "reward": 1.7849040031433105, |
| "reward_std": 0.05029381439089775, |
| "rewards/accuracy_reward": 0.7849039435386658, |
| "rewards/format_reward": 1.0, |
| "step": 13 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 98.33534240722656, |
| "epoch": 0.20588235294117646, |
| "grad_norm": 1.5658868221984554, |
| "kl": 0.01055908203125, |
| "learning_rate": 9.897058823529412e-07, |
| "loss": 0.0004, |
| "reward": 1.803476095199585, |
| "reward_std": 0.04388910159468651, |
| "rewards/accuracy_reward": 0.803476095199585, |
| "rewards/format_reward": 1.0, |
| "step": 14 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 99.25601196289062, |
| "epoch": 0.22058823529411764, |
| "grad_norm": 9.7316953003379, |
| "kl": 0.01141357421875, |
| "learning_rate": 9.889705882352941e-07, |
| "loss": 0.0005, |
| "reward": 1.791574239730835, |
| "reward_std": 0.05360911041498184, |
| "rewards/accuracy_reward": 0.7927761077880859, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 15 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 98.83293914794922, |
| "epoch": 0.23529411764705882, |
| "grad_norm": 2.8478472800650585, |
| "kl": 0.01220703125, |
| "learning_rate": 9.88235294117647e-07, |
| "loss": 0.0005, |
| "reward": 1.7902849912643433, |
| "reward_std": 0.051616914570331573, |
| "rewards/accuracy_reward": 0.7914868593215942, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 16 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 98.93509674072266, |
| "epoch": 0.25, |
| "grad_norm": 2.544057708424019, |
| "kl": 0.013427734375, |
| "learning_rate": 9.875e-07, |
| "loss": 0.0005, |
| "reward": 1.8025610446929932, |
| "reward_std": 0.04462622106075287, |
| "rewards/accuracy_reward": 0.8025608658790588, |
| "rewards/format_reward": 1.0, |
| "step": 17 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 100.12860870361328, |
| "epoch": 0.2647058823529412, |
| "grad_norm": 2.256336954402879, |
| "kl": 0.0137939453125, |
| "learning_rate": 9.86764705882353e-07, |
| "loss": 0.0005, |
| "reward": 1.8160377740859985, |
| "reward_std": 0.049329712986946106, |
| "rewards/accuracy_reward": 0.81844162940979, |
| "rewards/format_reward": 0.9975962042808533, |
| "step": 18 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 100.83413696289062, |
| "epoch": 0.27941176470588236, |
| "grad_norm": 1.341985441593011, |
| "kl": 0.01318359375, |
| "learning_rate": 9.860294117647058e-07, |
| "loss": 0.0005, |
| "reward": 1.7914906740188599, |
| "reward_std": 0.04698958992958069, |
| "rewards/accuracy_reward": 0.7914904952049255, |
| "rewards/format_reward": 1.0, |
| "step": 19 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 102.64183044433594, |
| "epoch": 0.29411764705882354, |
| "grad_norm": 1.1417020647628722, |
| "kl": 0.0126953125, |
| "learning_rate": 9.85294117647059e-07, |
| "loss": 0.0005, |
| "reward": 1.785807728767395, |
| "reward_std": 0.05590025335550308, |
| "rewards/accuracy_reward": 0.7906152606010437, |
| "rewards/format_reward": 0.9951923489570618, |
| "step": 20 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.41106414794922, |
| "epoch": 0.3088235294117647, |
| "grad_norm": 1.3996656626391477, |
| "kl": 0.0118408203125, |
| "learning_rate": 9.845588235294118e-07, |
| "loss": 0.0005, |
| "reward": 1.8126260042190552, |
| "reward_std": 0.04200880602002144, |
| "rewards/accuracy_reward": 0.8126258254051208, |
| "rewards/format_reward": 1.0, |
| "step": 21 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.72956848144531, |
| "epoch": 0.3235294117647059, |
| "grad_norm": 1.472735800879064, |
| "kl": 0.01214599609375, |
| "learning_rate": 9.838235294117647e-07, |
| "loss": 0.0005, |
| "reward": 1.819865107536316, |
| "reward_std": 0.046603430062532425, |
| "rewards/accuracy_reward": 0.8210669755935669, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 22 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.89904022216797, |
| "epoch": 0.3382352941176471, |
| "grad_norm": 1.5355589885672927, |
| "kl": 0.011962890625, |
| "learning_rate": 9.830882352941175e-07, |
| "loss": 0.0005, |
| "reward": 1.795883059501648, |
| "reward_std": 0.047029443085193634, |
| "rewards/accuracy_reward": 0.795883059501648, |
| "rewards/format_reward": 1.0, |
| "step": 23 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.52764892578125, |
| "epoch": 0.35294117647058826, |
| "grad_norm": 1.7659139234832795, |
| "kl": 0.01116943359375, |
| "learning_rate": 9.823529411764704e-07, |
| "loss": 0.0005, |
| "reward": 1.819062352180481, |
| "reward_std": 0.045326538383960724, |
| "rewards/accuracy_reward": 0.819062352180481, |
| "rewards/format_reward": 1.0, |
| "step": 24 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.76202392578125, |
| "epoch": 0.36764705882352944, |
| "grad_norm": 1.8387756083505316, |
| "kl": 0.0118408203125, |
| "learning_rate": 9.816176470588235e-07, |
| "loss": 0.0005, |
| "reward": 1.8244253396987915, |
| "reward_std": 0.04212662950158119, |
| "rewards/accuracy_reward": 0.8256272673606873, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 25 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.07572174072266, |
| "epoch": 0.38235294117647056, |
| "grad_norm": 2.5672205894145668, |
| "kl": 0.01123046875, |
| "learning_rate": 9.808823529411764e-07, |
| "loss": 0.0004, |
| "reward": 1.8216837644577026, |
| "reward_std": 0.03997746482491493, |
| "rewards/accuracy_reward": 0.8216836452484131, |
| "rewards/format_reward": 1.0, |
| "step": 26 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.86659240722656, |
| "epoch": 0.39705882352941174, |
| "grad_norm": 1.9492202384769068, |
| "kl": 0.0111083984375, |
| "learning_rate": 9.801470588235292e-07, |
| "loss": 0.0004, |
| "reward": 1.8016602993011475, |
| "reward_std": 0.041357386857271194, |
| "rewards/accuracy_reward": 0.8028621077537537, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 27 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.33173370361328, |
| "epoch": 0.4117647058823529, |
| "grad_norm": 1.5368419770860244, |
| "kl": 0.01300048828125, |
| "learning_rate": 9.794117647058823e-07, |
| "loss": 0.0005, |
| "reward": 1.8133355379104614, |
| "reward_std": 0.040824707597494125, |
| "rewards/accuracy_reward": 0.8145372271537781, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 28 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.08413696289062, |
| "epoch": 0.4264705882352941, |
| "grad_norm": 1.922839715769838, |
| "kl": 0.01397705078125, |
| "learning_rate": 9.786764705882352e-07, |
| "loss": 0.0006, |
| "reward": 1.828223705291748, |
| "reward_std": 0.038493864238262177, |
| "rewards/accuracy_reward": 0.8294253945350647, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 29 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.17668914794922, |
| "epoch": 0.4411764705882353, |
| "grad_norm": 1.136750055086542, |
| "kl": 0.0137939453125, |
| "learning_rate": 9.77941176470588e-07, |
| "loss": 0.0006, |
| "reward": 1.7888784408569336, |
| "reward_std": 0.03950018435716629, |
| "rewards/accuracy_reward": 0.7888783812522888, |
| "rewards/format_reward": 1.0, |
| "step": 30 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.23678588867188, |
| "epoch": 0.45588235294117646, |
| "grad_norm": 1.5207542293833838, |
| "kl": 0.013916015625, |
| "learning_rate": 9.772058823529412e-07, |
| "loss": 0.0006, |
| "reward": 1.8419325351715088, |
| "reward_std": 0.04379018768668175, |
| "rewards/accuracy_reward": 0.8443362712860107, |
| "rewards/format_reward": 0.9975962042808533, |
| "step": 31 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.47956848144531, |
| "epoch": 0.47058823529411764, |
| "grad_norm": 1.7773456348293046, |
| "kl": 0.0145263671875, |
| "learning_rate": 9.76470588235294e-07, |
| "loss": 0.0006, |
| "reward": 1.8314036130905151, |
| "reward_std": 0.03341993689537048, |
| "rewards/accuracy_reward": 0.8314037919044495, |
| "rewards/format_reward": 1.0, |
| "step": 32 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.88702392578125, |
| "epoch": 0.4852941176470588, |
| "grad_norm": 1.4109993423092686, |
| "kl": 0.01544189453125, |
| "learning_rate": 9.75735294117647e-07, |
| "loss": 0.0006, |
| "reward": 1.8262065649032593, |
| "reward_std": 0.039565280079841614, |
| "rewards/accuracy_reward": 0.8286103010177612, |
| "rewards/format_reward": 0.9975962042808533, |
| "step": 33 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.64063262939453, |
| "epoch": 0.5, |
| "grad_norm": 1.8369759357942195, |
| "kl": 0.0150146484375, |
| "learning_rate": 9.75e-07, |
| "loss": 0.0006, |
| "reward": 1.7878358364105225, |
| "reward_std": 0.03487839177250862, |
| "rewards/accuracy_reward": 0.7890377044677734, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 34 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.1658706665039, |
| "epoch": 0.5147058823529411, |
| "grad_norm": 1.4792457830014905, |
| "kl": 0.0169677734375, |
| "learning_rate": 9.742647058823529e-07, |
| "loss": 0.0007, |
| "reward": 1.8230891227722168, |
| "reward_std": 0.03256754204630852, |
| "rewards/accuracy_reward": 0.823089063167572, |
| "rewards/format_reward": 1.0, |
| "step": 35 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.7596206665039, |
| "epoch": 0.5294117647058824, |
| "grad_norm": 1.2939561711920475, |
| "kl": 0.0167236328125, |
| "learning_rate": 9.735294117647057e-07, |
| "loss": 0.0007, |
| "reward": 1.8332778215408325, |
| "reward_std": 0.03907008469104767, |
| "rewards/accuracy_reward": 0.835681676864624, |
| "rewards/format_reward": 0.9975962042808533, |
| "step": 36 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.78366088867188, |
| "epoch": 0.5441176470588235, |
| "grad_norm": 1.1548157711618783, |
| "kl": 0.0172119140625, |
| "learning_rate": 9.727941176470588e-07, |
| "loss": 0.0007, |
| "reward": 1.8284510374069214, |
| "reward_std": 0.03708866238594055, |
| "rewards/accuracy_reward": 0.8296528458595276, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 37 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 114.41346740722656, |
| "epoch": 0.5588235294117647, |
| "grad_norm": 1.811998457382516, |
| "kl": 0.0174560546875, |
| "learning_rate": 9.720588235294117e-07, |
| "loss": 0.0007, |
| "reward": 1.8531993627548218, |
| "reward_std": 0.03520160913467407, |
| "rewards/accuracy_reward": 0.8544012308120728, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 38 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 114.73798370361328, |
| "epoch": 0.5735294117647058, |
| "grad_norm": 5.782853992835495, |
| "kl": 0.0179443359375, |
| "learning_rate": 9.713235294117646e-07, |
| "loss": 0.0007, |
| "reward": 1.820478081703186, |
| "reward_std": 0.0349760465323925, |
| "rewards/accuracy_reward": 0.8204782009124756, |
| "rewards/format_reward": 1.0, |
| "step": 39 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 117.25360870361328, |
| "epoch": 0.5882352941176471, |
| "grad_norm": 2.095875673750828, |
| "kl": 0.02001953125, |
| "learning_rate": 9.705882352941176e-07, |
| "loss": 0.0008, |
| "reward": 1.818333625793457, |
| "reward_std": 0.036586832255125046, |
| "rewards/accuracy_reward": 0.8195353746414185, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 40 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 119.5564956665039, |
| "epoch": 0.6029411764705882, |
| "grad_norm": 1.5695506518790454, |
| "kl": 0.0205078125, |
| "learning_rate": 9.698529411764705e-07, |
| "loss": 0.0008, |
| "reward": 1.7950727939605713, |
| "reward_std": 0.043350886553525925, |
| "rewards/accuracy_reward": 0.797476589679718, |
| "rewards/format_reward": 0.9975962042808533, |
| "step": 41 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 115.74159240722656, |
| "epoch": 0.6176470588235294, |
| "grad_norm": 4.732453320301059, |
| "kl": 0.0220947265625, |
| "learning_rate": 9.691176470588234e-07, |
| "loss": 0.0009, |
| "reward": 1.8049029111862183, |
| "reward_std": 0.03959096223115921, |
| "rewards/accuracy_reward": 0.8061047792434692, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 42 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 116.66706848144531, |
| "epoch": 0.6323529411764706, |
| "grad_norm": 2.4880576589131533, |
| "kl": 0.021484375, |
| "learning_rate": 9.683823529411765e-07, |
| "loss": 0.0009, |
| "reward": 1.7938587665557861, |
| "reward_std": 0.03622143343091011, |
| "rewards/accuracy_reward": 0.7938587665557861, |
| "rewards/format_reward": 1.0, |
| "step": 43 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 115.97356414794922, |
| "epoch": 0.6470588235294118, |
| "grad_norm": 2.4237436345937122, |
| "kl": 0.022705078125, |
| "learning_rate": 9.676470588235294e-07, |
| "loss": 0.0009, |
| "reward": 1.8250298500061035, |
| "reward_std": 0.03410777822136879, |
| "rewards/accuracy_reward": 0.8262317776679993, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 44 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.92788696289062, |
| "epoch": 0.6617647058823529, |
| "grad_norm": 2.2338566414155463, |
| "kl": 0.022705078125, |
| "learning_rate": 9.669117647058822e-07, |
| "loss": 0.0009, |
| "reward": 1.8153414726257324, |
| "reward_std": 0.043387699872255325, |
| "rewards/accuracy_reward": 0.8189471364021301, |
| "rewards/format_reward": 0.9963942766189575, |
| "step": 45 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.89543914794922, |
| "epoch": 0.6764705882352942, |
| "grad_norm": 11.225902629577423, |
| "kl": 0.0225830078125, |
| "learning_rate": 9.661764705882353e-07, |
| "loss": 0.0009, |
| "reward": 1.8124586343765259, |
| "reward_std": 0.033101461827754974, |
| "rewards/accuracy_reward": 0.8124586343765259, |
| "rewards/format_reward": 1.0, |
| "step": 46 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.8738021850586, |
| "epoch": 0.6911764705882353, |
| "grad_norm": 2.4337206941159564, |
| "kl": 0.0216064453125, |
| "learning_rate": 9.654411764705882e-07, |
| "loss": 0.0009, |
| "reward": 1.82453191280365, |
| "reward_std": 0.033501800149679184, |
| "rewards/accuracy_reward": 0.8257339596748352, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 47 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.66466522216797, |
| "epoch": 0.7058823529411765, |
| "grad_norm": 1.6698840088478868, |
| "kl": 0.02587890625, |
| "learning_rate": 9.64705882352941e-07, |
| "loss": 0.001, |
| "reward": 1.8177635669708252, |
| "reward_std": 0.03327854722738266, |
| "rewards/accuracy_reward": 0.8177635073661804, |
| "rewards/format_reward": 1.0, |
| "step": 48 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.1189956665039, |
| "epoch": 0.7205882352941176, |
| "grad_norm": 2.838491633259207, |
| "kl": 0.02294921875, |
| "learning_rate": 9.639705882352941e-07, |
| "loss": 0.0009, |
| "reward": 1.8294517993927002, |
| "reward_std": 0.03862498328089714, |
| "rewards/accuracy_reward": 0.8318556547164917, |
| "rewards/format_reward": 0.9975962042808533, |
| "step": 49 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.31250762939453, |
| "epoch": 0.7352941176470589, |
| "grad_norm": 1.6937318420428593, |
| "kl": 0.0234375, |
| "learning_rate": 9.63235294117647e-07, |
| "loss": 0.0009, |
| "reward": 1.8327486515045166, |
| "reward_std": 0.03481902554631233, |
| "rewards/accuracy_reward": 0.8339505195617676, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 50 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.50241088867188, |
| "epoch": 0.75, |
| "grad_norm": 1.9451567930831182, |
| "kl": 0.0245361328125, |
| "learning_rate": 9.624999999999999e-07, |
| "loss": 0.001, |
| "reward": 1.839182734489441, |
| "reward_std": 0.029496613889932632, |
| "rewards/accuracy_reward": 0.8391827940940857, |
| "rewards/format_reward": 1.0, |
| "step": 51 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.64904022216797, |
| "epoch": 0.7647058823529411, |
| "grad_norm": 1.9537307288218644, |
| "kl": 0.023681640625, |
| "learning_rate": 9.61764705882353e-07, |
| "loss": 0.0009, |
| "reward": 1.8222689628601074, |
| "reward_std": 0.038959018886089325, |
| "rewards/accuracy_reward": 0.8234708309173584, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 52 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.05529022216797, |
| "epoch": 0.7794117647058824, |
| "grad_norm": 2.0869715773084634, |
| "kl": 0.022705078125, |
| "learning_rate": 9.610294117647058e-07, |
| "loss": 0.0009, |
| "reward": 1.8396120071411133, |
| "reward_std": 0.030058706179261208, |
| "rewards/accuracy_reward": 0.8396119475364685, |
| "rewards/format_reward": 1.0, |
| "step": 53 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.25241088867188, |
| "epoch": 0.7941176470588235, |
| "grad_norm": 1.399697409291221, |
| "kl": 0.0230712890625, |
| "learning_rate": 9.602941176470587e-07, |
| "loss": 0.0009, |
| "reward": 1.8265293836593628, |
| "reward_std": 0.02903612144291401, |
| "rewards/accuracy_reward": 0.8265291452407837, |
| "rewards/format_reward": 1.0, |
| "step": 54 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.35697174072266, |
| "epoch": 0.8088235294117647, |
| "grad_norm": 1.9480357994696749, |
| "kl": 0.022705078125, |
| "learning_rate": 9.595588235294118e-07, |
| "loss": 0.0009, |
| "reward": 1.8046581745147705, |
| "reward_std": 0.030282124876976013, |
| "rewards/accuracy_reward": 0.8046582341194153, |
| "rewards/format_reward": 1.0, |
| "step": 55 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.97356414794922, |
| "epoch": 0.8235294117647058, |
| "grad_norm": 1.8194034842124802, |
| "kl": 0.0218505859375, |
| "learning_rate": 9.588235294117647e-07, |
| "loss": 0.0009, |
| "reward": 1.8263245820999146, |
| "reward_std": 0.029453594237565994, |
| "rewards/accuracy_reward": 0.8275262713432312, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 56 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.06731414794922, |
| "epoch": 0.8382352941176471, |
| "grad_norm": 1.747459486428414, |
| "kl": 0.02001953125, |
| "learning_rate": 9.580882352941176e-07, |
| "loss": 0.0008, |
| "reward": 1.8315790891647339, |
| "reward_std": 0.02826782502233982, |
| "rewards/accuracy_reward": 0.8315791487693787, |
| "rewards/format_reward": 1.0, |
| "step": 57 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.09616088867188, |
| "epoch": 0.8529411764705882, |
| "grad_norm": 2.2648825774011856, |
| "kl": 0.019775390625, |
| "learning_rate": 9.573529411764706e-07, |
| "loss": 0.0008, |
| "reward": 1.8353791236877441, |
| "reward_std": 0.028635086491703987, |
| "rewards/accuracy_reward": 0.8353790640830994, |
| "rewards/format_reward": 1.0, |
| "step": 58 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.56851196289062, |
| "epoch": 0.8676470588235294, |
| "grad_norm": 1.3353893541096462, |
| "kl": 0.02001953125, |
| "learning_rate": 9.566176470588235e-07, |
| "loss": 0.0008, |
| "reward": 1.8366105556488037, |
| "reward_std": 0.029847299680113792, |
| "rewards/accuracy_reward": 0.8378124833106995, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 59 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.87741088867188, |
| "epoch": 0.8823529411764706, |
| "grad_norm": 1.2231786582956035, |
| "kl": 0.021484375, |
| "learning_rate": 9.558823529411764e-07, |
| "loss": 0.0009, |
| "reward": 1.8441990613937378, |
| "reward_std": 0.029375040903687477, |
| "rewards/accuracy_reward": 0.8441989421844482, |
| "rewards/format_reward": 1.0, |
| "step": 60 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.85697174072266, |
| "epoch": 0.8970588235294118, |
| "grad_norm": 1.6690678560710506, |
| "kl": 0.01904296875, |
| "learning_rate": 9.551470588235295e-07, |
| "loss": 0.0008, |
| "reward": 1.8236721754074097, |
| "reward_std": 0.02699529379606247, |
| "rewards/accuracy_reward": 0.8236721158027649, |
| "rewards/format_reward": 1.0, |
| "step": 61 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.2908706665039, |
| "epoch": 0.9117647058823529, |
| "grad_norm": 6.089899029100757, |
| "kl": 0.019287109375, |
| "learning_rate": 9.544117647058823e-07, |
| "loss": 0.0008, |
| "reward": 1.8286242485046387, |
| "reward_std": 0.02875707298517227, |
| "rewards/accuracy_reward": 0.8286241888999939, |
| "rewards/format_reward": 1.0, |
| "step": 62 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.53846740722656, |
| "epoch": 0.9264705882352942, |
| "grad_norm": 1.8276415184151562, |
| "kl": 0.0201416015625, |
| "learning_rate": 9.536764705882352e-07, |
| "loss": 0.0008, |
| "reward": 1.8171989917755127, |
| "reward_std": 0.033383674919605255, |
| "rewards/accuracy_reward": 0.8184008002281189, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 63 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.20553588867188, |
| "epoch": 0.9411764705882353, |
| "grad_norm": 2.253394799784533, |
| "kl": 0.0206298828125, |
| "learning_rate": 9.529411764705881e-07, |
| "loss": 0.0008, |
| "reward": 1.8275461196899414, |
| "reward_std": 0.033603277057409286, |
| "rewards/accuracy_reward": 0.8287478685379028, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 64 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.28366088867188, |
| "epoch": 0.9558823529411765, |
| "grad_norm": 1.9640142035092505, |
| "kl": 0.0203857421875, |
| "learning_rate": 9.522058823529411e-07, |
| "loss": 0.0008, |
| "reward": 1.8377821445465088, |
| "reward_std": 0.027394887059926987, |
| "rewards/accuracy_reward": 0.8389840126037598, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 65 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.56971740722656, |
| "epoch": 0.9705882352941176, |
| "grad_norm": 1.7539526199640756, |
| "kl": 0.021484375, |
| "learning_rate": 9.51470588235294e-07, |
| "loss": 0.0009, |
| "reward": 1.8167920112609863, |
| "reward_std": 0.02794465236365795, |
| "rewards/accuracy_reward": 0.8167920112609863, |
| "rewards/format_reward": 1.0, |
| "step": 66 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.77284240722656, |
| "epoch": 0.9852941176470589, |
| "grad_norm": 1.2641268462386457, |
| "kl": 0.0228271484375, |
| "learning_rate": 9.507352941176469e-07, |
| "loss": 0.0009, |
| "reward": 1.8030400276184082, |
| "reward_std": 0.03198782354593277, |
| "rewards/accuracy_reward": 0.8030399680137634, |
| "rewards/format_reward": 1.0, |
| "step": 67 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.8310546875, |
| "epoch": 1.0, |
| "grad_norm": 1.63346443563275, |
| "kl": 0.022216796875, |
| "learning_rate": 9.499999999999999e-07, |
| "loss": 0.0009, |
| "reward": 1.8402751684188843, |
| "reward_std": 0.028587892651557922, |
| "rewards/accuracy_reward": 0.8402751684188843, |
| "rewards/format_reward": 1.0, |
| "step": 68 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.18991088867188, |
| "epoch": 1.0147058823529411, |
| "grad_norm": 2.0248373653875955, |
| "kl": 0.0225830078125, |
| "learning_rate": 9.492647058823529e-07, |
| "loss": 0.0009, |
| "reward": 1.8379939794540405, |
| "reward_std": 0.02731536142528057, |
| "rewards/accuracy_reward": 0.8379938006401062, |
| "rewards/format_reward": 1.0, |
| "step": 69 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.76923370361328, |
| "epoch": 1.0294117647058822, |
| "grad_norm": 1.2467348089074899, |
| "kl": 0.0224609375, |
| "learning_rate": 9.485294117647058e-07, |
| "loss": 0.0009, |
| "reward": 1.839119553565979, |
| "reward_std": 0.027092870324850082, |
| "rewards/accuracy_reward": 0.8391194939613342, |
| "rewards/format_reward": 1.0, |
| "step": 70 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.04447174072266, |
| "epoch": 1.0441176470588236, |
| "grad_norm": 1.7733203946188572, |
| "kl": 0.024169921875, |
| "learning_rate": 9.477941176470587e-07, |
| "loss": 0.001, |
| "reward": 1.835349678993225, |
| "reward_std": 0.028652215376496315, |
| "rewards/accuracy_reward": 0.8365515470504761, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 71 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.54206848144531, |
| "epoch": 1.0588235294117647, |
| "grad_norm": 1.417496550727975, |
| "kl": 0.0255126953125, |
| "learning_rate": 9.470588235294117e-07, |
| "loss": 0.001, |
| "reward": 1.8174885511398315, |
| "reward_std": 0.03328551724553108, |
| "rewards/accuracy_reward": 0.8186904191970825, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 72 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.22956848144531, |
| "epoch": 1.0735294117647058, |
| "grad_norm": 2.347300550273842, |
| "kl": 0.022705078125, |
| "learning_rate": 9.463235294117646e-07, |
| "loss": 0.0009, |
| "reward": 1.8195103406906128, |
| "reward_std": 0.028913108631968498, |
| "rewards/accuracy_reward": 0.8195103406906128, |
| "rewards/format_reward": 1.0, |
| "step": 73 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.82572174072266, |
| "epoch": 1.088235294117647, |
| "grad_norm": 2.5948956191316306, |
| "kl": 0.024169921875, |
| "learning_rate": 9.455882352941176e-07, |
| "loss": 0.001, |
| "reward": 1.8160063028335571, |
| "reward_std": 0.03166431561112404, |
| "rewards/accuracy_reward": 0.8160063028335571, |
| "rewards/format_reward": 1.0, |
| "step": 74 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.91106414794922, |
| "epoch": 1.1029411764705883, |
| "grad_norm": 2.2984611372489847, |
| "kl": 0.0245361328125, |
| "learning_rate": 9.448529411764705e-07, |
| "loss": 0.001, |
| "reward": 1.841678261756897, |
| "reward_std": 0.028328273445367813, |
| "rewards/accuracy_reward": 0.8416782021522522, |
| "rewards/format_reward": 1.0, |
| "step": 75 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.30769348144531, |
| "epoch": 1.1176470588235294, |
| "grad_norm": 1.2558402021274238, |
| "kl": 0.025146484375, |
| "learning_rate": 9.441176470588234e-07, |
| "loss": 0.001, |
| "reward": 1.8383431434631348, |
| "reward_std": 0.02794811502099037, |
| "rewards/accuracy_reward": 0.8395450711250305, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 76 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.67428588867188, |
| "epoch": 1.1323529411764706, |
| "grad_norm": 1.4150029392598835, |
| "kl": 0.02392578125, |
| "learning_rate": 9.433823529411764e-07, |
| "loss": 0.001, |
| "reward": 1.8399676084518433, |
| "reward_std": 0.025521790608763695, |
| "rewards/accuracy_reward": 0.8399675488471985, |
| "rewards/format_reward": 1.0, |
| "step": 77 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.96034240722656, |
| "epoch": 1.1470588235294117, |
| "grad_norm": 1.1950823033585802, |
| "kl": 0.0264892578125, |
| "learning_rate": 9.426470588235294e-07, |
| "loss": 0.0011, |
| "reward": 1.83869206905365, |
| "reward_std": 0.030266188085079193, |
| "rewards/accuracy_reward": 0.8398938775062561, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 78 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.77284240722656, |
| "epoch": 1.161764705882353, |
| "grad_norm": 3.5214271570942355, |
| "kl": 0.0272216796875, |
| "learning_rate": 9.419117647058823e-07, |
| "loss": 0.0011, |
| "reward": 1.8469470739364624, |
| "reward_std": 0.025917939841747284, |
| "rewards/accuracy_reward": 0.8469470143318176, |
| "rewards/format_reward": 1.0, |
| "step": 79 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 114.43991088867188, |
| "epoch": 1.1764705882352942, |
| "grad_norm": 1.5192920476546525, |
| "kl": 0.027099609375, |
| "learning_rate": 9.411764705882352e-07, |
| "loss": 0.0011, |
| "reward": 1.8514484167099, |
| "reward_std": 0.02656027302145958, |
| "rewards/accuracy_reward": 0.8514483571052551, |
| "rewards/format_reward": 1.0, |
| "step": 80 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 114.13581848144531, |
| "epoch": 1.1911764705882353, |
| "grad_norm": 1.4307870600496726, |
| "kl": 0.027099609375, |
| "learning_rate": 9.404411764705882e-07, |
| "loss": 0.0011, |
| "reward": 1.848158359527588, |
| "reward_std": 0.03505507484078407, |
| "rewards/accuracy_reward": 0.8505621552467346, |
| "rewards/format_reward": 0.9975962042808533, |
| "step": 81 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 114.22356414794922, |
| "epoch": 1.2058823529411764, |
| "grad_norm": 1.9122780286883485, |
| "kl": 0.042724609375, |
| "learning_rate": 9.397058823529411e-07, |
| "loss": 0.0017, |
| "reward": 1.8507599830627441, |
| "reward_std": 0.027777796611189842, |
| "rewards/accuracy_reward": 0.8519617319107056, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 82 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 114.24159240722656, |
| "epoch": 1.2205882352941178, |
| "grad_norm": 1.2485868300090033, |
| "kl": 0.0281982421875, |
| "learning_rate": 9.389705882352941e-07, |
| "loss": 0.0011, |
| "reward": 1.8573682308197021, |
| "reward_std": 0.028743639588356018, |
| "rewards/accuracy_reward": 0.8585702180862427, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 83 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.99038696289062, |
| "epoch": 1.2352941176470589, |
| "grad_norm": 1.6395721460640882, |
| "kl": 0.02587890625, |
| "learning_rate": 9.38235294117647e-07, |
| "loss": 0.001, |
| "reward": 1.849989414215088, |
| "reward_std": 0.03292028233408928, |
| "rewards/accuracy_reward": 0.8523932695388794, |
| "rewards/format_reward": 0.9975962042808533, |
| "step": 84 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.15144348144531, |
| "epoch": 1.25, |
| "grad_norm": 1.8788708892556627, |
| "kl": 0.029296875, |
| "learning_rate": 9.374999999999999e-07, |
| "loss": 0.0012, |
| "reward": 1.8299332857131958, |
| "reward_std": 0.030824122950434685, |
| "rewards/accuracy_reward": 0.832336962223053, |
| "rewards/format_reward": 0.9975962042808533, |
| "step": 85 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.70072174072266, |
| "epoch": 1.2647058823529411, |
| "grad_norm": 1.284626382800343, |
| "kl": 0.0257568359375, |
| "learning_rate": 9.367647058823529e-07, |
| "loss": 0.001, |
| "reward": 1.8247534036636353, |
| "reward_std": 0.02943887747824192, |
| "rewards/accuracy_reward": 0.825955331325531, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 86 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.10938262939453, |
| "epoch": 1.2794117647058822, |
| "grad_norm": 1.800940971026441, |
| "kl": 0.0291748046875, |
| "learning_rate": 9.360294117647059e-07, |
| "loss": 0.0012, |
| "reward": 1.829021692276001, |
| "reward_std": 0.03014710173010826, |
| "rewards/accuracy_reward": 0.8290215730667114, |
| "rewards/format_reward": 1.0, |
| "step": 87 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.72356414794922, |
| "epoch": 1.2941176470588236, |
| "grad_norm": 1.6669503142403084, |
| "kl": 0.027587890625, |
| "learning_rate": 9.352941176470588e-07, |
| "loss": 0.0011, |
| "reward": 1.8365869522094727, |
| "reward_std": 0.035319458693265915, |
| "rewards/accuracy_reward": 0.8389906287193298, |
| "rewards/format_reward": 0.9975962042808533, |
| "step": 88 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.84495544433594, |
| "epoch": 1.3088235294117647, |
| "grad_norm": 1.9827979032477325, |
| "kl": 0.0267333984375, |
| "learning_rate": 9.345588235294117e-07, |
| "loss": 0.0011, |
| "reward": 1.8515337705612183, |
| "reward_std": 0.032718025147914886, |
| "rewards/accuracy_reward": 0.8527354598045349, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 89 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.54688262939453, |
| "epoch": 1.3235294117647058, |
| "grad_norm": 1.855181016525942, |
| "kl": 0.029541015625, |
| "learning_rate": 9.338235294117647e-07, |
| "loss": 0.0012, |
| "reward": 1.8531250953674316, |
| "reward_std": 0.03333236649632454, |
| "rewards/accuracy_reward": 0.8555287718772888, |
| "rewards/format_reward": 0.9975962042808533, |
| "step": 90 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.51683044433594, |
| "epoch": 1.3382352941176472, |
| "grad_norm": 1.542405666476553, |
| "kl": 0.0299072265625, |
| "learning_rate": 9.330882352941176e-07, |
| "loss": 0.0012, |
| "reward": 1.8587943315505981, |
| "reward_std": 0.02777322567999363, |
| "rewards/accuracy_reward": 0.8587943315505981, |
| "rewards/format_reward": 1.0, |
| "step": 91 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.65866088867188, |
| "epoch": 1.3529411764705883, |
| "grad_norm": 6.044720279013163, |
| "kl": 0.027099609375, |
| "learning_rate": 9.323529411764706e-07, |
| "loss": 0.0011, |
| "reward": 1.8246790170669556, |
| "reward_std": 0.03338155895471573, |
| "rewards/accuracy_reward": 0.8258809447288513, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 92 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.25481414794922, |
| "epoch": 1.3676470588235294, |
| "grad_norm": 3.698518846068984, |
| "kl": 0.026611328125, |
| "learning_rate": 9.316176470588235e-07, |
| "loss": 0.0011, |
| "reward": 1.8631399869918823, |
| "reward_std": 0.027002353221178055, |
| "rewards/accuracy_reward": 0.8631399869918823, |
| "rewards/format_reward": 1.0, |
| "step": 93 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.18870544433594, |
| "epoch": 1.3823529411764706, |
| "grad_norm": 7.435870249889465, |
| "kl": 0.027587890625, |
| "learning_rate": 9.308823529411764e-07, |
| "loss": 0.0011, |
| "reward": 1.8605844974517822, |
| "reward_std": 0.02926478162407875, |
| "rewards/accuracy_reward": 0.8605843782424927, |
| "rewards/format_reward": 1.0, |
| "step": 94 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.17188262939453, |
| "epoch": 1.3970588235294117, |
| "grad_norm": 3.53776006589639, |
| "kl": 0.038818359375, |
| "learning_rate": 9.301470588235294e-07, |
| "loss": 0.0016, |
| "reward": 1.8662550449371338, |
| "reward_std": 0.028098618611693382, |
| "rewards/accuracy_reward": 0.8662549257278442, |
| "rewards/format_reward": 1.0, |
| "step": 95 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.53606414794922, |
| "epoch": 1.4117647058823528, |
| "grad_norm": 2.008765777352944, |
| "kl": 0.0277099609375, |
| "learning_rate": 9.294117647058824e-07, |
| "loss": 0.0011, |
| "reward": 1.8323391675949097, |
| "reward_std": 0.03244079276919365, |
| "rewards/accuracy_reward": 0.8335410356521606, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 96 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.32813262939453, |
| "epoch": 1.4264705882352942, |
| "grad_norm": 1.6277308554524441, |
| "kl": 0.026123046875, |
| "learning_rate": 9.286764705882352e-07, |
| "loss": 0.001, |
| "reward": 1.8551498651504517, |
| "reward_std": 0.027026960626244545, |
| "rewards/accuracy_reward": 0.8551498055458069, |
| "rewards/format_reward": 1.0, |
| "step": 97 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.96514892578125, |
| "epoch": 1.4411764705882353, |
| "grad_norm": 1.8674445586310056, |
| "kl": 0.027587890625, |
| "learning_rate": 9.279411764705882e-07, |
| "loss": 0.0011, |
| "reward": 1.8556426763534546, |
| "reward_std": 0.02985842153429985, |
| "rewards/accuracy_reward": 0.855642557144165, |
| "rewards/format_reward": 1.0, |
| "step": 98 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.22116088867188, |
| "epoch": 1.4558823529411764, |
| "grad_norm": 2.564978355227823, |
| "kl": 0.031494140625, |
| "learning_rate": 9.272058823529412e-07, |
| "loss": 0.0013, |
| "reward": 1.8616751432418823, |
| "reward_std": 0.028857456520199776, |
| "rewards/accuracy_reward": 0.8628771305084229, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 99 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.96034240722656, |
| "epoch": 1.4705882352941178, |
| "grad_norm": 1.5062130671090712, |
| "kl": 0.029052734375, |
| "learning_rate": 9.264705882352941e-07, |
| "loss": 0.0012, |
| "reward": 1.8399274349212646, |
| "reward_std": 0.033529430627822876, |
| "rewards/accuracy_reward": 0.8411291837692261, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 100 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.31731414794922, |
| "epoch": 1.4852941176470589, |
| "grad_norm": 1.7672687152261588, |
| "kl": 0.0301513671875, |
| "learning_rate": 9.257352941176471e-07, |
| "loss": 0.0012, |
| "reward": 1.8533381223678589, |
| "reward_std": 0.03230126574635506, |
| "rewards/accuracy_reward": 0.8545399308204651, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 101 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.50601196289062, |
| "epoch": 1.5, |
| "grad_norm": 1.2513961138527014, |
| "kl": 0.0294189453125, |
| "learning_rate": 9.25e-07, |
| "loss": 0.0012, |
| "reward": 1.8355013132095337, |
| "reward_std": 0.03523796796798706, |
| "rewards/accuracy_reward": 0.8367031812667847, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 102 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.85096740722656, |
| "epoch": 1.5147058823529411, |
| "grad_norm": 1.7612446743928787, |
| "kl": 0.0286865234375, |
| "learning_rate": 9.242647058823529e-07, |
| "loss": 0.0011, |
| "reward": 1.835523247718811, |
| "reward_std": 0.028739692643284798, |
| "rewards/accuracy_reward": 0.8355231285095215, |
| "rewards/format_reward": 1.0, |
| "step": 103 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.06971740722656, |
| "epoch": 1.5294117647058822, |
| "grad_norm": 1.2684013897529154, |
| "kl": 0.0296630859375, |
| "learning_rate": 9.235294117647059e-07, |
| "loss": 0.0012, |
| "reward": 1.8417116403579712, |
| "reward_std": 0.03294159471988678, |
| "rewards/accuracy_reward": 0.8429136872291565, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 104 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.39063262939453, |
| "epoch": 1.5441176470588234, |
| "grad_norm": 1.5110927030527903, |
| "kl": 0.0263671875, |
| "learning_rate": 9.227941176470589e-07, |
| "loss": 0.0011, |
| "reward": 1.8465179204940796, |
| "reward_std": 0.035068195313215256, |
| "rewards/accuracy_reward": 0.8489217162132263, |
| "rewards/format_reward": 0.9975962042808533, |
| "step": 105 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.76202392578125, |
| "epoch": 1.5588235294117647, |
| "grad_norm": 4.89157193030413, |
| "kl": 0.0299072265625, |
| "learning_rate": 9.220588235294117e-07, |
| "loss": 0.0012, |
| "reward": 1.8534250259399414, |
| "reward_std": 0.029864592477679253, |
| "rewards/accuracy_reward": 0.8534249663352966, |
| "rewards/format_reward": 1.0, |
| "step": 106 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.63822174072266, |
| "epoch": 1.5735294117647058, |
| "grad_norm": 1.397310457995284, |
| "kl": 0.0264892578125, |
| "learning_rate": 9.213235294117646e-07, |
| "loss": 0.0011, |
| "reward": 1.8602780103683472, |
| "reward_std": 0.0308562982827425, |
| "rewards/accuracy_reward": 0.8614799380302429, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 107 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.93870544433594, |
| "epoch": 1.5882352941176472, |
| "grad_norm": 2.2153563584401867, |
| "kl": 0.027587890625, |
| "learning_rate": 9.205882352941176e-07, |
| "loss": 0.0011, |
| "reward": 1.8766840696334839, |
| "reward_std": 0.032767944037914276, |
| "rewards/accuracy_reward": 0.8778861165046692, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 108 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.41706848144531, |
| "epoch": 1.6029411764705883, |
| "grad_norm": 1.3441095236077332, |
| "kl": 0.0277099609375, |
| "learning_rate": 9.198529411764705e-07, |
| "loss": 0.0011, |
| "reward": 1.8476648330688477, |
| "reward_std": 0.03170119971036911, |
| "rewards/accuracy_reward": 0.8476646542549133, |
| "rewards/format_reward": 1.0, |
| "step": 109 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 114.37500762939453, |
| "epoch": 1.6176470588235294, |
| "grad_norm": 2.087440395995137, |
| "kl": 0.0302734375, |
| "learning_rate": 9.191176470588234e-07, |
| "loss": 0.0012, |
| "reward": 1.8472872972488403, |
| "reward_std": 0.032663196325302124, |
| "rewards/accuracy_reward": 0.8484892249107361, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 110 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.37259674072266, |
| "epoch": 1.6323529411764706, |
| "grad_norm": 1.540582934098744, |
| "kl": 0.031494140625, |
| "learning_rate": 9.183823529411764e-07, |
| "loss": 0.0013, |
| "reward": 1.8425360918045044, |
| "reward_std": 0.0323411226272583, |
| "rewards/accuracy_reward": 0.8425360918045044, |
| "rewards/format_reward": 1.0, |
| "step": 111 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 114.81851196289062, |
| "epoch": 1.6470588235294117, |
| "grad_norm": 2.4116409582345297, |
| "kl": 0.02978515625, |
| "learning_rate": 9.176470588235293e-07, |
| "loss": 0.0012, |
| "reward": 1.8361573219299316, |
| "reward_std": 0.0333629846572876, |
| "rewards/accuracy_reward": 0.8373592495918274, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 112 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 114.86659240722656, |
| "epoch": 1.6617647058823528, |
| "grad_norm": 2.020278808245713, |
| "kl": 0.031005859375, |
| "learning_rate": 9.169117647058823e-07, |
| "loss": 0.0012, |
| "reward": 1.8621249198913574, |
| "reward_std": 0.030686957761645317, |
| "rewards/accuracy_reward": 0.8621248602867126, |
| "rewards/format_reward": 1.0, |
| "step": 113 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 117.34134674072266, |
| "epoch": 1.6764705882352942, |
| "grad_norm": 2.599149661763795, |
| "kl": 0.0272216796875, |
| "learning_rate": 9.161764705882353e-07, |
| "loss": 0.0011, |
| "reward": 1.8338128328323364, |
| "reward_std": 0.03273686766624451, |
| "rewards/accuracy_reward": 0.8350145220756531, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 114 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 117.04206848144531, |
| "epoch": 1.6911764705882353, |
| "grad_norm": 1.696964274521337, |
| "kl": 0.02783203125, |
| "learning_rate": 9.154411764705881e-07, |
| "loss": 0.0011, |
| "reward": 1.8655701875686646, |
| "reward_std": 0.026736797764897346, |
| "rewards/accuracy_reward": 0.8655699491500854, |
| "rewards/format_reward": 1.0, |
| "step": 115 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 117.17788696289062, |
| "epoch": 1.7058823529411766, |
| "grad_norm": 1.6668443133949866, |
| "kl": 0.02880859375, |
| "learning_rate": 9.147058823529411e-07, |
| "loss": 0.0012, |
| "reward": 1.859294056892395, |
| "reward_std": 0.02911749854683876, |
| "rewards/accuracy_reward": 0.8604959845542908, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 116 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 116.41706848144531, |
| "epoch": 1.7205882352941178, |
| "grad_norm": 1.8917048442273852, |
| "kl": 0.0289306640625, |
| "learning_rate": 9.139705882352941e-07, |
| "loss": 0.0012, |
| "reward": 1.8411531448364258, |
| "reward_std": 0.03658343106508255, |
| "rewards/accuracy_reward": 0.8471627831459045, |
| "rewards/format_reward": 0.993990421295166, |
| "step": 117 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 115.32813262939453, |
| "epoch": 1.7352941176470589, |
| "grad_norm": 1.6450031082088983, |
| "kl": 0.0279541015625, |
| "learning_rate": 9.13235294117647e-07, |
| "loss": 0.0011, |
| "reward": 1.8670862913131714, |
| "reward_std": 0.026890011504292488, |
| "rewards/accuracy_reward": 0.8670862913131714, |
| "rewards/format_reward": 1.0, |
| "step": 118 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.88341522216797, |
| "epoch": 1.75, |
| "grad_norm": 3.248672120006505, |
| "kl": 0.029052734375, |
| "learning_rate": 9.124999999999999e-07, |
| "loss": 0.0012, |
| "reward": 1.8559081554412842, |
| "reward_std": 0.02590222656726837, |
| "rewards/accuracy_reward": 0.8559079170227051, |
| "rewards/format_reward": 1.0, |
| "step": 119 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.36779022216797, |
| "epoch": 1.7647058823529411, |
| "grad_norm": 20.773145033253595, |
| "kl": 0.0279541015625, |
| "learning_rate": 9.117647058823529e-07, |
| "loss": 0.0011, |
| "reward": 1.8575422763824463, |
| "reward_std": 0.024342134594917297, |
| "rewards/accuracy_reward": 0.8575422763824463, |
| "rewards/format_reward": 1.0, |
| "step": 120 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.59375762939453, |
| "epoch": 1.7794117647058822, |
| "grad_norm": 1.4987523214554013, |
| "kl": 0.0283203125, |
| "learning_rate": 9.110294117647058e-07, |
| "loss": 0.0011, |
| "reward": 1.8683563470840454, |
| "reward_std": 0.023474786430597305, |
| "rewards/accuracy_reward": 0.8683561682701111, |
| "rewards/format_reward": 1.0, |
| "step": 121 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.4363021850586, |
| "epoch": 1.7941176470588234, |
| "grad_norm": 2.899436959677707, |
| "kl": 0.02783203125, |
| "learning_rate": 9.102941176470588e-07, |
| "loss": 0.0011, |
| "reward": 1.82746160030365, |
| "reward_std": 0.027238713577389717, |
| "rewards/accuracy_reward": 0.8286634683609009, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 122 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.24279022216797, |
| "epoch": 1.8088235294117647, |
| "grad_norm": 1.4877031960129163, |
| "kl": 0.0286865234375, |
| "learning_rate": 9.095588235294118e-07, |
| "loss": 0.0011, |
| "reward": 1.8483201265335083, |
| "reward_std": 0.024529799818992615, |
| "rewards/accuracy_reward": 0.8483200669288635, |
| "rewards/format_reward": 1.0, |
| "step": 123 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.35456848144531, |
| "epoch": 1.8235294117647058, |
| "grad_norm": 2.1149935830252473, |
| "kl": 0.0308837890625, |
| "learning_rate": 9.088235294117646e-07, |
| "loss": 0.0012, |
| "reward": 1.8580820560455322, |
| "reward_std": 0.026496434584259987, |
| "rewards/accuracy_reward": 0.8580819964408875, |
| "rewards/format_reward": 1.0, |
| "step": 124 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.50721740722656, |
| "epoch": 1.8382352941176472, |
| "grad_norm": 5.027371264695678, |
| "kl": 0.031005859375, |
| "learning_rate": 9.080882352941176e-07, |
| "loss": 0.0012, |
| "reward": 1.867140293121338, |
| "reward_std": 0.02153668738901615, |
| "rewards/accuracy_reward": 0.8671402931213379, |
| "rewards/format_reward": 1.0, |
| "step": 125 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.4675521850586, |
| "epoch": 1.8529411764705883, |
| "grad_norm": 1.2899741793420323, |
| "kl": 0.028076171875, |
| "learning_rate": 9.073529411764706e-07, |
| "loss": 0.0011, |
| "reward": 1.8652453422546387, |
| "reward_std": 0.02116318792104721, |
| "rewards/accuracy_reward": 0.8652453422546387, |
| "rewards/format_reward": 1.0, |
| "step": 126 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.6502456665039, |
| "epoch": 1.8676470588235294, |
| "grad_norm": 2.179252488296008, |
| "kl": 0.031494140625, |
| "learning_rate": 9.066176470588235e-07, |
| "loss": 0.0013, |
| "reward": 1.8535821437835693, |
| "reward_std": 0.02199378050863743, |
| "rewards/accuracy_reward": 0.8535822033882141, |
| "rewards/format_reward": 1.0, |
| "step": 127 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.88581848144531, |
| "epoch": 1.8823529411764706, |
| "grad_norm": 1.4612044453819617, |
| "kl": 0.033447265625, |
| "learning_rate": 9.058823529411764e-07, |
| "loss": 0.0013, |
| "reward": 1.8556383848190308, |
| "reward_std": 0.018579188734292984, |
| "rewards/accuracy_reward": 0.8556383848190308, |
| "rewards/format_reward": 1.0, |
| "step": 128 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.41346740722656, |
| "epoch": 1.8970588235294117, |
| "grad_norm": 1.5286682380949737, |
| "kl": 0.0296630859375, |
| "learning_rate": 9.051470588235294e-07, |
| "loss": 0.0012, |
| "reward": 1.83783757686615, |
| "reward_std": 0.025251705199480057, |
| "rewards/accuracy_reward": 0.8378376364707947, |
| "rewards/format_reward": 1.0, |
| "step": 129 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.98438262939453, |
| "epoch": 1.9117647058823528, |
| "grad_norm": 2.4536921352293892, |
| "kl": 0.029296875, |
| "learning_rate": 9.044117647058823e-07, |
| "loss": 0.0012, |
| "reward": 1.866936445236206, |
| "reward_std": 0.022518165409564972, |
| "rewards/accuracy_reward": 0.8669363260269165, |
| "rewards/format_reward": 1.0, |
| "step": 130 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.33534240722656, |
| "epoch": 1.9264705882352942, |
| "grad_norm": 1.5117857063832663, |
| "kl": 0.0284423828125, |
| "learning_rate": 9.036764705882353e-07, |
| "loss": 0.0011, |
| "reward": 1.8531559705734253, |
| "reward_std": 0.024603500962257385, |
| "rewards/accuracy_reward": 0.8543578386306763, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 131 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.3533706665039, |
| "epoch": 1.9411764705882353, |
| "grad_norm": 2.703283581674193, |
| "kl": 0.0308837890625, |
| "learning_rate": 9.029411764705883e-07, |
| "loss": 0.0012, |
| "reward": 1.8581082820892334, |
| "reward_std": 0.02527976781129837, |
| "rewards/accuracy_reward": 0.8581082820892334, |
| "rewards/format_reward": 1.0, |
| "step": 132 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.80048370361328, |
| "epoch": 1.9558823529411766, |
| "grad_norm": 2.173287786867425, |
| "kl": 0.029052734375, |
| "learning_rate": 9.022058823529411e-07, |
| "loss": 0.0012, |
| "reward": 1.8408198356628418, |
| "reward_std": 0.02469984069466591, |
| "rewards/accuracy_reward": 0.8408196568489075, |
| "rewards/format_reward": 1.0, |
| "step": 133 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.5769271850586, |
| "epoch": 1.9705882352941178, |
| "grad_norm": 1.4534120952263125, |
| "kl": 0.0306396484375, |
| "learning_rate": 9.014705882352941e-07, |
| "loss": 0.0012, |
| "reward": 1.828810214996338, |
| "reward_std": 0.023025646805763245, |
| "rewards/accuracy_reward": 0.8288100361824036, |
| "rewards/format_reward": 1.0, |
| "step": 134 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.14904022216797, |
| "epoch": 1.9852941176470589, |
| "grad_norm": 2.390365626608761, |
| "kl": 0.031494140625, |
| "learning_rate": 9.007352941176471e-07, |
| "loss": 0.0013, |
| "reward": 1.8384226560592651, |
| "reward_std": 0.023323819041252136, |
| "rewards/accuracy_reward": 0.8384228348731995, |
| "rewards/format_reward": 1.0, |
| "step": 135 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.38813018798828, |
| "epoch": 2.0, |
| "grad_norm": 1.3167695891772675, |
| "kl": 0.033203125, |
| "learning_rate": 9e-07, |
| "loss": 0.0013, |
| "reward": 1.8518694639205933, |
| "reward_std": 0.021665828302502632, |
| "rewards/accuracy_reward": 0.8518694639205933, |
| "rewards/format_reward": 1.0, |
| "step": 136 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.4158706665039, |
| "epoch": 2.014705882352941, |
| "grad_norm": 1.1129043277785302, |
| "kl": 0.0296630859375, |
| "learning_rate": 8.992647058823529e-07, |
| "loss": 0.0012, |
| "reward": 1.8364787101745605, |
| "reward_std": 0.02524709887802601, |
| "rewards/accuracy_reward": 0.8364787697792053, |
| "rewards/format_reward": 1.0, |
| "step": 137 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.6189956665039, |
| "epoch": 2.0294117647058822, |
| "grad_norm": 11.77858401808986, |
| "kl": 0.0341796875, |
| "learning_rate": 8.985294117647059e-07, |
| "loss": 0.0014, |
| "reward": 1.8593671321868896, |
| "reward_std": 0.02559875324368477, |
| "rewards/accuracy_reward": 0.8605691194534302, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 138 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.89063262939453, |
| "epoch": 2.0441176470588234, |
| "grad_norm": 1.4515857339540483, |
| "kl": 0.035400390625, |
| "learning_rate": 8.977941176470588e-07, |
| "loss": 0.0014, |
| "reward": 1.8674319982528687, |
| "reward_std": 0.02322627604007721, |
| "rewards/accuracy_reward": 0.8674321174621582, |
| "rewards/format_reward": 1.0, |
| "step": 139 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.36058044433594, |
| "epoch": 2.0588235294117645, |
| "grad_norm": 1.7247871087577205, |
| "kl": 0.030517578125, |
| "learning_rate": 8.970588235294118e-07, |
| "loss": 0.0012, |
| "reward": 1.8335715532302856, |
| "reward_std": 0.028056330978870392, |
| "rewards/accuracy_reward": 0.8347733020782471, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 140 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.83894348144531, |
| "epoch": 2.073529411764706, |
| "grad_norm": 1.700568612592954, |
| "kl": 0.029296875, |
| "learning_rate": 8.963235294117648e-07, |
| "loss": 0.0012, |
| "reward": 1.841809868812561, |
| "reward_std": 0.02446519024670124, |
| "rewards/accuracy_reward": 0.8418098092079163, |
| "rewards/format_reward": 1.0, |
| "step": 141 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.85577392578125, |
| "epoch": 2.088235294117647, |
| "grad_norm": 1.9489963115328834, |
| "kl": 0.0296630859375, |
| "learning_rate": 8.955882352941176e-07, |
| "loss": 0.0012, |
| "reward": 1.8537096977233887, |
| "reward_std": 0.025433821603655815, |
| "rewards/accuracy_reward": 0.8537096381187439, |
| "rewards/format_reward": 1.0, |
| "step": 142 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 115.17788696289062, |
| "epoch": 2.1029411764705883, |
| "grad_norm": 1.930319438615514, |
| "kl": 0.0303955078125, |
| "learning_rate": 8.948529411764706e-07, |
| "loss": 0.0012, |
| "reward": 1.8577303886413574, |
| "reward_std": 0.025013966485857964, |
| "rewards/accuracy_reward": 0.858932375907898, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 143 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 115.77404022216797, |
| "epoch": 2.1176470588235294, |
| "grad_norm": 2.741052616431375, |
| "kl": 0.031005859375, |
| "learning_rate": 8.941176470588236e-07, |
| "loss": 0.0012, |
| "reward": 1.8680880069732666, |
| "reward_std": 0.025277523323893547, |
| "rewards/accuracy_reward": 0.8692898750305176, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 144 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 116.88581848144531, |
| "epoch": 2.1323529411764706, |
| "grad_norm": 2.1856417966833543, |
| "kl": 0.030029296875, |
| "learning_rate": 8.933823529411765e-07, |
| "loss": 0.0012, |
| "reward": 1.851528286933899, |
| "reward_std": 0.024104848504066467, |
| "rewards/accuracy_reward": 0.8515281081199646, |
| "rewards/format_reward": 1.0, |
| "step": 145 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 116.79808044433594, |
| "epoch": 2.1470588235294117, |
| "grad_norm": 1.7095764168354908, |
| "kl": 0.03076171875, |
| "learning_rate": 8.926470588235294e-07, |
| "loss": 0.0012, |
| "reward": 1.8471791744232178, |
| "reward_std": 0.028594160452485085, |
| "rewards/accuracy_reward": 0.8483811020851135, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 146 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 115.95673370361328, |
| "epoch": 2.161764705882353, |
| "grad_norm": 1.8204981844802233, |
| "kl": 0.03125, |
| "learning_rate": 8.919117647058824e-07, |
| "loss": 0.0013, |
| "reward": 1.8616198301315308, |
| "reward_std": 0.0273142047226429, |
| "rewards/accuracy_reward": 0.8628216981887817, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 147 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 115.73438262939453, |
| "epoch": 2.176470588235294, |
| "grad_norm": 2.6286179471239906, |
| "kl": 0.0299072265625, |
| "learning_rate": 8.911764705882353e-07, |
| "loss": 0.0012, |
| "reward": 1.864734172821045, |
| "reward_std": 0.027529988437891006, |
| "rewards/accuracy_reward": 0.8647341132164001, |
| "rewards/format_reward": 1.0, |
| "step": 148 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 116.94471740722656, |
| "epoch": 2.1911764705882355, |
| "grad_norm": 1.2094663657381695, |
| "kl": 0.033447265625, |
| "learning_rate": 8.904411764705882e-07, |
| "loss": 0.0013, |
| "reward": 1.8421967029571533, |
| "reward_std": 0.030433619394898415, |
| "rewards/accuracy_reward": 0.8446005582809448, |
| "rewards/format_reward": 0.9975962042808533, |
| "step": 149 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 116.39904022216797, |
| "epoch": 2.2058823529411766, |
| "grad_norm": 1.9663042039131189, |
| "kl": 0.032470703125, |
| "learning_rate": 8.89705882352941e-07, |
| "loss": 0.0013, |
| "reward": 1.8600176572799683, |
| "reward_std": 0.024169858545064926, |
| "rewards/accuracy_reward": 0.860017716884613, |
| "rewards/format_reward": 1.0, |
| "step": 150 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.9627456665039, |
| "epoch": 2.2205882352941178, |
| "grad_norm": 2.5776707309950115, |
| "kl": 0.02978515625, |
| "learning_rate": 8.88970588235294e-07, |
| "loss": 0.0012, |
| "reward": 1.871897578239441, |
| "reward_std": 0.023594854399561882, |
| "rewards/accuracy_reward": 0.8718973398208618, |
| "rewards/format_reward": 1.0, |
| "step": 151 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.98678588867188, |
| "epoch": 2.235294117647059, |
| "grad_norm": 1.5441369185409464, |
| "kl": 0.0306396484375, |
| "learning_rate": 8.88235294117647e-07, |
| "loss": 0.0012, |
| "reward": 1.8490618467330933, |
| "reward_std": 0.027410874143242836, |
| "rewards/accuracy_reward": 0.8502638339996338, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 152 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.43029022216797, |
| "epoch": 2.25, |
| "grad_norm": 1.8027917380235519, |
| "kl": 0.031494140625, |
| "learning_rate": 8.874999999999999e-07, |
| "loss": 0.0013, |
| "reward": 1.8541759252548218, |
| "reward_std": 0.024694515392184258, |
| "rewards/accuracy_reward": 0.8553777933120728, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 153 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.06491088867188, |
| "epoch": 2.264705882352941, |
| "grad_norm": 2.271542926940373, |
| "kl": 0.0308837890625, |
| "learning_rate": 8.867647058823528e-07, |
| "loss": 0.0012, |
| "reward": 1.8564496040344238, |
| "reward_std": 0.023326044902205467, |
| "rewards/accuracy_reward": 0.856449544429779, |
| "rewards/format_reward": 1.0, |
| "step": 154 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.58053588867188, |
| "epoch": 2.2794117647058822, |
| "grad_norm": 1.5375482020791835, |
| "kl": 0.032470703125, |
| "learning_rate": 8.860294117647058e-07, |
| "loss": 0.0013, |
| "reward": 1.8531345129013062, |
| "reward_std": 0.025648273527622223, |
| "rewards/accuracy_reward": 0.8543363809585571, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 155 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.21514892578125, |
| "epoch": 2.2941176470588234, |
| "grad_norm": 1.6774011174439099, |
| "kl": 0.041259765625, |
| "learning_rate": 8.852941176470587e-07, |
| "loss": 0.0017, |
| "reward": 1.8623310327529907, |
| "reward_std": 0.024891231209039688, |
| "rewards/accuracy_reward": 0.862330973148346, |
| "rewards/format_reward": 1.0, |
| "step": 156 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.63702392578125, |
| "epoch": 2.3088235294117645, |
| "grad_norm": 1.5805094517168279, |
| "kl": 0.03515625, |
| "learning_rate": 8.845588235294117e-07, |
| "loss": 0.0014, |
| "reward": 1.8760714530944824, |
| "reward_std": 0.028571614995598793, |
| "rewards/accuracy_reward": 0.8784750699996948, |
| "rewards/format_reward": 0.9975962042808533, |
| "step": 157 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.48798370361328, |
| "epoch": 2.323529411764706, |
| "grad_norm": 2.903562674226702, |
| "kl": 0.03466796875, |
| "learning_rate": 8.838235294117647e-07, |
| "loss": 0.0014, |
| "reward": 1.8514492511749268, |
| "reward_std": 0.023621495813131332, |
| "rewards/accuracy_reward": 0.8514493107795715, |
| "rewards/format_reward": 1.0, |
| "step": 158 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.39543914794922, |
| "epoch": 2.338235294117647, |
| "grad_norm": 2.257173366564063, |
| "kl": 0.035888671875, |
| "learning_rate": 8.830882352941175e-07, |
| "loss": 0.0014, |
| "reward": 1.8729958534240723, |
| "reward_std": 0.026356099173426628, |
| "rewards/accuracy_reward": 0.8729958534240723, |
| "rewards/format_reward": 1.0, |
| "step": 159 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.02764892578125, |
| "epoch": 2.3529411764705883, |
| "grad_norm": 1.6931345428419227, |
| "kl": 0.047607421875, |
| "learning_rate": 8.823529411764705e-07, |
| "loss": 0.0019, |
| "reward": 1.871155023574829, |
| "reward_std": 0.02590741030871868, |
| "rewards/accuracy_reward": 0.8723568916320801, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 160 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.74519348144531, |
| "epoch": 2.3676470588235294, |
| "grad_norm": 2.3920729679008232, |
| "kl": 0.03955078125, |
| "learning_rate": 8.816176470588235e-07, |
| "loss": 0.0016, |
| "reward": 1.870141863822937, |
| "reward_std": 0.025313733145594597, |
| "rewards/accuracy_reward": 0.870141863822937, |
| "rewards/format_reward": 1.0, |
| "step": 161 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 101.9831771850586, |
| "epoch": 2.3823529411764706, |
| "grad_norm": 1.7968109427756567, |
| "kl": 0.03955078125, |
| "learning_rate": 8.808823529411764e-07, |
| "loss": 0.0016, |
| "reward": 1.8542630672454834, |
| "reward_std": 0.025294849649071693, |
| "rewards/accuracy_reward": 0.8542630672454834, |
| "rewards/format_reward": 1.0, |
| "step": 162 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 100.81731414794922, |
| "epoch": 2.3970588235294117, |
| "grad_norm": 3.2616152503315536, |
| "kl": 0.040771484375, |
| "learning_rate": 8.801470588235293e-07, |
| "loss": 0.0016, |
| "reward": 1.8570067882537842, |
| "reward_std": 0.02526533231139183, |
| "rewards/accuracy_reward": 0.8570066094398499, |
| "rewards/format_reward": 1.0, |
| "step": 163 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 100.97716522216797, |
| "epoch": 2.411764705882353, |
| "grad_norm": 1.8975512236370329, |
| "kl": 0.03955078125, |
| "learning_rate": 8.794117647058823e-07, |
| "loss": 0.0016, |
| "reward": 1.8464734554290771, |
| "reward_std": 0.02490406110882759, |
| "rewards/accuracy_reward": 0.8464734554290771, |
| "rewards/format_reward": 1.0, |
| "step": 164 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 101.0769271850586, |
| "epoch": 2.426470588235294, |
| "grad_norm": 3.1230913633145096, |
| "kl": 0.04931640625, |
| "learning_rate": 8.786764705882352e-07, |
| "loss": 0.002, |
| "reward": 1.8685623407363892, |
| "reward_std": 0.024073276668787003, |
| "rewards/accuracy_reward": 0.8685624599456787, |
| "rewards/format_reward": 1.0, |
| "step": 165 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 101.92668914794922, |
| "epoch": 2.4411764705882355, |
| "grad_norm": 2.501654602692275, |
| "kl": 0.04296875, |
| "learning_rate": 8.779411764705882e-07, |
| "loss": 0.0017, |
| "reward": 1.8564651012420654, |
| "reward_std": 0.024265745654702187, |
| "rewards/accuracy_reward": 0.8564649820327759, |
| "rewards/format_reward": 1.0, |
| "step": 166 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 101.86538696289062, |
| "epoch": 2.4558823529411766, |
| "grad_norm": 6.016419703798391, |
| "kl": 0.0439453125, |
| "learning_rate": 8.772058823529412e-07, |
| "loss": 0.0018, |
| "reward": 1.8798558712005615, |
| "reward_std": 0.021158261224627495, |
| "rewards/accuracy_reward": 0.8798558712005615, |
| "rewards/format_reward": 1.0, |
| "step": 167 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.84014892578125, |
| "epoch": 2.4705882352941178, |
| "grad_norm": 1.9371757598853314, |
| "kl": 0.04052734375, |
| "learning_rate": 8.76470588235294e-07, |
| "loss": 0.0016, |
| "reward": 1.882270336151123, |
| "reward_std": 0.023024968802928925, |
| "rewards/accuracy_reward": 0.882270336151123, |
| "rewards/format_reward": 1.0, |
| "step": 168 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.30529022216797, |
| "epoch": 2.485294117647059, |
| "grad_norm": 1.4411287692419261, |
| "kl": 0.036376953125, |
| "learning_rate": 8.75735294117647e-07, |
| "loss": 0.0015, |
| "reward": 1.862653374671936, |
| "reward_std": 0.021055346354842186, |
| "rewards/accuracy_reward": 0.8626533150672913, |
| "rewards/format_reward": 1.0, |
| "step": 169 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.5877456665039, |
| "epoch": 2.5, |
| "grad_norm": 1.4740075609659171, |
| "kl": 0.03857421875, |
| "learning_rate": 8.75e-07, |
| "loss": 0.0015, |
| "reward": 1.8584967851638794, |
| "reward_std": 0.02370167151093483, |
| "rewards/accuracy_reward": 0.8584967255592346, |
| "rewards/format_reward": 1.0, |
| "step": 170 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.28245544433594, |
| "epoch": 2.514705882352941, |
| "grad_norm": 1.3370741346003325, |
| "kl": 0.03662109375, |
| "learning_rate": 8.742647058823529e-07, |
| "loss": 0.0015, |
| "reward": 1.8700354099273682, |
| "reward_std": 0.023715250194072723, |
| "rewards/accuracy_reward": 0.8700354099273682, |
| "rewards/format_reward": 1.0, |
| "step": 171 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.4050521850586, |
| "epoch": 2.5294117647058822, |
| "grad_norm": 2.6769462130534345, |
| "kl": 0.03369140625, |
| "learning_rate": 8.735294117647058e-07, |
| "loss": 0.0013, |
| "reward": 1.8732657432556152, |
| "reward_std": 0.01899472437798977, |
| "rewards/accuracy_reward": 0.8732655644416809, |
| "rewards/format_reward": 1.0, |
| "step": 172 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.87741088867188, |
| "epoch": 2.5441176470588234, |
| "grad_norm": 2.5769454874734152, |
| "kl": 0.0322265625, |
| "learning_rate": 8.727941176470588e-07, |
| "loss": 0.0013, |
| "reward": 1.8616483211517334, |
| "reward_std": 0.02438032068312168, |
| "rewards/accuracy_reward": 0.8616481423377991, |
| "rewards/format_reward": 1.0, |
| "step": 173 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 102.12259674072266, |
| "epoch": 2.5588235294117645, |
| "grad_norm": 1.8562667369690105, |
| "kl": 0.033935546875, |
| "learning_rate": 8.720588235294117e-07, |
| "loss": 0.0013, |
| "reward": 1.855645775794983, |
| "reward_std": 0.02895743027329445, |
| "rewards/accuracy_reward": 0.8580495715141296, |
| "rewards/format_reward": 0.9975962042808533, |
| "step": 174 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.21154022216797, |
| "epoch": 2.5735294117647056, |
| "grad_norm": 2.558018085056357, |
| "kl": 0.034423828125, |
| "learning_rate": 8.713235294117647e-07, |
| "loss": 0.0014, |
| "reward": 1.884694218635559, |
| "reward_std": 0.026144152507185936, |
| "rewards/accuracy_reward": 0.8858961462974548, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 175 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.8894271850586, |
| "epoch": 2.588235294117647, |
| "grad_norm": 4.093759859461043, |
| "kl": 0.0322265625, |
| "learning_rate": 8.705882352941177e-07, |
| "loss": 0.0013, |
| "reward": 1.859119176864624, |
| "reward_std": 0.023388409987092018, |
| "rewards/accuracy_reward": 0.859119176864624, |
| "rewards/format_reward": 1.0, |
| "step": 176 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.76923370361328, |
| "epoch": 2.6029411764705883, |
| "grad_norm": 2.565372217350874, |
| "kl": 0.039794921875, |
| "learning_rate": 8.698529411764705e-07, |
| "loss": 0.0016, |
| "reward": 1.8814976215362549, |
| "reward_std": 0.024835962802171707, |
| "rewards/accuracy_reward": 0.8826994895935059, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 177 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.4158706665039, |
| "epoch": 2.6176470588235294, |
| "grad_norm": 1.546586759316415, |
| "kl": 0.03662109375, |
| "learning_rate": 8.691176470588235e-07, |
| "loss": 0.0015, |
| "reward": 1.8796207904815674, |
| "reward_std": 0.022139329463243484, |
| "rewards/accuracy_reward": 0.8796206712722778, |
| "rewards/format_reward": 1.0, |
| "step": 178 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.92668914794922, |
| "epoch": 2.6323529411764706, |
| "grad_norm": 1.5303977681102032, |
| "kl": 0.03369140625, |
| "learning_rate": 8.683823529411765e-07, |
| "loss": 0.0014, |
| "reward": 1.8701624870300293, |
| "reward_std": 0.025447282940149307, |
| "rewards/accuracy_reward": 0.8701622486114502, |
| "rewards/format_reward": 1.0, |
| "step": 179 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.36659240722656, |
| "epoch": 2.6470588235294117, |
| "grad_norm": 1.518878529368709, |
| "kl": 0.033203125, |
| "learning_rate": 8.676470588235294e-07, |
| "loss": 0.0013, |
| "reward": 1.8654332160949707, |
| "reward_std": 0.026392469182610512, |
| "rewards/accuracy_reward": 0.8654330372810364, |
| "rewards/format_reward": 1.0, |
| "step": 180 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.12620544433594, |
| "epoch": 2.661764705882353, |
| "grad_norm": 1.403191410442849, |
| "kl": 0.03173828125, |
| "learning_rate": 8.669117647058823e-07, |
| "loss": 0.0013, |
| "reward": 1.86957585811615, |
| "reward_std": 0.025720389559864998, |
| "rewards/accuracy_reward": 0.8695757985115051, |
| "rewards/format_reward": 1.0, |
| "step": 181 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.7283706665039, |
| "epoch": 2.6764705882352944, |
| "grad_norm": 2.022045986077511, |
| "kl": 0.031494140625, |
| "learning_rate": 8.661764705882353e-07, |
| "loss": 0.0013, |
| "reward": 1.8482249975204468, |
| "reward_std": 0.03184087574481964, |
| "rewards/accuracy_reward": 0.8494267463684082, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 182 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.31491088867188, |
| "epoch": 2.6911764705882355, |
| "grad_norm": 1.4526871175791842, |
| "kl": 0.043701171875, |
| "learning_rate": 8.654411764705882e-07, |
| "loss": 0.0017, |
| "reward": 1.8396512269973755, |
| "reward_std": 0.02649850584566593, |
| "rewards/accuracy_reward": 0.8396511673927307, |
| "rewards/format_reward": 1.0, |
| "step": 183 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.32933044433594, |
| "epoch": 2.7058823529411766, |
| "grad_norm": 1.962569471145186, |
| "kl": 0.034423828125, |
| "learning_rate": 8.647058823529412e-07, |
| "loss": 0.0014, |
| "reward": 1.864058017730713, |
| "reward_std": 0.026860656216740608, |
| "rewards/accuracy_reward": 0.8640578389167786, |
| "rewards/format_reward": 1.0, |
| "step": 184 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.7331771850586, |
| "epoch": 2.7205882352941178, |
| "grad_norm": 2.2484467422992944, |
| "kl": 0.03369140625, |
| "learning_rate": 8.639705882352941e-07, |
| "loss": 0.0013, |
| "reward": 1.8754035234451294, |
| "reward_std": 0.024205412715673447, |
| "rewards/accuracy_reward": 0.8754035234451294, |
| "rewards/format_reward": 1.0, |
| "step": 185 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.37860870361328, |
| "epoch": 2.735294117647059, |
| "grad_norm": 1.997975820505161, |
| "kl": 0.033935546875, |
| "learning_rate": 8.63235294117647e-07, |
| "loss": 0.0014, |
| "reward": 1.860775351524353, |
| "reward_std": 0.02340296283364296, |
| "rewards/accuracy_reward": 0.8607752919197083, |
| "rewards/format_reward": 1.0, |
| "step": 186 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 114.65866088867188, |
| "epoch": 2.75, |
| "grad_norm": 1.7044143261510771, |
| "kl": 0.0341796875, |
| "learning_rate": 8.625e-07, |
| "loss": 0.0014, |
| "reward": 1.8710038661956787, |
| "reward_std": 0.026455307379364967, |
| "rewards/accuracy_reward": 0.8722056746482849, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 187 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.46514892578125, |
| "epoch": 2.764705882352941, |
| "grad_norm": 1.8000725865597647, |
| "kl": 0.038330078125, |
| "learning_rate": 8.61764705882353e-07, |
| "loss": 0.0015, |
| "reward": 1.8618617057800293, |
| "reward_std": 0.03181543946266174, |
| "rewards/accuracy_reward": 0.8642653822898865, |
| "rewards/format_reward": 0.9975962042808533, |
| "step": 188 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.53125762939453, |
| "epoch": 2.7794117647058822, |
| "grad_norm": 2.339206486710046, |
| "kl": 0.0322265625, |
| "learning_rate": 8.610294117647059e-07, |
| "loss": 0.0013, |
| "reward": 1.875985860824585, |
| "reward_std": 0.02267822064459324, |
| "rewards/accuracy_reward": 0.875985860824585, |
| "rewards/format_reward": 1.0, |
| "step": 189 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.21634674072266, |
| "epoch": 2.7941176470588234, |
| "grad_norm": 1.4965339815672982, |
| "kl": 0.033447265625, |
| "learning_rate": 8.602941176470588e-07, |
| "loss": 0.0013, |
| "reward": 1.8847665786743164, |
| "reward_std": 0.023552460595965385, |
| "rewards/accuracy_reward": 0.8847665190696716, |
| "rewards/format_reward": 1.0, |
| "step": 190 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.6238021850586, |
| "epoch": 2.8088235294117645, |
| "grad_norm": 2.3107985557542774, |
| "kl": 0.03369140625, |
| "learning_rate": 8.595588235294118e-07, |
| "loss": 0.0014, |
| "reward": 1.8373372554779053, |
| "reward_std": 0.02436283603310585, |
| "rewards/accuracy_reward": 0.8373372554779053, |
| "rewards/format_reward": 1.0, |
| "step": 191 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.30409240722656, |
| "epoch": 2.8235294117647056, |
| "grad_norm": 1.92475811318894, |
| "kl": 0.0361328125, |
| "learning_rate": 8.588235294117646e-07, |
| "loss": 0.0014, |
| "reward": 1.8702090978622437, |
| "reward_std": 0.026847295463085175, |
| "rewards/accuracy_reward": 0.8714108467102051, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 192 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.48197174072266, |
| "epoch": 2.838235294117647, |
| "grad_norm": 2.0449651951501866, |
| "kl": 0.034423828125, |
| "learning_rate": 8.580882352941176e-07, |
| "loss": 0.0014, |
| "reward": 1.843615174293518, |
| "reward_std": 0.02285837009549141, |
| "rewards/accuracy_reward": 0.8436151742935181, |
| "rewards/format_reward": 1.0, |
| "step": 193 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.01202392578125, |
| "epoch": 2.8529411764705883, |
| "grad_norm": 1.2736084553332587, |
| "kl": 0.036865234375, |
| "learning_rate": 8.573529411764705e-07, |
| "loss": 0.0015, |
| "reward": 1.8695170879364014, |
| "reward_std": 0.022367505356669426, |
| "rewards/accuracy_reward": 0.8695171475410461, |
| "rewards/format_reward": 1.0, |
| "step": 194 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.91827392578125, |
| "epoch": 2.8676470588235294, |
| "grad_norm": 1.9579889152211967, |
| "kl": 0.047119140625, |
| "learning_rate": 8.566176470588234e-07, |
| "loss": 0.0019, |
| "reward": 1.8573535680770874, |
| "reward_std": 0.026318097487092018, |
| "rewards/accuracy_reward": 0.8585554361343384, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 195 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.89543914794922, |
| "epoch": 2.8823529411764706, |
| "grad_norm": 1.6397435637270048, |
| "kl": 0.03173828125, |
| "learning_rate": 8.558823529411764e-07, |
| "loss": 0.0013, |
| "reward": 1.8664311170578003, |
| "reward_std": 0.02275273948907852, |
| "rewards/accuracy_reward": 0.8664309978485107, |
| "rewards/format_reward": 1.0, |
| "step": 196 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.19351196289062, |
| "epoch": 2.8970588235294117, |
| "grad_norm": 1.3453321422064775, |
| "kl": 0.03271484375, |
| "learning_rate": 8.551470588235294e-07, |
| "loss": 0.0013, |
| "reward": 1.8688963651657104, |
| "reward_std": 0.02163490653038025, |
| "rewards/accuracy_reward": 0.8688963651657104, |
| "rewards/format_reward": 1.0, |
| "step": 197 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.34495544433594, |
| "epoch": 2.911764705882353, |
| "grad_norm": 1.6994019668289626, |
| "kl": 0.034423828125, |
| "learning_rate": 8.544117647058822e-07, |
| "loss": 0.0014, |
| "reward": 1.8594874143600464, |
| "reward_std": 0.025675183162093163, |
| "rewards/accuracy_reward": 0.8606892824172974, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 198 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.97956848144531, |
| "epoch": 2.9264705882352944, |
| "grad_norm": 1.443393412750695, |
| "kl": 0.033935546875, |
| "learning_rate": 8.536764705882352e-07, |
| "loss": 0.0014, |
| "reward": 1.8734104633331299, |
| "reward_std": 0.02485453523695469, |
| "rewards/accuracy_reward": 0.8734104633331299, |
| "rewards/format_reward": 1.0, |
| "step": 199 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.54447174072266, |
| "epoch": 2.9411764705882355, |
| "grad_norm": 1.3185192214107238, |
| "kl": 0.0322265625, |
| "learning_rate": 8.529411764705882e-07, |
| "loss": 0.0013, |
| "reward": 1.8638544082641602, |
| "reward_std": 0.024562494829297066, |
| "rewards/accuracy_reward": 0.8650562167167664, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 200 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.21995544433594, |
| "epoch": 2.9558823529411766, |
| "grad_norm": 1.9943222079885201, |
| "kl": 0.032470703125, |
| "learning_rate": 8.522058823529411e-07, |
| "loss": 0.0013, |
| "reward": 1.8713494539260864, |
| "reward_std": 0.027820177376270294, |
| "rewards/accuracy_reward": 0.8725512623786926, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 201 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.25000762939453, |
| "epoch": 2.9705882352941178, |
| "grad_norm": 1.3647685703194712, |
| "kl": 0.033203125, |
| "learning_rate": 8.514705882352941e-07, |
| "loss": 0.0013, |
| "reward": 1.8598443269729614, |
| "reward_std": 0.025986719876527786, |
| "rewards/accuracy_reward": 0.8598442673683167, |
| "rewards/format_reward": 1.0, |
| "step": 202 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.90625762939453, |
| "epoch": 2.985294117647059, |
| "grad_norm": 1.6126487460912005, |
| "kl": 0.037841796875, |
| "learning_rate": 8.50735294117647e-07, |
| "loss": 0.0015, |
| "reward": 1.8783414363861084, |
| "reward_std": 0.023664260283112526, |
| "rewards/accuracy_reward": 0.8783413171768188, |
| "rewards/format_reward": 1.0, |
| "step": 203 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.15220642089844, |
| "epoch": 3.0, |
| "grad_norm": 1.780161633176048, |
| "kl": 0.034423828125, |
| "learning_rate": 8.499999999999999e-07, |
| "loss": 0.0014, |
| "reward": 1.8632652759552002, |
| "reward_std": 0.02758597768843174, |
| "rewards/accuracy_reward": 0.8647872805595398, |
| "rewards/format_reward": 0.9984779357910156, |
| "step": 204 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.47956848144531, |
| "epoch": 3.014705882352941, |
| "grad_norm": 1.9805220919990834, |
| "kl": 0.03271484375, |
| "learning_rate": 8.492647058823529e-07, |
| "loss": 0.0013, |
| "reward": 1.8707077503204346, |
| "reward_std": 0.023023683577775955, |
| "rewards/accuracy_reward": 0.8707077503204346, |
| "rewards/format_reward": 1.0, |
| "step": 205 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 114.53366088867188, |
| "epoch": 3.0294117647058822, |
| "grad_norm": 1.759835112928017, |
| "kl": 0.03271484375, |
| "learning_rate": 8.485294117647059e-07, |
| "loss": 0.0013, |
| "reward": 1.8758447170257568, |
| "reward_std": 0.02334214560687542, |
| "rewards/accuracy_reward": 0.8758447170257568, |
| "rewards/format_reward": 1.0, |
| "step": 206 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 115.1346206665039, |
| "epoch": 3.0441176470588234, |
| "grad_norm": 1.9299225316519943, |
| "kl": 0.032958984375, |
| "learning_rate": 8.477941176470587e-07, |
| "loss": 0.0013, |
| "reward": 1.8904192447662354, |
| "reward_std": 0.021329201757907867, |
| "rewards/accuracy_reward": 0.8904191255569458, |
| "rewards/format_reward": 1.0, |
| "step": 207 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 114.8221206665039, |
| "epoch": 3.0588235294117645, |
| "grad_norm": 1.5162706953943, |
| "kl": 0.03759765625, |
| "learning_rate": 8.470588235294117e-07, |
| "loss": 0.0015, |
| "reward": 1.8666112422943115, |
| "reward_std": 0.02535596303641796, |
| "rewards/accuracy_reward": 0.8666113615036011, |
| "rewards/format_reward": 1.0, |
| "step": 208 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 115.12019348144531, |
| "epoch": 3.073529411764706, |
| "grad_norm": 3.4426286843801193, |
| "kl": 0.08642578125, |
| "learning_rate": 8.463235294117647e-07, |
| "loss": 0.0035, |
| "reward": 1.8811273574829102, |
| "reward_std": 0.022378716617822647, |
| "rewards/accuracy_reward": 0.8811271786689758, |
| "rewards/format_reward": 1.0, |
| "step": 209 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 117.17909240722656, |
| "epoch": 3.088235294117647, |
| "grad_norm": 1.7571727846632814, |
| "kl": 0.0380859375, |
| "learning_rate": 8.455882352941176e-07, |
| "loss": 0.0015, |
| "reward": 1.8708646297454834, |
| "reward_std": 0.027770236134529114, |
| "rewards/accuracy_reward": 0.8732684254646301, |
| "rewards/format_reward": 0.9975962042808533, |
| "step": 210 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 117.16947174072266, |
| "epoch": 3.1029411764705883, |
| "grad_norm": 1.336840943007517, |
| "kl": 0.041259765625, |
| "learning_rate": 8.448529411764706e-07, |
| "loss": 0.0016, |
| "reward": 1.8811522722244263, |
| "reward_std": 0.02304122783243656, |
| "rewards/accuracy_reward": 0.8811522722244263, |
| "rewards/format_reward": 1.0, |
| "step": 211 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 116.03846740722656, |
| "epoch": 3.1176470588235294, |
| "grad_norm": 1.8533547410757114, |
| "kl": 0.0400390625, |
| "learning_rate": 8.441176470588235e-07, |
| "loss": 0.0016, |
| "reward": 1.8757656812667847, |
| "reward_std": 0.027794932946562767, |
| "rewards/accuracy_reward": 0.8769674897193909, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 212 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 117.97356414794922, |
| "epoch": 3.1323529411764706, |
| "grad_norm": 1.2600737914736126, |
| "kl": 0.0380859375, |
| "learning_rate": 8.433823529411764e-07, |
| "loss": 0.0015, |
| "reward": 1.8561041355133057, |
| "reward_std": 0.031202280893921852, |
| "rewards/accuracy_reward": 0.8585077524185181, |
| "rewards/format_reward": 0.9975962042808533, |
| "step": 213 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 117.35938262939453, |
| "epoch": 3.1470588235294117, |
| "grad_norm": 1.7979464745878548, |
| "kl": 0.043701171875, |
| "learning_rate": 8.426470588235294e-07, |
| "loss": 0.0017, |
| "reward": 1.8608158826828003, |
| "reward_std": 0.026735760271549225, |
| "rewards/accuracy_reward": 0.8608157634735107, |
| "rewards/format_reward": 1.0, |
| "step": 214 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 115.00241088867188, |
| "epoch": 3.161764705882353, |
| "grad_norm": 5.398535880427292, |
| "kl": 0.05322265625, |
| "learning_rate": 8.419117647058824e-07, |
| "loss": 0.0021, |
| "reward": 1.8839178085327148, |
| "reward_std": 0.02646607905626297, |
| "rewards/accuracy_reward": 0.8851196765899658, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 215 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 116.87620544433594, |
| "epoch": 3.176470588235294, |
| "grad_norm": 1.6689071059712295, |
| "kl": 0.041748046875, |
| "learning_rate": 8.411764705882352e-07, |
| "loss": 0.0017, |
| "reward": 1.8544951677322388, |
| "reward_std": 0.02808697521686554, |
| "rewards/accuracy_reward": 0.854495108127594, |
| "rewards/format_reward": 1.0, |
| "step": 216 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.43029022216797, |
| "epoch": 3.1911764705882355, |
| "grad_norm": 1.6258182876226766, |
| "kl": 0.04296875, |
| "learning_rate": 8.404411764705882e-07, |
| "loss": 0.0017, |
| "reward": 1.8663568496704102, |
| "reward_std": 0.029328130185604095, |
| "rewards/accuracy_reward": 0.8675586581230164, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 217 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 114.61418914794922, |
| "epoch": 3.2058823529411766, |
| "grad_norm": 1.4035611828729841, |
| "kl": 0.040771484375, |
| "learning_rate": 8.397058823529412e-07, |
| "loss": 0.0016, |
| "reward": 1.8575928211212158, |
| "reward_std": 0.02494001016020775, |
| "rewards/accuracy_reward": 0.8575928211212158, |
| "rewards/format_reward": 1.0, |
| "step": 218 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.87620544433594, |
| "epoch": 3.2205882352941178, |
| "grad_norm": 1.5589742726291336, |
| "kl": 0.0419921875, |
| "learning_rate": 8.389705882352941e-07, |
| "loss": 0.0017, |
| "reward": 1.8885266780853271, |
| "reward_std": 0.023614982143044472, |
| "rewards/accuracy_reward": 0.8897286653518677, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 219 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.81851196289062, |
| "epoch": 3.235294117647059, |
| "grad_norm": 1.385282340462598, |
| "kl": 0.041259765625, |
| "learning_rate": 8.38235294117647e-07, |
| "loss": 0.0017, |
| "reward": 1.855526328086853, |
| "reward_std": 0.030946535989642143, |
| "rewards/accuracy_reward": 0.8567281365394592, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 220 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.7439956665039, |
| "epoch": 3.25, |
| "grad_norm": 2.962059838231098, |
| "kl": 0.044189453125, |
| "learning_rate": 8.375e-07, |
| "loss": 0.0018, |
| "reward": 1.855432152748108, |
| "reward_std": 0.02485704980790615, |
| "rewards/accuracy_reward": 0.8554319739341736, |
| "rewards/format_reward": 1.0, |
| "step": 221 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.8533706665039, |
| "epoch": 3.264705882352941, |
| "grad_norm": 1.3569137376665106, |
| "kl": 0.04296875, |
| "learning_rate": 8.367647058823529e-07, |
| "loss": 0.0017, |
| "reward": 1.8471739292144775, |
| "reward_std": 0.028255922719836235, |
| "rewards/accuracy_reward": 0.8483757972717285, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 222 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 102.22956848144531, |
| "epoch": 3.2794117647058822, |
| "grad_norm": 1.5185134752025757, |
| "kl": 0.04345703125, |
| "learning_rate": 8.360294117647059e-07, |
| "loss": 0.0017, |
| "reward": 1.8899589776992798, |
| "reward_std": 0.02284637838602066, |
| "rewards/accuracy_reward": 0.889958918094635, |
| "rewards/format_reward": 1.0, |
| "step": 223 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 99.96394348144531, |
| "epoch": 3.2941176470588234, |
| "grad_norm": 2.4678802175191117, |
| "kl": 0.0458984375, |
| "learning_rate": 8.352941176470589e-07, |
| "loss": 0.0018, |
| "reward": 1.8883994817733765, |
| "reward_std": 0.021819431334733963, |
| "rewards/accuracy_reward": 0.8883993625640869, |
| "rewards/format_reward": 1.0, |
| "step": 224 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 97.47356414794922, |
| "epoch": 3.3088235294117645, |
| "grad_norm": 1.7983286178946878, |
| "kl": 0.04736328125, |
| "learning_rate": 8.345588235294117e-07, |
| "loss": 0.0019, |
| "reward": 1.8640559911727905, |
| "reward_std": 0.023397305980324745, |
| "rewards/accuracy_reward": 0.8640559315681458, |
| "rewards/format_reward": 1.0, |
| "step": 225 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 97.4675521850586, |
| "epoch": 3.323529411764706, |
| "grad_norm": 1.8613936498025156, |
| "kl": 0.04736328125, |
| "learning_rate": 8.338235294117647e-07, |
| "loss": 0.0019, |
| "reward": 1.8887497186660767, |
| "reward_std": 0.022856805473566055, |
| "rewards/accuracy_reward": 0.8887495994567871, |
| "rewards/format_reward": 1.0, |
| "step": 226 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 97.93029022216797, |
| "epoch": 3.338235294117647, |
| "grad_norm": 1.5411506435438795, |
| "kl": 0.045654296875, |
| "learning_rate": 8.330882352941177e-07, |
| "loss": 0.0018, |
| "reward": 1.8804612159729004, |
| "reward_std": 0.02065611444413662, |
| "rewards/accuracy_reward": 0.8804610371589661, |
| "rewards/format_reward": 1.0, |
| "step": 227 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 97.00601196289062, |
| "epoch": 3.3529411764705883, |
| "grad_norm": 2.851058373606194, |
| "kl": 0.048583984375, |
| "learning_rate": 8.323529411764706e-07, |
| "loss": 0.0019, |
| "reward": 1.8770747184753418, |
| "reward_std": 0.0234384723007679, |
| "rewards/accuracy_reward": 0.8770747184753418, |
| "rewards/format_reward": 1.0, |
| "step": 228 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 97.92909240722656, |
| "epoch": 3.3676470588235294, |
| "grad_norm": 1.437225684861633, |
| "kl": 0.04345703125, |
| "learning_rate": 8.316176470588235e-07, |
| "loss": 0.0017, |
| "reward": 1.8713281154632568, |
| "reward_std": 0.02329498529434204, |
| "rewards/accuracy_reward": 0.8713279962539673, |
| "rewards/format_reward": 1.0, |
| "step": 229 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 97.06610870361328, |
| "epoch": 3.3823529411764706, |
| "grad_norm": 1.9896299970328106, |
| "kl": 0.04296875, |
| "learning_rate": 8.308823529411765e-07, |
| "loss": 0.0017, |
| "reward": 1.878764271736145, |
| "reward_std": 0.025371139869093895, |
| "rewards/accuracy_reward": 0.8787642121315002, |
| "rewards/format_reward": 1.0, |
| "step": 230 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 98.38341522216797, |
| "epoch": 3.3970588235294117, |
| "grad_norm": 2.234055405089306, |
| "kl": 0.04345703125, |
| "learning_rate": 8.301470588235294e-07, |
| "loss": 0.0017, |
| "reward": 1.898535132408142, |
| "reward_std": 0.02297838404774666, |
| "rewards/accuracy_reward": 0.8985350131988525, |
| "rewards/format_reward": 1.0, |
| "step": 231 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 98.45072174072266, |
| "epoch": 3.411764705882353, |
| "grad_norm": 1.776208061156291, |
| "kl": 0.04052734375, |
| "learning_rate": 8.294117647058824e-07, |
| "loss": 0.0016, |
| "reward": 1.873674750328064, |
| "reward_std": 0.022656356915831566, |
| "rewards/accuracy_reward": 0.8736746907234192, |
| "rewards/format_reward": 1.0, |
| "step": 232 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 102.99038696289062, |
| "epoch": 3.426470588235294, |
| "grad_norm": 1.5187065914939428, |
| "kl": 0.0390625, |
| "learning_rate": 8.286764705882354e-07, |
| "loss": 0.0016, |
| "reward": 1.8770731687545776, |
| "reward_std": 0.02269612066447735, |
| "rewards/accuracy_reward": 0.8770731091499329, |
| "rewards/format_reward": 1.0, |
| "step": 233 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.18389892578125, |
| "epoch": 3.4411764705882355, |
| "grad_norm": 1.6681947750734276, |
| "kl": 0.040771484375, |
| "learning_rate": 8.279411764705881e-07, |
| "loss": 0.0016, |
| "reward": 1.8845463991165161, |
| "reward_std": 0.02028636820614338, |
| "rewards/accuracy_reward": 0.8845462203025818, |
| "rewards/format_reward": 1.0, |
| "step": 234 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.5613021850586, |
| "epoch": 3.4558823529411766, |
| "grad_norm": 1.4760305455453029, |
| "kl": 0.03759765625, |
| "learning_rate": 8.272058823529411e-07, |
| "loss": 0.0015, |
| "reward": 1.890356183052063, |
| "reward_std": 0.02070949226617813, |
| "rewards/accuracy_reward": 0.8903560042381287, |
| "rewards/format_reward": 1.0, |
| "step": 235 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.21394348144531, |
| "epoch": 3.4705882352941178, |
| "grad_norm": 16.169977810311337, |
| "kl": 0.03759765625, |
| "learning_rate": 8.264705882352941e-07, |
| "loss": 0.0015, |
| "reward": 1.8766230344772339, |
| "reward_std": 0.02773961052298546, |
| "rewards/accuracy_reward": 0.8778248429298401, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 236 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 114.24759674072266, |
| "epoch": 3.485294117647059, |
| "grad_norm": 6.784504465196233, |
| "kl": 0.046630859375, |
| "learning_rate": 8.25735294117647e-07, |
| "loss": 0.0019, |
| "reward": 1.8772554397583008, |
| "reward_std": 0.022827137261629105, |
| "rewards/accuracy_reward": 0.8772553205490112, |
| "rewards/format_reward": 1.0, |
| "step": 237 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 115.28485870361328, |
| "epoch": 3.5, |
| "grad_norm": 1.5048421325605899, |
| "kl": 0.040283203125, |
| "learning_rate": 8.249999999999999e-07, |
| "loss": 0.0016, |
| "reward": 1.8868188858032227, |
| "reward_std": 0.022883420810103416, |
| "rewards/accuracy_reward": 0.8868188261985779, |
| "rewards/format_reward": 1.0, |
| "step": 238 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 115.67428588867188, |
| "epoch": 3.514705882352941, |
| "grad_norm": 1.9797082947474498, |
| "kl": 0.041259765625, |
| "learning_rate": 8.242647058823528e-07, |
| "loss": 0.0017, |
| "reward": 1.8824166059494019, |
| "reward_std": 0.025011856108903885, |
| "rewards/accuracy_reward": 0.8824164867401123, |
| "rewards/format_reward": 1.0, |
| "step": 239 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 115.68991088867188, |
| "epoch": 3.5294117647058822, |
| "grad_norm": 1.3808876653722937, |
| "kl": 0.038818359375, |
| "learning_rate": 8.235294117647058e-07, |
| "loss": 0.0016, |
| "reward": 1.8804829120635986, |
| "reward_std": 0.019605407491326332, |
| "rewards/accuracy_reward": 0.8804829716682434, |
| "rewards/format_reward": 1.0, |
| "step": 240 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 115.4675521850586, |
| "epoch": 3.5441176470588234, |
| "grad_norm": 2.2656927922991352, |
| "kl": 0.03955078125, |
| "learning_rate": 8.227941176470588e-07, |
| "loss": 0.0016, |
| "reward": 1.8745205402374268, |
| "reward_std": 0.019595161080360413, |
| "rewards/accuracy_reward": 0.8745204210281372, |
| "rewards/format_reward": 1.0, |
| "step": 241 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 117.08293914794922, |
| "epoch": 3.5588235294117645, |
| "grad_norm": 2.2969538457560796, |
| "kl": 0.040771484375, |
| "learning_rate": 8.220588235294116e-07, |
| "loss": 0.0016, |
| "reward": 1.8878697156906128, |
| "reward_std": 0.02017536200582981, |
| "rewards/accuracy_reward": 0.8878695964813232, |
| "rewards/format_reward": 1.0, |
| "step": 242 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 116.17788696289062, |
| "epoch": 3.5735294117647056, |
| "grad_norm": 1.3250386411482078, |
| "kl": 0.036865234375, |
| "learning_rate": 8.213235294117646e-07, |
| "loss": 0.0015, |
| "reward": 1.882023572921753, |
| "reward_std": 0.019443001598119736, |
| "rewards/accuracy_reward": 0.8820234537124634, |
| "rewards/format_reward": 1.0, |
| "step": 243 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 116.6863021850586, |
| "epoch": 3.588235294117647, |
| "grad_norm": 1.666209048217596, |
| "kl": 0.041259765625, |
| "learning_rate": 8.205882352941176e-07, |
| "loss": 0.0016, |
| "reward": 1.8930561542510986, |
| "reward_std": 0.023166730999946594, |
| "rewards/accuracy_reward": 0.8942580223083496, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 244 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.91947174072266, |
| "epoch": 3.6029411764705883, |
| "grad_norm": 8.010516964998228, |
| "kl": 0.0419921875, |
| "learning_rate": 8.198529411764705e-07, |
| "loss": 0.0017, |
| "reward": 1.8905775547027588, |
| "reward_std": 0.01961654983460903, |
| "rewards/accuracy_reward": 0.890577495098114, |
| "rewards/format_reward": 1.0, |
| "step": 245 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.99759674072266, |
| "epoch": 3.6176470588235294, |
| "grad_norm": 2.0494021896720236, |
| "kl": 0.07470703125, |
| "learning_rate": 8.191176470588235e-07, |
| "loss": 0.003, |
| "reward": 1.8533637523651123, |
| "reward_std": 0.022241346538066864, |
| "rewards/accuracy_reward": 0.8533636331558228, |
| "rewards/format_reward": 1.0, |
| "step": 246 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 116.3377456665039, |
| "epoch": 3.6323529411764706, |
| "grad_norm": 2.0730725351705477, |
| "kl": 0.041748046875, |
| "learning_rate": 8.183823529411764e-07, |
| "loss": 0.0017, |
| "reward": 1.8775808811187744, |
| "reward_std": 0.02502569369971752, |
| "rewards/accuracy_reward": 0.8787827491760254, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 247 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 114.89663696289062, |
| "epoch": 3.6470588235294117, |
| "grad_norm": 2.0409255064434504, |
| "kl": 0.04931640625, |
| "learning_rate": 8.176470588235293e-07, |
| "loss": 0.002, |
| "reward": 1.887112021446228, |
| "reward_std": 0.02062002196907997, |
| "rewards/accuracy_reward": 0.8871120810508728, |
| "rewards/format_reward": 1.0, |
| "step": 248 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 114.95913696289062, |
| "epoch": 3.661764705882353, |
| "grad_norm": 1.4309091507183769, |
| "kl": 0.04638671875, |
| "learning_rate": 8.169117647058823e-07, |
| "loss": 0.0019, |
| "reward": 1.8621455430984497, |
| "reward_std": 0.02185475081205368, |
| "rewards/accuracy_reward": 0.8621454834938049, |
| "rewards/format_reward": 1.0, |
| "step": 249 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 115.15144348144531, |
| "epoch": 3.6764705882352944, |
| "grad_norm": 2.5697762019528216, |
| "kl": 0.0390625, |
| "learning_rate": 8.161764705882353e-07, |
| "loss": 0.0016, |
| "reward": 1.885780692100525, |
| "reward_std": 0.020246701315045357, |
| "rewards/accuracy_reward": 0.8857808113098145, |
| "rewards/format_reward": 1.0, |
| "step": 250 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.49159240722656, |
| "epoch": 3.6911764705882355, |
| "grad_norm": 1.879132846743946, |
| "kl": 0.040283203125, |
| "learning_rate": 8.154411764705881e-07, |
| "loss": 0.0016, |
| "reward": 1.8713446855545044, |
| "reward_std": 0.024630680680274963, |
| "rewards/accuracy_reward": 0.8725464940071106, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 251 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 114.16226196289062, |
| "epoch": 3.7058823529411766, |
| "grad_norm": 2.1971208610905966, |
| "kl": 0.0400390625, |
| "learning_rate": 8.147058823529411e-07, |
| "loss": 0.0016, |
| "reward": 1.8803523778915405, |
| "reward_std": 0.02186383679509163, |
| "rewards/accuracy_reward": 0.8803524971008301, |
| "rewards/format_reward": 1.0, |
| "step": 252 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 114.24279022216797, |
| "epoch": 3.7205882352941178, |
| "grad_norm": 1.8334229890682787, |
| "kl": 0.0380859375, |
| "learning_rate": 8.139705882352941e-07, |
| "loss": 0.0015, |
| "reward": 1.8805620670318604, |
| "reward_std": 0.021345175802707672, |
| "rewards/accuracy_reward": 0.880561888217926, |
| "rewards/format_reward": 1.0, |
| "step": 253 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.02764892578125, |
| "epoch": 3.735294117647059, |
| "grad_norm": 3.7258956882443703, |
| "kl": 0.045654296875, |
| "learning_rate": 8.13235294117647e-07, |
| "loss": 0.0018, |
| "reward": 1.9009848833084106, |
| "reward_std": 0.02094642072916031, |
| "rewards/accuracy_reward": 0.9009847044944763, |
| "rewards/format_reward": 1.0, |
| "step": 254 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 114.55769348144531, |
| "epoch": 3.75, |
| "grad_norm": 1.678601436345225, |
| "kl": 0.045166015625, |
| "learning_rate": 8.125e-07, |
| "loss": 0.0018, |
| "reward": 1.8814560174942017, |
| "reward_std": 0.021061250939965248, |
| "rewards/accuracy_reward": 0.8814561367034912, |
| "rewards/format_reward": 1.0, |
| "step": 255 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.65264892578125, |
| "epoch": 3.764705882352941, |
| "grad_norm": 1.745259980031886, |
| "kl": 0.04052734375, |
| "learning_rate": 8.117647058823529e-07, |
| "loss": 0.0016, |
| "reward": 1.8852721452713013, |
| "reward_std": 0.020116155967116356, |
| "rewards/accuracy_reward": 0.8852721452713013, |
| "rewards/format_reward": 1.0, |
| "step": 256 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.62019348144531, |
| "epoch": 3.7794117647058822, |
| "grad_norm": 2.461830018990095, |
| "kl": 0.041015625, |
| "learning_rate": 8.110294117647058e-07, |
| "loss": 0.0016, |
| "reward": 1.8814154863357544, |
| "reward_std": 0.02218296006321907, |
| "rewards/accuracy_reward": 0.8814153075218201, |
| "rewards/format_reward": 1.0, |
| "step": 257 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.7175521850586, |
| "epoch": 3.7941176470588234, |
| "grad_norm": 1.68704697360333, |
| "kl": 0.03955078125, |
| "learning_rate": 8.102941176470588e-07, |
| "loss": 0.0016, |
| "reward": 1.8427629470825195, |
| "reward_std": 0.023804400116205215, |
| "rewards/accuracy_reward": 0.8427630066871643, |
| "rewards/format_reward": 1.0, |
| "step": 258 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.58894348144531, |
| "epoch": 3.8088235294117645, |
| "grad_norm": 1.709855378144107, |
| "kl": 0.04052734375, |
| "learning_rate": 8.095588235294118e-07, |
| "loss": 0.0016, |
| "reward": 1.882066011428833, |
| "reward_std": 0.024021144956350327, |
| "rewards/accuracy_reward": 0.8832679986953735, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 259 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.78245544433594, |
| "epoch": 3.8235294117647056, |
| "grad_norm": 1.8255695685831124, |
| "kl": 0.04150390625, |
| "learning_rate": 8.088235294117646e-07, |
| "loss": 0.0016, |
| "reward": 1.8689602613449097, |
| "reward_std": 0.019115259870886803, |
| "rewards/accuracy_reward": 0.8689602017402649, |
| "rewards/format_reward": 1.0, |
| "step": 260 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.55409240722656, |
| "epoch": 3.838235294117647, |
| "grad_norm": 3.3416809660726403, |
| "kl": 0.04052734375, |
| "learning_rate": 8.080882352941176e-07, |
| "loss": 0.0017, |
| "reward": 1.8896381855010986, |
| "reward_std": 0.019461615011096, |
| "rewards/accuracy_reward": 0.8896381855010986, |
| "rewards/format_reward": 1.0, |
| "step": 261 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.46634674072266, |
| "epoch": 3.8529411764705883, |
| "grad_norm": 2.630911489725864, |
| "kl": 0.0380859375, |
| "learning_rate": 8.073529411764706e-07, |
| "loss": 0.0015, |
| "reward": 1.87398362159729, |
| "reward_std": 0.023873545229434967, |
| "rewards/accuracy_reward": 0.8739834427833557, |
| "rewards/format_reward": 1.0, |
| "step": 262 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.01683044433594, |
| "epoch": 3.8676470588235294, |
| "grad_norm": 1.843021207465952, |
| "kl": 0.040283203125, |
| "learning_rate": 8.066176470588235e-07, |
| "loss": 0.0016, |
| "reward": 1.8928405046463013, |
| "reward_std": 0.019875595346093178, |
| "rewards/accuracy_reward": 0.892840564250946, |
| "rewards/format_reward": 1.0, |
| "step": 263 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.4627456665039, |
| "epoch": 3.8823529411764706, |
| "grad_norm": 2.191986984722032, |
| "kl": 0.040283203125, |
| "learning_rate": 8.058823529411764e-07, |
| "loss": 0.0016, |
| "reward": 1.868396282196045, |
| "reward_std": 0.025705352425575256, |
| "rewards/accuracy_reward": 0.8683962225914001, |
| "rewards/format_reward": 1.0, |
| "step": 264 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.62139892578125, |
| "epoch": 3.8970588235294117, |
| "grad_norm": 1.9987217362414245, |
| "kl": 0.037353515625, |
| "learning_rate": 8.051470588235294e-07, |
| "loss": 0.0015, |
| "reward": 1.864815592765808, |
| "reward_std": 0.02225598692893982, |
| "rewards/accuracy_reward": 0.8648155331611633, |
| "rewards/format_reward": 1.0, |
| "step": 265 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.4988021850586, |
| "epoch": 3.911764705882353, |
| "grad_norm": 2.050016597438298, |
| "kl": 0.0400390625, |
| "learning_rate": 8.044117647058823e-07, |
| "loss": 0.0016, |
| "reward": 1.8612146377563477, |
| "reward_std": 0.02399851381778717, |
| "rewards/accuracy_reward": 0.8612145781517029, |
| "rewards/format_reward": 1.0, |
| "step": 266 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.59375762939453, |
| "epoch": 3.9264705882352944, |
| "grad_norm": 1.6627115812158326, |
| "kl": 0.043701171875, |
| "learning_rate": 8.036764705882353e-07, |
| "loss": 0.0017, |
| "reward": 1.8737601041793823, |
| "reward_std": 0.024240681901574135, |
| "rewards/accuracy_reward": 0.8737599849700928, |
| "rewards/format_reward": 1.0, |
| "step": 267 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.94351196289062, |
| "epoch": 3.9411764705882355, |
| "grad_norm": 1.9515217712056225, |
| "kl": 0.0439453125, |
| "learning_rate": 8.029411764705883e-07, |
| "loss": 0.0018, |
| "reward": 1.866271734237671, |
| "reward_std": 0.023485863581299782, |
| "rewards/accuracy_reward": 0.8662715554237366, |
| "rewards/format_reward": 1.0, |
| "step": 268 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.50721740722656, |
| "epoch": 3.9558823529411766, |
| "grad_norm": 1.741853556080663, |
| "kl": 0.05029296875, |
| "learning_rate": 8.022058823529411e-07, |
| "loss": 0.002, |
| "reward": 1.88307523727417, |
| "reward_std": 0.021244022995233536, |
| "rewards/accuracy_reward": 0.8830749988555908, |
| "rewards/format_reward": 1.0, |
| "step": 269 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.47356414794922, |
| "epoch": 3.9705882352941178, |
| "grad_norm": 4.485332406331801, |
| "kl": 0.05712890625, |
| "learning_rate": 8.014705882352941e-07, |
| "loss": 0.0023, |
| "reward": 1.8939332962036133, |
| "reward_std": 0.025166498497128487, |
| "rewards/accuracy_reward": 0.8939332365989685, |
| "rewards/format_reward": 1.0, |
| "step": 270 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.5613021850586, |
| "epoch": 3.985294117647059, |
| "grad_norm": 1.6003108364548544, |
| "kl": 0.060302734375, |
| "learning_rate": 8.007352941176471e-07, |
| "loss": 0.0024, |
| "reward": 1.883571982383728, |
| "reward_std": 0.025911405682563782, |
| "rewards/accuracy_reward": 0.8847737312316895, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 271 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.46575164794922, |
| "epoch": 4.0, |
| "grad_norm": 1.5983921955450255, |
| "kl": 0.0537109375, |
| "learning_rate": 8e-07, |
| "loss": 0.0022, |
| "reward": 1.8933547735214233, |
| "reward_std": 0.02186509408056736, |
| "rewards/accuracy_reward": 0.8933547735214233, |
| "rewards/format_reward": 1.0, |
| "step": 272 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.33413696289062, |
| "epoch": 4.014705882352941, |
| "grad_norm": 2.727093502391197, |
| "kl": 0.053466796875, |
| "learning_rate": 7.992647058823529e-07, |
| "loss": 0.0021, |
| "reward": 1.8750649690628052, |
| "reward_std": 0.019625522196292877, |
| "rewards/accuracy_reward": 0.8750649094581604, |
| "rewards/format_reward": 1.0, |
| "step": 273 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.62019348144531, |
| "epoch": 4.029411764705882, |
| "grad_norm": 1.8871571408393943, |
| "kl": 0.06103515625, |
| "learning_rate": 7.985294117647059e-07, |
| "loss": 0.0024, |
| "reward": 1.8743054866790771, |
| "reward_std": 0.024585077539086342, |
| "rewards/accuracy_reward": 0.8743056058883667, |
| "rewards/format_reward": 1.0, |
| "step": 274 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 101.87019348144531, |
| "epoch": 4.044117647058823, |
| "grad_norm": 2.102514327126164, |
| "kl": 0.05810546875, |
| "learning_rate": 7.977941176470588e-07, |
| "loss": 0.0023, |
| "reward": 1.89711594581604, |
| "reward_std": 0.021427426487207413, |
| "rewards/accuracy_reward": 0.8971157670021057, |
| "rewards/format_reward": 1.0, |
| "step": 275 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 101.05529022216797, |
| "epoch": 4.0588235294117645, |
| "grad_norm": 1.5123356737294997, |
| "kl": 0.05810546875, |
| "learning_rate": 7.970588235294118e-07, |
| "loss": 0.0023, |
| "reward": 1.8750731945037842, |
| "reward_std": 0.022188987582921982, |
| "rewards/accuracy_reward": 0.875073254108429, |
| "rewards/format_reward": 1.0, |
| "step": 276 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 100.95072174072266, |
| "epoch": 4.073529411764706, |
| "grad_norm": 1.4897094983434982, |
| "kl": 0.060791015625, |
| "learning_rate": 7.963235294117646e-07, |
| "loss": 0.0024, |
| "reward": 1.886474847793579, |
| "reward_std": 0.021508391946554184, |
| "rewards/accuracy_reward": 0.8864747881889343, |
| "rewards/format_reward": 1.0, |
| "step": 277 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 100.2800521850586, |
| "epoch": 4.088235294117647, |
| "grad_norm": 3.202169591157568, |
| "kl": 0.05517578125, |
| "learning_rate": 7.955882352941175e-07, |
| "loss": 0.0022, |
| "reward": 1.8837499618530273, |
| "reward_std": 0.02077949419617653, |
| "rewards/accuracy_reward": 0.8849518299102783, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 278 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 102.98798370361328, |
| "epoch": 4.102941176470588, |
| "grad_norm": 2.3522518543200115, |
| "kl": 0.0498046875, |
| "learning_rate": 7.948529411764705e-07, |
| "loss": 0.002, |
| "reward": 1.8832675218582153, |
| "reward_std": 0.02067939005792141, |
| "rewards/accuracy_reward": 0.883267343044281, |
| "rewards/format_reward": 1.0, |
| "step": 279 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.70072174072266, |
| "epoch": 4.117647058823529, |
| "grad_norm": 1.4424360808940544, |
| "kl": 0.0478515625, |
| "learning_rate": 7.941176470588235e-07, |
| "loss": 0.0019, |
| "reward": 1.8915950059890747, |
| "reward_std": 0.018967358395457268, |
| "rewards/accuracy_reward": 0.8915950059890747, |
| "rewards/format_reward": 1.0, |
| "step": 280 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.0096206665039, |
| "epoch": 4.132352941176471, |
| "grad_norm": 1.6320380993577588, |
| "kl": 0.057373046875, |
| "learning_rate": 7.933823529411764e-07, |
| "loss": 0.0023, |
| "reward": 1.8841755390167236, |
| "reward_std": 0.019956490024924278, |
| "rewards/accuracy_reward": 0.8841755390167236, |
| "rewards/format_reward": 1.0, |
| "step": 281 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.82452392578125, |
| "epoch": 4.147058823529412, |
| "grad_norm": 1.6748728201505565, |
| "kl": 0.0478515625, |
| "learning_rate": 7.926470588235293e-07, |
| "loss": 0.0019, |
| "reward": 1.881099820137024, |
| "reward_std": 0.01961349882185459, |
| "rewards/accuracy_reward": 0.8810995817184448, |
| "rewards/format_reward": 1.0, |
| "step": 282 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.52404022216797, |
| "epoch": 4.161764705882353, |
| "grad_norm": 3.540072890075679, |
| "kl": 0.04736328125, |
| "learning_rate": 7.919117647058823e-07, |
| "loss": 0.0019, |
| "reward": 1.8972245454788208, |
| "reward_std": 0.020341571420431137, |
| "rewards/accuracy_reward": 0.897224485874176, |
| "rewards/format_reward": 1.0, |
| "step": 283 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.55288696289062, |
| "epoch": 4.176470588235294, |
| "grad_norm": 1.9653905551543096, |
| "kl": 0.045654296875, |
| "learning_rate": 7.911764705882352e-07, |
| "loss": 0.0018, |
| "reward": 1.8803445100784302, |
| "reward_std": 0.022631661966443062, |
| "rewards/accuracy_reward": 0.8803444504737854, |
| "rewards/format_reward": 1.0, |
| "step": 284 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.54688262939453, |
| "epoch": 4.1911764705882355, |
| "grad_norm": 3.95007121019553, |
| "kl": 0.046875, |
| "learning_rate": 7.904411764705882e-07, |
| "loss": 0.0019, |
| "reward": 1.8853232860565186, |
| "reward_std": 0.02392488718032837, |
| "rewards/accuracy_reward": 0.8865249752998352, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 285 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.4627456665039, |
| "epoch": 4.205882352941177, |
| "grad_norm": 1.6574614891623431, |
| "kl": 0.05078125, |
| "learning_rate": 7.897058823529411e-07, |
| "loss": 0.002, |
| "reward": 1.8902332782745361, |
| "reward_std": 0.024903936311602592, |
| "rewards/accuracy_reward": 0.8914350867271423, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 286 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.29688262939453, |
| "epoch": 4.220588235294118, |
| "grad_norm": 3.256913991460478, |
| "kl": 0.046142578125, |
| "learning_rate": 7.88970588235294e-07, |
| "loss": 0.0018, |
| "reward": 1.8858234882354736, |
| "reward_std": 0.024734003469347954, |
| "rewards/accuracy_reward": 0.8882272243499756, |
| "rewards/format_reward": 0.9975962042808533, |
| "step": 287 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.60697174072266, |
| "epoch": 4.235294117647059, |
| "grad_norm": 1.3114281230029299, |
| "kl": 0.0458984375, |
| "learning_rate": 7.88235294117647e-07, |
| "loss": 0.0019, |
| "reward": 1.890177607536316, |
| "reward_std": 0.02220279537141323, |
| "rewards/accuracy_reward": 0.8901773691177368, |
| "rewards/format_reward": 1.0, |
| "step": 288 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 116.62741088867188, |
| "epoch": 4.25, |
| "grad_norm": 1.4493002218382003, |
| "kl": 0.049560546875, |
| "learning_rate": 7.875e-07, |
| "loss": 0.002, |
| "reward": 1.8985555171966553, |
| "reward_std": 0.020094698294997215, |
| "rewards/accuracy_reward": 0.8985555768013, |
| "rewards/format_reward": 1.0, |
| "step": 289 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 116.68389892578125, |
| "epoch": 4.264705882352941, |
| "grad_norm": 4.352470640135198, |
| "kl": 0.04931640625, |
| "learning_rate": 7.867647058823529e-07, |
| "loss": 0.002, |
| "reward": 1.8766732215881348, |
| "reward_std": 0.027829263359308243, |
| "rewards/accuracy_reward": 0.8790771961212158, |
| "rewards/format_reward": 0.9975962042808533, |
| "step": 290 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 114.04688262939453, |
| "epoch": 4.279411764705882, |
| "grad_norm": 3.874276314718884, |
| "kl": 0.0830078125, |
| "learning_rate": 7.860294117647058e-07, |
| "loss": 0.0033, |
| "reward": 1.8896328210830688, |
| "reward_std": 0.021389471367001534, |
| "rewards/accuracy_reward": 0.8896327614784241, |
| "rewards/format_reward": 1.0, |
| "step": 291 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.61178588867188, |
| "epoch": 4.294117647058823, |
| "grad_norm": 4.875295082608794, |
| "kl": 0.04541015625, |
| "learning_rate": 7.852941176470588e-07, |
| "loss": 0.0018, |
| "reward": 1.8860790729522705, |
| "reward_std": 0.021392209455370903, |
| "rewards/accuracy_reward": 0.8872808814048767, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 292 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.39543914794922, |
| "epoch": 4.3088235294117645, |
| "grad_norm": 1.5633011375678962, |
| "kl": 0.04345703125, |
| "learning_rate": 7.845588235294117e-07, |
| "loss": 0.0017, |
| "reward": 1.8786873817443848, |
| "reward_std": 0.021603846922516823, |
| "rewards/accuracy_reward": 0.8786874413490295, |
| "rewards/format_reward": 1.0, |
| "step": 293 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 114.50360870361328, |
| "epoch": 4.323529411764706, |
| "grad_norm": 1.4773757778833134, |
| "kl": 0.047119140625, |
| "learning_rate": 7.838235294117647e-07, |
| "loss": 0.0019, |
| "reward": 1.8730394840240479, |
| "reward_std": 0.02080570161342621, |
| "rewards/accuracy_reward": 0.8730394244194031, |
| "rewards/format_reward": 1.0, |
| "step": 294 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.4363021850586, |
| "epoch": 4.338235294117647, |
| "grad_norm": 1.8801422889509678, |
| "kl": 0.04638671875, |
| "learning_rate": 7.830882352941176e-07, |
| "loss": 0.0019, |
| "reward": 1.8871146440505981, |
| "reward_std": 0.020325137302279472, |
| "rewards/accuracy_reward": 0.8871145844459534, |
| "rewards/format_reward": 1.0, |
| "step": 295 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.09014892578125, |
| "epoch": 4.352941176470588, |
| "grad_norm": 2.5097065769254594, |
| "kl": 0.0498046875, |
| "learning_rate": 7.823529411764705e-07, |
| "loss": 0.002, |
| "reward": 1.868382453918457, |
| "reward_std": 0.02320130541920662, |
| "rewards/accuracy_reward": 0.8695842623710632, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 296 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 114.3377456665039, |
| "epoch": 4.367647058823529, |
| "grad_norm": 2.553942778131699, |
| "kl": 0.044189453125, |
| "learning_rate": 7.816176470588235e-07, |
| "loss": 0.0018, |
| "reward": 1.9048007726669312, |
| "reward_std": 0.019152075052261353, |
| "rewards/accuracy_reward": 0.9048006534576416, |
| "rewards/format_reward": 1.0, |
| "step": 297 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.56610870361328, |
| "epoch": 4.382352941176471, |
| "grad_norm": 1.2815769671843946, |
| "kl": 0.049560546875, |
| "learning_rate": 7.808823529411765e-07, |
| "loss": 0.002, |
| "reward": 1.8882514238357544, |
| "reward_std": 0.023224936798214912, |
| "rewards/accuracy_reward": 0.8906552791595459, |
| "rewards/format_reward": 0.9975962042808533, |
| "step": 298 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.27043914794922, |
| "epoch": 4.397058823529412, |
| "grad_norm": 1.4236109919766304, |
| "kl": 0.0439453125, |
| "learning_rate": 7.801470588235293e-07, |
| "loss": 0.0018, |
| "reward": 1.8794277906417847, |
| "reward_std": 0.022136587649583817, |
| "rewards/accuracy_reward": 0.8818315863609314, |
| "rewards/format_reward": 0.9975962042808533, |
| "step": 299 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.25000762939453, |
| "epoch": 4.411764705882353, |
| "grad_norm": 1.4092988693440858, |
| "kl": 0.044189453125, |
| "learning_rate": 7.794117647058823e-07, |
| "loss": 0.0018, |
| "reward": 1.8663372993469238, |
| "reward_std": 0.02092735283076763, |
| "rewards/accuracy_reward": 0.8663372993469238, |
| "rewards/format_reward": 1.0, |
| "step": 300 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 115.10577392578125, |
| "epoch": 4.426470588235294, |
| "grad_norm": 2.054063342384727, |
| "kl": 0.0517578125, |
| "learning_rate": 7.786764705882353e-07, |
| "loss": 0.0021, |
| "reward": 1.8976588249206543, |
| "reward_std": 0.01807982660830021, |
| "rewards/accuracy_reward": 0.8976587653160095, |
| "rewards/format_reward": 1.0, |
| "step": 301 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.3377456665039, |
| "epoch": 4.4411764705882355, |
| "grad_norm": 1.9093857237441312, |
| "kl": 0.047607421875, |
| "learning_rate": 7.779411764705882e-07, |
| "loss": 0.0019, |
| "reward": 1.8833731412887573, |
| "reward_std": 0.01974165067076683, |
| "rewards/accuracy_reward": 0.8833730816841125, |
| "rewards/format_reward": 1.0, |
| "step": 302 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.30409240722656, |
| "epoch": 4.455882352941177, |
| "grad_norm": 2.6735018202140988, |
| "kl": 0.04541015625, |
| "learning_rate": 7.772058823529412e-07, |
| "loss": 0.0018, |
| "reward": 1.8876824378967285, |
| "reward_std": 0.019306330010294914, |
| "rewards/accuracy_reward": 0.8876824975013733, |
| "rewards/format_reward": 1.0, |
| "step": 303 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.91106414794922, |
| "epoch": 4.470588235294118, |
| "grad_norm": 1.4970895120586163, |
| "kl": 0.0439453125, |
| "learning_rate": 7.764705882352941e-07, |
| "loss": 0.0018, |
| "reward": 1.8828318119049072, |
| "reward_std": 0.020149705931544304, |
| "rewards/accuracy_reward": 0.8840336799621582, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 304 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.24519348144531, |
| "epoch": 4.485294117647059, |
| "grad_norm": 5.117496046280143, |
| "kl": 0.04296875, |
| "learning_rate": 7.75735294117647e-07, |
| "loss": 0.0017, |
| "reward": 1.8947540521621704, |
| "reward_std": 0.024016601964831352, |
| "rewards/accuracy_reward": 0.8971578478813171, |
| "rewards/format_reward": 0.9975962042808533, |
| "step": 305 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.50360870361328, |
| "epoch": 4.5, |
| "grad_norm": 2.5813219086011028, |
| "kl": 0.052001953125, |
| "learning_rate": 7.75e-07, |
| "loss": 0.0021, |
| "reward": 1.8869359493255615, |
| "reward_std": 0.02114337682723999, |
| "rewards/accuracy_reward": 0.8869357705116272, |
| "rewards/format_reward": 1.0, |
| "step": 306 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.6550521850586, |
| "epoch": 4.514705882352941, |
| "grad_norm": 2.5546889887402235, |
| "kl": 0.0458984375, |
| "learning_rate": 7.74264705882353e-07, |
| "loss": 0.0018, |
| "reward": 1.8768377304077148, |
| "reward_std": 0.020170308649539948, |
| "rewards/accuracy_reward": 0.8768375515937805, |
| "rewards/format_reward": 1.0, |
| "step": 307 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.10216522216797, |
| "epoch": 4.529411764705882, |
| "grad_norm": 2.0444532213674775, |
| "kl": 0.048828125, |
| "learning_rate": 7.735294117647058e-07, |
| "loss": 0.002, |
| "reward": 1.8887766599655151, |
| "reward_std": 0.019136058166623116, |
| "rewards/accuracy_reward": 0.8887766599655151, |
| "rewards/format_reward": 1.0, |
| "step": 308 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.73918914794922, |
| "epoch": 4.544117647058823, |
| "grad_norm": 1.4865300788991052, |
| "kl": 0.05419921875, |
| "learning_rate": 7.727941176470588e-07, |
| "loss": 0.0022, |
| "reward": 1.8884928226470947, |
| "reward_std": 0.018670309334993362, |
| "rewards/accuracy_reward": 0.88849276304245, |
| "rewards/format_reward": 1.0, |
| "step": 309 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.81610870361328, |
| "epoch": 4.5588235294117645, |
| "grad_norm": 1.8317615570289676, |
| "kl": 0.0498046875, |
| "learning_rate": 7.720588235294118e-07, |
| "loss": 0.002, |
| "reward": 1.8673958778381348, |
| "reward_std": 0.022479576990008354, |
| "rewards/accuracy_reward": 0.8673959374427795, |
| "rewards/format_reward": 1.0, |
| "step": 310 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.82933044433594, |
| "epoch": 4.573529411764706, |
| "grad_norm": 1.3908315302560892, |
| "kl": 0.047119140625, |
| "learning_rate": 7.713235294117647e-07, |
| "loss": 0.0019, |
| "reward": 1.8690990209579468, |
| "reward_std": 0.01971607096493244, |
| "rewards/accuracy_reward": 0.8690990805625916, |
| "rewards/format_reward": 1.0, |
| "step": 311 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.58053588867188, |
| "epoch": 4.588235294117647, |
| "grad_norm": 2.2483220834096223, |
| "kl": 0.0458984375, |
| "learning_rate": 7.705882352941177e-07, |
| "loss": 0.0018, |
| "reward": 1.8699548244476318, |
| "reward_std": 0.02238030731678009, |
| "rewards/accuracy_reward": 0.8699548840522766, |
| "rewards/format_reward": 1.0, |
| "step": 312 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.37741088867188, |
| "epoch": 4.602941176470588, |
| "grad_norm": 1.9027749884419907, |
| "kl": 0.049560546875, |
| "learning_rate": 7.698529411764706e-07, |
| "loss": 0.002, |
| "reward": 1.8906946182250977, |
| "reward_std": 0.02228682115674019, |
| "rewards/accuracy_reward": 0.8918964266777039, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 313 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.70673370361328, |
| "epoch": 4.617647058823529, |
| "grad_norm": 1.2672783381025527, |
| "kl": 0.045654296875, |
| "learning_rate": 7.691176470588235e-07, |
| "loss": 0.0018, |
| "reward": 1.8969817161560059, |
| "reward_std": 0.01983834058046341, |
| "rewards/accuracy_reward": 0.8969816565513611, |
| "rewards/format_reward": 1.0, |
| "step": 314 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.74279022216797, |
| "epoch": 4.632352941176471, |
| "grad_norm": 1.3268803478175482, |
| "kl": 0.0458984375, |
| "learning_rate": 7.683823529411765e-07, |
| "loss": 0.0018, |
| "reward": 1.884127140045166, |
| "reward_std": 0.02158300392329693, |
| "rewards/accuracy_reward": 0.884127140045166, |
| "rewards/format_reward": 1.0, |
| "step": 315 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.26683044433594, |
| "epoch": 4.647058823529412, |
| "grad_norm": 4.254278045186561, |
| "kl": 0.045654296875, |
| "learning_rate": 7.676470588235295e-07, |
| "loss": 0.0018, |
| "reward": 1.8847973346710205, |
| "reward_std": 0.023614434525370598, |
| "rewards/accuracy_reward": 0.887201189994812, |
| "rewards/format_reward": 0.9975962042808533, |
| "step": 316 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.02644348144531, |
| "epoch": 4.661764705882353, |
| "grad_norm": 1.4138173515058006, |
| "kl": 0.045654296875, |
| "learning_rate": 7.669117647058823e-07, |
| "loss": 0.0018, |
| "reward": 1.8916139602661133, |
| "reward_std": 0.021671609953045845, |
| "rewards/accuracy_reward": 0.8916139006614685, |
| "rewards/format_reward": 1.0, |
| "step": 317 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.03485870361328, |
| "epoch": 4.676470588235294, |
| "grad_norm": 1.4297077613699323, |
| "kl": 0.041748046875, |
| "learning_rate": 7.661764705882353e-07, |
| "loss": 0.0017, |
| "reward": 1.8865143060684204, |
| "reward_std": 0.022390324622392654, |
| "rewards/accuracy_reward": 0.8877161145210266, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 318 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.99759674072266, |
| "epoch": 4.6911764705882355, |
| "grad_norm": 1.2033391337835728, |
| "kl": 0.041259765625, |
| "learning_rate": 7.654411764705882e-07, |
| "loss": 0.0017, |
| "reward": 1.8856085538864136, |
| "reward_std": 0.024989847093820572, |
| "rewards/accuracy_reward": 0.8868104815483093, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 319 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.29447174072266, |
| "epoch": 4.705882352941177, |
| "grad_norm": 1.3733454266831897, |
| "kl": 0.044189453125, |
| "learning_rate": 7.647058823529411e-07, |
| "loss": 0.0018, |
| "reward": 1.878577709197998, |
| "reward_std": 0.021778374910354614, |
| "rewards/accuracy_reward": 0.8785776495933533, |
| "rewards/format_reward": 1.0, |
| "step": 320 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.90745544433594, |
| "epoch": 4.720588235294118, |
| "grad_norm": 1.5274339339441825, |
| "kl": 0.039794921875, |
| "learning_rate": 7.63970588235294e-07, |
| "loss": 0.0016, |
| "reward": 1.872501015663147, |
| "reward_std": 0.01973523572087288, |
| "rewards/accuracy_reward": 0.8725009560585022, |
| "rewards/format_reward": 1.0, |
| "step": 321 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.89063262939453, |
| "epoch": 4.735294117647059, |
| "grad_norm": 1.5962482033199081, |
| "kl": 0.041015625, |
| "learning_rate": 7.63235294117647e-07, |
| "loss": 0.0016, |
| "reward": 1.8869467973709106, |
| "reward_std": 0.022963199764490128, |
| "rewards/accuracy_reward": 0.8881486058235168, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 322 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.35938262939453, |
| "epoch": 4.75, |
| "grad_norm": 2.017481820546224, |
| "kl": 0.039794921875, |
| "learning_rate": 7.624999999999999e-07, |
| "loss": 0.0016, |
| "reward": 1.8841745853424072, |
| "reward_std": 0.025449533015489578, |
| "rewards/accuracy_reward": 0.886578381061554, |
| "rewards/format_reward": 0.9975962042808533, |
| "step": 323 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.48918914794922, |
| "epoch": 4.764705882352941, |
| "grad_norm": 5.893277558364017, |
| "kl": 0.04052734375, |
| "learning_rate": 7.617647058823529e-07, |
| "loss": 0.0016, |
| "reward": 1.8966243267059326, |
| "reward_std": 0.017657946795225143, |
| "rewards/accuracy_reward": 0.8966242074966431, |
| "rewards/format_reward": 1.0, |
| "step": 324 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.59495544433594, |
| "epoch": 4.779411764705882, |
| "grad_norm": 1.9364427240337212, |
| "kl": 0.03857421875, |
| "learning_rate": 7.610294117647059e-07, |
| "loss": 0.0015, |
| "reward": 1.8959040641784668, |
| "reward_std": 0.01980137825012207, |
| "rewards/accuracy_reward": 0.8959041237831116, |
| "rewards/format_reward": 1.0, |
| "step": 325 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.75000762939453, |
| "epoch": 4.794117647058823, |
| "grad_norm": 1.8831495019330111, |
| "kl": 0.042724609375, |
| "learning_rate": 7.602941176470587e-07, |
| "loss": 0.0017, |
| "reward": 1.8923227787017822, |
| "reward_std": 0.021822385489940643, |
| "rewards/accuracy_reward": 0.8935245871543884, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 326 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.97476196289062, |
| "epoch": 4.8088235294117645, |
| "grad_norm": 3.7686300331111924, |
| "kl": 0.0478515625, |
| "learning_rate": 7.595588235294117e-07, |
| "loss": 0.0019, |
| "reward": 1.9007014036178589, |
| "reward_std": 0.020531214773654938, |
| "rewards/accuracy_reward": 0.9031053781509399, |
| "rewards/format_reward": 0.9975962042808533, |
| "step": 327 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.31610870361328, |
| "epoch": 4.823529411764706, |
| "grad_norm": 2.3075627417580566, |
| "kl": 0.043212890625, |
| "learning_rate": 7.588235294117647e-07, |
| "loss": 0.0017, |
| "reward": 1.8857510089874268, |
| "reward_std": 0.019290894269943237, |
| "rewards/accuracy_reward": 0.8857510089874268, |
| "rewards/format_reward": 1.0, |
| "step": 328 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.52284240722656, |
| "epoch": 4.838235294117647, |
| "grad_norm": 2.4505241717663218, |
| "kl": 0.044677734375, |
| "learning_rate": 7.580882352941176e-07, |
| "loss": 0.0018, |
| "reward": 1.8795123100280762, |
| "reward_std": 0.017949938774108887, |
| "rewards/accuracy_reward": 0.8795123100280762, |
| "rewards/format_reward": 1.0, |
| "step": 329 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.17548370361328, |
| "epoch": 4.852941176470588, |
| "grad_norm": 1.715533751803751, |
| "kl": 0.04443359375, |
| "learning_rate": 7.573529411764705e-07, |
| "loss": 0.0018, |
| "reward": 1.8877590894699097, |
| "reward_std": 0.01760478876531124, |
| "rewards/accuracy_reward": 0.8877590298652649, |
| "rewards/format_reward": 1.0, |
| "step": 330 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.52644348144531, |
| "epoch": 4.867647058823529, |
| "grad_norm": 1.6176970942840496, |
| "kl": 0.049560546875, |
| "learning_rate": 7.566176470588235e-07, |
| "loss": 0.002, |
| "reward": 1.8978863954544067, |
| "reward_std": 0.01671546697616577, |
| "rewards/accuracy_reward": 0.8978862166404724, |
| "rewards/format_reward": 1.0, |
| "step": 331 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.68870544433594, |
| "epoch": 4.882352941176471, |
| "grad_norm": 1.7008680805466612, |
| "kl": 0.04541015625, |
| "learning_rate": 7.558823529411764e-07, |
| "loss": 0.0018, |
| "reward": 1.8799728155136108, |
| "reward_std": 0.017163129523396492, |
| "rewards/accuracy_reward": 0.8799727559089661, |
| "rewards/format_reward": 1.0, |
| "step": 332 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.3064956665039, |
| "epoch": 4.897058823529412, |
| "grad_norm": 5.121313120929418, |
| "kl": 0.04443359375, |
| "learning_rate": 7.551470588235294e-07, |
| "loss": 0.0018, |
| "reward": 1.8954551219940186, |
| "reward_std": 0.01760135032236576, |
| "rewards/accuracy_reward": 0.8954549431800842, |
| "rewards/format_reward": 1.0, |
| "step": 333 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.14784240722656, |
| "epoch": 4.911764705882353, |
| "grad_norm": 1.3552009766911794, |
| "kl": 0.046142578125, |
| "learning_rate": 7.544117647058824e-07, |
| "loss": 0.0018, |
| "reward": 1.8577083349227905, |
| "reward_std": 0.018814481794834137, |
| "rewards/accuracy_reward": 0.8577081561088562, |
| "rewards/format_reward": 1.0, |
| "step": 334 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.55889892578125, |
| "epoch": 4.926470588235294, |
| "grad_norm": 2.140682303302643, |
| "kl": 0.05078125, |
| "learning_rate": 7.536764705882352e-07, |
| "loss": 0.002, |
| "reward": 1.881057620048523, |
| "reward_std": 0.019572759047150612, |
| "rewards/accuracy_reward": 0.881057620048523, |
| "rewards/format_reward": 1.0, |
| "step": 335 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.62620544433594, |
| "epoch": 4.9411764705882355, |
| "grad_norm": 3.272070631910461, |
| "kl": 0.046630859375, |
| "learning_rate": 7.529411764705882e-07, |
| "loss": 0.0019, |
| "reward": 1.8901574611663818, |
| "reward_std": 0.015406808815896511, |
| "rewards/accuracy_reward": 0.8901574611663818, |
| "rewards/format_reward": 1.0, |
| "step": 336 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.64784240722656, |
| "epoch": 4.955882352941177, |
| "grad_norm": 3.6448062675563726, |
| "kl": 0.053955078125, |
| "learning_rate": 7.522058823529412e-07, |
| "loss": 0.0022, |
| "reward": 1.875773310661316, |
| "reward_std": 0.02223069965839386, |
| "rewards/accuracy_reward": 0.8769751191139221, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 337 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.64904022216797, |
| "epoch": 4.970588235294118, |
| "grad_norm": 1.4354791062562573, |
| "kl": 0.06103515625, |
| "learning_rate": 7.514705882352941e-07, |
| "loss": 0.0024, |
| "reward": 1.8930749893188477, |
| "reward_std": 0.018565351143479347, |
| "rewards/accuracy_reward": 0.8930751085281372, |
| "rewards/format_reward": 1.0, |
| "step": 338 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.71995544433594, |
| "epoch": 4.985294117647059, |
| "grad_norm": 1.8660094880654952, |
| "kl": 0.05810546875, |
| "learning_rate": 7.50735294117647e-07, |
| "loss": 0.0023, |
| "reward": 1.8842915296554565, |
| "reward_std": 0.02008167654275894, |
| "rewards/accuracy_reward": 0.8842914700508118, |
| "rewards/format_reward": 1.0, |
| "step": 339 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.37290954589844, |
| "epoch": 5.0, |
| "grad_norm": 1.508014879105866, |
| "kl": 0.05078125, |
| "learning_rate": 7.5e-07, |
| "loss": 0.002, |
| "reward": 1.8771898746490479, |
| "reward_std": 0.02064666524529457, |
| "rewards/accuracy_reward": 0.8771898150444031, |
| "rewards/format_reward": 1.0, |
| "step": 340 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.55889892578125, |
| "epoch": 5.014705882352941, |
| "grad_norm": 1.6330775894781793, |
| "kl": 0.0478515625, |
| "learning_rate": 7.492647058823529e-07, |
| "loss": 0.0019, |
| "reward": 1.8954782485961914, |
| "reward_std": 0.017391271889209747, |
| "rewards/accuracy_reward": 0.8954780697822571, |
| "rewards/format_reward": 1.0, |
| "step": 341 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.3738021850586, |
| "epoch": 5.029411764705882, |
| "grad_norm": 1.5308902611838158, |
| "kl": 0.04833984375, |
| "learning_rate": 7.485294117647059e-07, |
| "loss": 0.0019, |
| "reward": 1.9155678749084473, |
| "reward_std": 0.018119707703590393, |
| "rewards/accuracy_reward": 0.9167699217796326, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 342 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.8269271850586, |
| "epoch": 5.044117647058823, |
| "grad_norm": 3.6296477320844893, |
| "kl": 0.06298828125, |
| "learning_rate": 7.477941176470587e-07, |
| "loss": 0.0025, |
| "reward": 1.8854138851165771, |
| "reward_std": 0.02051103487610817, |
| "rewards/accuracy_reward": 0.8854138851165771, |
| "rewards/format_reward": 1.0, |
| "step": 343 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.56370544433594, |
| "epoch": 5.0588235294117645, |
| "grad_norm": 3.0646702385899562, |
| "kl": 0.047119140625, |
| "learning_rate": 7.470588235294117e-07, |
| "loss": 0.0019, |
| "reward": 1.9035041332244873, |
| "reward_std": 0.0216043870896101, |
| "rewards/accuracy_reward": 0.9047059416770935, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 344 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.50241088867188, |
| "epoch": 5.073529411764706, |
| "grad_norm": 1.7545084393443575, |
| "kl": 0.049560546875, |
| "learning_rate": 7.463235294117647e-07, |
| "loss": 0.002, |
| "reward": 1.879274606704712, |
| "reward_std": 0.020033851265907288, |
| "rewards/accuracy_reward": 0.8792744278907776, |
| "rewards/format_reward": 1.0, |
| "step": 345 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.07572174072266, |
| "epoch": 5.088235294117647, |
| "grad_norm": 2.265810746797048, |
| "kl": 0.0458984375, |
| "learning_rate": 7.455882352941176e-07, |
| "loss": 0.0018, |
| "reward": 1.8713810443878174, |
| "reward_std": 0.019391771405935287, |
| "rewards/accuracy_reward": 0.8713809251785278, |
| "rewards/format_reward": 1.0, |
| "step": 346 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.33053588867188, |
| "epoch": 5.102941176470588, |
| "grad_norm": 4.678653380466847, |
| "kl": 0.058349609375, |
| "learning_rate": 7.448529411764706e-07, |
| "loss": 0.0023, |
| "reward": 1.8980027437210083, |
| "reward_std": 0.019409772008657455, |
| "rewards/accuracy_reward": 0.8992045521736145, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 347 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.4519271850586, |
| "epoch": 5.117647058823529, |
| "grad_norm": 5.587657670792633, |
| "kl": 0.044921875, |
| "learning_rate": 7.441176470588235e-07, |
| "loss": 0.0018, |
| "reward": 1.8735015392303467, |
| "reward_std": 0.019884636625647545, |
| "rewards/accuracy_reward": 0.8735013604164124, |
| "rewards/format_reward": 1.0, |
| "step": 348 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.25000762939453, |
| "epoch": 5.132352941176471, |
| "grad_norm": 1.4732972699207496, |
| "kl": 0.050048828125, |
| "learning_rate": 7.433823529411764e-07, |
| "loss": 0.002, |
| "reward": 1.8633267879486084, |
| "reward_std": 0.020572712644934654, |
| "rewards/accuracy_reward": 0.8633267879486084, |
| "rewards/format_reward": 1.0, |
| "step": 349 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.14303588867188, |
| "epoch": 5.147058823529412, |
| "grad_norm": 1.4434229984530293, |
| "kl": 0.051025390625, |
| "learning_rate": 7.426470588235294e-07, |
| "loss": 0.002, |
| "reward": 1.8769383430480957, |
| "reward_std": 0.02142389491200447, |
| "rewards/accuracy_reward": 0.8781403303146362, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 350 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.86418914794922, |
| "epoch": 5.161764705882353, |
| "grad_norm": 2.15469914353684, |
| "kl": 0.04541015625, |
| "learning_rate": 7.419117647058824e-07, |
| "loss": 0.0018, |
| "reward": 1.882927417755127, |
| "reward_std": 0.019372547045350075, |
| "rewards/accuracy_reward": 0.882927417755127, |
| "rewards/format_reward": 1.0, |
| "step": 351 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.43389892578125, |
| "epoch": 5.176470588235294, |
| "grad_norm": 1.7010413230089305, |
| "kl": 0.04345703125, |
| "learning_rate": 7.411764705882352e-07, |
| "loss": 0.0017, |
| "reward": 1.8917574882507324, |
| "reward_std": 0.021130278706550598, |
| "rewards/accuracy_reward": 0.8929594159126282, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 352 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.03726196289062, |
| "epoch": 5.1911764705882355, |
| "grad_norm": 1.4178252033931626, |
| "kl": 0.0458984375, |
| "learning_rate": 7.404411764705882e-07, |
| "loss": 0.0018, |
| "reward": 1.902689814567566, |
| "reward_std": 0.020760996267199516, |
| "rewards/accuracy_reward": 0.9038916826248169, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 353 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.5721206665039, |
| "epoch": 5.205882352941177, |
| "grad_norm": 3.030997635370593, |
| "kl": 0.049072265625, |
| "learning_rate": 7.397058823529412e-07, |
| "loss": 0.002, |
| "reward": 1.8927603960037231, |
| "reward_std": 0.017168840393424034, |
| "rewards/accuracy_reward": 0.8927604556083679, |
| "rewards/format_reward": 1.0, |
| "step": 354 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.91106414794922, |
| "epoch": 5.220588235294118, |
| "grad_norm": 1.680026683397978, |
| "kl": 0.046630859375, |
| "learning_rate": 7.389705882352941e-07, |
| "loss": 0.0019, |
| "reward": 1.9062864780426025, |
| "reward_std": 0.021335626021027565, |
| "rewards/accuracy_reward": 0.9074884653091431, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 355 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.44231414794922, |
| "epoch": 5.235294117647059, |
| "grad_norm": 1.9153566616206694, |
| "kl": 0.044189453125, |
| "learning_rate": 7.382352941176471e-07, |
| "loss": 0.0018, |
| "reward": 1.8809791803359985, |
| "reward_std": 0.02271600253880024, |
| "rewards/accuracy_reward": 0.8821810483932495, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 356 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.19231414794922, |
| "epoch": 5.25, |
| "grad_norm": 3.558694171062752, |
| "kl": 0.048583984375, |
| "learning_rate": 7.375e-07, |
| "loss": 0.002, |
| "reward": 1.9085917472839355, |
| "reward_std": 0.01498875766992569, |
| "rewards/accuracy_reward": 0.9085916876792908, |
| "rewards/format_reward": 1.0, |
| "step": 357 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 114.37019348144531, |
| "epoch": 5.264705882352941, |
| "grad_norm": 2.0945595152655208, |
| "kl": 0.04443359375, |
| "learning_rate": 7.367647058823529e-07, |
| "loss": 0.0018, |
| "reward": 1.8866630792617798, |
| "reward_std": 0.021569877862930298, |
| "rewards/accuracy_reward": 0.8890668749809265, |
| "rewards/format_reward": 0.9975962042808533, |
| "step": 358 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.05769348144531, |
| "epoch": 5.279411764705882, |
| "grad_norm": 1.8644120471415602, |
| "kl": 0.048095703125, |
| "learning_rate": 7.360294117647059e-07, |
| "loss": 0.0019, |
| "reward": 1.890414834022522, |
| "reward_std": 0.021897265687584877, |
| "rewards/accuracy_reward": 0.891616702079773, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 359 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.48438262939453, |
| "epoch": 5.294117647058823, |
| "grad_norm": 2.9245490886556236, |
| "kl": 0.044921875, |
| "learning_rate": 7.352941176470589e-07, |
| "loss": 0.0018, |
| "reward": 1.8733696937561035, |
| "reward_std": 0.019373353570699692, |
| "rewards/accuracy_reward": 0.8745716214179993, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 360 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.13101196289062, |
| "epoch": 5.3088235294117645, |
| "grad_norm": 2.4171192429296933, |
| "kl": 0.04833984375, |
| "learning_rate": 7.345588235294117e-07, |
| "loss": 0.0019, |
| "reward": 1.8899723291397095, |
| "reward_std": 0.020467419177293777, |
| "rewards/accuracy_reward": 0.8911741971969604, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 361 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.25000762939453, |
| "epoch": 5.323529411764706, |
| "grad_norm": 2.1361138297509035, |
| "kl": 0.0439453125, |
| "learning_rate": 7.338235294117646e-07, |
| "loss": 0.0018, |
| "reward": 1.892791509628296, |
| "reward_std": 0.015906570479273796, |
| "rewards/accuracy_reward": 0.8927913308143616, |
| "rewards/format_reward": 1.0, |
| "step": 362 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.56370544433594, |
| "epoch": 5.338235294117647, |
| "grad_norm": 1.92362426869984, |
| "kl": 0.045166015625, |
| "learning_rate": 7.330882352941176e-07, |
| "loss": 0.0018, |
| "reward": 1.8666787147521973, |
| "reward_std": 0.019119028002023697, |
| "rewards/accuracy_reward": 0.8666786551475525, |
| "rewards/format_reward": 1.0, |
| "step": 363 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.6971206665039, |
| "epoch": 5.352941176470588, |
| "grad_norm": 1.6581703712574578, |
| "kl": 0.049072265625, |
| "learning_rate": 7.323529411764705e-07, |
| "loss": 0.002, |
| "reward": 1.874783992767334, |
| "reward_std": 0.01916446164250374, |
| "rewards/accuracy_reward": 0.8759857416152954, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 364 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.17428588867188, |
| "epoch": 5.367647058823529, |
| "grad_norm": 1.316540945229491, |
| "kl": 0.0478515625, |
| "learning_rate": 7.316176470588234e-07, |
| "loss": 0.0019, |
| "reward": 1.8962125778198242, |
| "reward_std": 0.01739468425512314, |
| "rewards/accuracy_reward": 0.8962125182151794, |
| "rewards/format_reward": 1.0, |
| "step": 365 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.37620544433594, |
| "epoch": 5.382352941176471, |
| "grad_norm": 8.565938358324495, |
| "kl": 0.046630859375, |
| "learning_rate": 7.308823529411764e-07, |
| "loss": 0.0019, |
| "reward": 1.8905795812606812, |
| "reward_std": 0.020930087193846703, |
| "rewards/accuracy_reward": 0.890579342842102, |
| "rewards/format_reward": 1.0, |
| "step": 366 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.68991088867188, |
| "epoch": 5.397058823529412, |
| "grad_norm": 3.469819197381053, |
| "kl": 0.05419921875, |
| "learning_rate": 7.301470588235293e-07, |
| "loss": 0.0022, |
| "reward": 1.8995338678359985, |
| "reward_std": 0.02121318317949772, |
| "rewards/accuracy_reward": 0.9007357358932495, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 367 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.24759674072266, |
| "epoch": 5.411764705882353, |
| "grad_norm": 3.6183994362346517, |
| "kl": 0.04248046875, |
| "learning_rate": 7.294117647058823e-07, |
| "loss": 0.0017, |
| "reward": 1.8944286108016968, |
| "reward_std": 0.019341690465807915, |
| "rewards/accuracy_reward": 0.894428551197052, |
| "rewards/format_reward": 1.0, |
| "step": 368 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.42188262939453, |
| "epoch": 5.426470588235294, |
| "grad_norm": 1.5985396075036147, |
| "kl": 0.046630859375, |
| "learning_rate": 7.286764705882353e-07, |
| "loss": 0.0019, |
| "reward": 1.9031792879104614, |
| "reward_std": 0.01647799275815487, |
| "rewards/accuracy_reward": 0.9031792283058167, |
| "rewards/format_reward": 1.0, |
| "step": 369 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.56851196289062, |
| "epoch": 5.4411764705882355, |
| "grad_norm": 1.2316701809022303, |
| "kl": 0.0439453125, |
| "learning_rate": 7.279411764705881e-07, |
| "loss": 0.0018, |
| "reward": 1.874247670173645, |
| "reward_std": 0.021124592050909996, |
| "rewards/accuracy_reward": 0.8754495978355408, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 370 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.75360870361328, |
| "epoch": 5.455882352941177, |
| "grad_norm": 2.02457072563307, |
| "kl": 0.046142578125, |
| "learning_rate": 7.272058823529411e-07, |
| "loss": 0.0018, |
| "reward": 1.8876434564590454, |
| "reward_std": 0.018463727086782455, |
| "rewards/accuracy_reward": 0.887643575668335, |
| "rewards/format_reward": 1.0, |
| "step": 371 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.8894271850586, |
| "epoch": 5.470588235294118, |
| "grad_norm": 1.9670592101281013, |
| "kl": 0.04296875, |
| "learning_rate": 7.264705882352941e-07, |
| "loss": 0.0017, |
| "reward": 1.8814630508422852, |
| "reward_std": 0.016402965411543846, |
| "rewards/accuracy_reward": 0.8814628720283508, |
| "rewards/format_reward": 1.0, |
| "step": 372 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.15625762939453, |
| "epoch": 5.485294117647059, |
| "grad_norm": 1.4520387331980766, |
| "kl": 0.044189453125, |
| "learning_rate": 7.25735294117647e-07, |
| "loss": 0.0018, |
| "reward": 1.8984284400939941, |
| "reward_std": 0.015249278396368027, |
| "rewards/accuracy_reward": 0.8984284400939941, |
| "rewards/format_reward": 1.0, |
| "step": 373 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.3738021850586, |
| "epoch": 5.5, |
| "grad_norm": 1.5706724220082209, |
| "kl": 0.04833984375, |
| "learning_rate": 7.249999999999999e-07, |
| "loss": 0.0019, |
| "reward": 1.8929314613342285, |
| "reward_std": 0.018274588510394096, |
| "rewards/accuracy_reward": 0.8929312825202942, |
| "rewards/format_reward": 1.0, |
| "step": 374 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.56370544433594, |
| "epoch": 5.514705882352941, |
| "grad_norm": 3.532450016398957, |
| "kl": 0.0439453125, |
| "learning_rate": 7.242647058823529e-07, |
| "loss": 0.0018, |
| "reward": 1.892069935798645, |
| "reward_std": 0.019071701914072037, |
| "rewards/accuracy_reward": 0.8920699954032898, |
| "rewards/format_reward": 1.0, |
| "step": 375 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.27284240722656, |
| "epoch": 5.529411764705882, |
| "grad_norm": 2.9126439595526112, |
| "kl": 0.052734375, |
| "learning_rate": 7.235294117647058e-07, |
| "loss": 0.0021, |
| "reward": 1.8809665441513062, |
| "reward_std": 0.01572202891111374, |
| "rewards/accuracy_reward": 0.8809664845466614, |
| "rewards/format_reward": 1.0, |
| "step": 376 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.28846740722656, |
| "epoch": 5.544117647058823, |
| "grad_norm": 2.3143306804592423, |
| "kl": 0.048095703125, |
| "learning_rate": 7.227941176470588e-07, |
| "loss": 0.0019, |
| "reward": 1.905410885810852, |
| "reward_std": 0.015785330906510353, |
| "rewards/accuracy_reward": 0.9054108262062073, |
| "rewards/format_reward": 1.0, |
| "step": 377 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.16947174072266, |
| "epoch": 5.5588235294117645, |
| "grad_norm": 2.6475439861754095, |
| "kl": 0.04638671875, |
| "learning_rate": 7.220588235294118e-07, |
| "loss": 0.0019, |
| "reward": 1.8877785205841064, |
| "reward_std": 0.01584857515990734, |
| "rewards/accuracy_reward": 0.8877785205841064, |
| "rewards/format_reward": 1.0, |
| "step": 378 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.41346740722656, |
| "epoch": 5.573529411764706, |
| "grad_norm": 2.025239158313011, |
| "kl": 0.04736328125, |
| "learning_rate": 7.213235294117646e-07, |
| "loss": 0.0019, |
| "reward": 1.9032666683197021, |
| "reward_std": 0.016153108328580856, |
| "rewards/accuracy_reward": 0.9032665491104126, |
| "rewards/format_reward": 1.0, |
| "step": 379 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.09616088867188, |
| "epoch": 5.588235294117647, |
| "grad_norm": 4.81037382465536, |
| "kl": 0.049072265625, |
| "learning_rate": 7.205882352941176e-07, |
| "loss": 0.002, |
| "reward": 1.8973829746246338, |
| "reward_std": 0.018959591165184975, |
| "rewards/accuracy_reward": 0.8985849618911743, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 380 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.60456848144531, |
| "epoch": 5.602941176470588, |
| "grad_norm": 2.0424310708762543, |
| "kl": 0.0478515625, |
| "learning_rate": 7.198529411764706e-07, |
| "loss": 0.0019, |
| "reward": 1.891837239265442, |
| "reward_std": 0.017265740782022476, |
| "rewards/accuracy_reward": 0.8918372392654419, |
| "rewards/format_reward": 1.0, |
| "step": 381 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.49639892578125, |
| "epoch": 5.617647058823529, |
| "grad_norm": 7.450414291777024, |
| "kl": 0.040283203125, |
| "learning_rate": 7.191176470588235e-07, |
| "loss": 0.0016, |
| "reward": 1.8869714736938477, |
| "reward_std": 0.014449415728449821, |
| "rewards/accuracy_reward": 0.8869713544845581, |
| "rewards/format_reward": 1.0, |
| "step": 382 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.50841522216797, |
| "epoch": 5.632352941176471, |
| "grad_norm": 2.035166070284095, |
| "kl": 0.05029296875, |
| "learning_rate": 7.183823529411764e-07, |
| "loss": 0.002, |
| "reward": 1.8704235553741455, |
| "reward_std": 0.01924438774585724, |
| "rewards/accuracy_reward": 0.8704236149787903, |
| "rewards/format_reward": 1.0, |
| "step": 383 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.93269348144531, |
| "epoch": 5.647058823529412, |
| "grad_norm": 1.5304768166279001, |
| "kl": 0.04638671875, |
| "learning_rate": 7.176470588235294e-07, |
| "loss": 0.0019, |
| "reward": 1.889725923538208, |
| "reward_std": 0.020155081525444984, |
| "rewards/accuracy_reward": 0.890927791595459, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 384 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.50481414794922, |
| "epoch": 5.661764705882353, |
| "grad_norm": 3.3086513126052437, |
| "kl": 0.046875, |
| "learning_rate": 7.169117647058823e-07, |
| "loss": 0.0019, |
| "reward": 1.9017019271850586, |
| "reward_std": 0.02102070115506649, |
| "rewards/accuracy_reward": 0.9029037952423096, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 385 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.59856414794922, |
| "epoch": 5.676470588235294, |
| "grad_norm": 1.9961992837605382, |
| "kl": 0.04443359375, |
| "learning_rate": 7.161764705882353e-07, |
| "loss": 0.0018, |
| "reward": 1.898403286933899, |
| "reward_std": 0.01933371089398861, |
| "rewards/accuracy_reward": 0.8996050953865051, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 386 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.00360870361328, |
| "epoch": 5.6911764705882355, |
| "grad_norm": 5.861203308278023, |
| "kl": 0.044189453125, |
| "learning_rate": 7.154411764705882e-07, |
| "loss": 0.0018, |
| "reward": 1.9097044467926025, |
| "reward_std": 0.017147168517112732, |
| "rewards/accuracy_reward": 0.9097043871879578, |
| "rewards/format_reward": 1.0, |
| "step": 387 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.95433044433594, |
| "epoch": 5.705882352941177, |
| "grad_norm": 1.8272044694889817, |
| "kl": 0.046142578125, |
| "learning_rate": 7.147058823529411e-07, |
| "loss": 0.0018, |
| "reward": 1.897651195526123, |
| "reward_std": 0.019564587622880936, |
| "rewards/accuracy_reward": 0.897650957107544, |
| "rewards/format_reward": 1.0, |
| "step": 388 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.26202392578125, |
| "epoch": 5.720588235294118, |
| "grad_norm": 1.945047556411728, |
| "kl": 0.053955078125, |
| "learning_rate": 7.139705882352941e-07, |
| "loss": 0.0022, |
| "reward": 1.8845726251602173, |
| "reward_std": 0.018275026232004166, |
| "rewards/accuracy_reward": 0.8845725655555725, |
| "rewards/format_reward": 1.0, |
| "step": 389 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.27764892578125, |
| "epoch": 5.735294117647059, |
| "grad_norm": 1.872210801006099, |
| "kl": 0.0419921875, |
| "learning_rate": 7.132352941176471e-07, |
| "loss": 0.0017, |
| "reward": 1.8972781896591187, |
| "reward_std": 0.019161643460392952, |
| "rewards/accuracy_reward": 0.8972780704498291, |
| "rewards/format_reward": 1.0, |
| "step": 390 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.4206771850586, |
| "epoch": 5.75, |
| "grad_norm": 1.628706031121031, |
| "kl": 0.044921875, |
| "learning_rate": 7.125e-07, |
| "loss": 0.0018, |
| "reward": 1.8868787288665771, |
| "reward_std": 0.017812801524996758, |
| "rewards/accuracy_reward": 0.8868787884712219, |
| "rewards/format_reward": 1.0, |
| "step": 391 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.36538696289062, |
| "epoch": 5.764705882352941, |
| "grad_norm": 2.025154646985711, |
| "kl": 0.043701171875, |
| "learning_rate": 7.117647058823529e-07, |
| "loss": 0.0017, |
| "reward": 1.8947402238845825, |
| "reward_std": 0.016441920772194862, |
| "rewards/accuracy_reward": 0.8947402834892273, |
| "rewards/format_reward": 1.0, |
| "step": 392 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.59375762939453, |
| "epoch": 5.779411764705882, |
| "grad_norm": 3.1223862690686466, |
| "kl": 0.04443359375, |
| "learning_rate": 7.110294117647059e-07, |
| "loss": 0.0018, |
| "reward": 1.884828805923462, |
| "reward_std": 0.02042386308312416, |
| "rewards/accuracy_reward": 0.8848287463188171, |
| "rewards/format_reward": 1.0, |
| "step": 393 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.52163696289062, |
| "epoch": 5.794117647058823, |
| "grad_norm": 1.2474938760026444, |
| "kl": 0.0439453125, |
| "learning_rate": 7.102941176470588e-07, |
| "loss": 0.0018, |
| "reward": 1.894531488418579, |
| "reward_std": 0.017643319442868233, |
| "rewards/accuracy_reward": 0.8945314288139343, |
| "rewards/format_reward": 1.0, |
| "step": 394 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.53966522216797, |
| "epoch": 5.8088235294117645, |
| "grad_norm": 2.4031954689781556, |
| "kl": 0.04443359375, |
| "learning_rate": 7.095588235294118e-07, |
| "loss": 0.0018, |
| "reward": 1.906731367111206, |
| "reward_std": 0.019448986276984215, |
| "rewards/accuracy_reward": 0.9079332947731018, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 395 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.2752456665039, |
| "epoch": 5.823529411764706, |
| "grad_norm": 2.3766132625090495, |
| "kl": 0.06005859375, |
| "learning_rate": 7.088235294117647e-07, |
| "loss": 0.0024, |
| "reward": 1.9001885652542114, |
| "reward_std": 0.018285507336258888, |
| "rewards/accuracy_reward": 0.9001885056495667, |
| "rewards/format_reward": 1.0, |
| "step": 396 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.21034240722656, |
| "epoch": 5.838235294117647, |
| "grad_norm": 1.3410708995130616, |
| "kl": 0.049560546875, |
| "learning_rate": 7.080882352941176e-07, |
| "loss": 0.002, |
| "reward": 1.8895319700241089, |
| "reward_std": 0.019128086045384407, |
| "rewards/accuracy_reward": 0.8907338976860046, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 397 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.0300521850586, |
| "epoch": 5.852941176470588, |
| "grad_norm": 2.1806221304312188, |
| "kl": 0.047607421875, |
| "learning_rate": 7.073529411764706e-07, |
| "loss": 0.0019, |
| "reward": 1.8873209953308105, |
| "reward_std": 0.015813617035746574, |
| "rewards/accuracy_reward": 0.8873209357261658, |
| "rewards/format_reward": 1.0, |
| "step": 398 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.05889892578125, |
| "epoch": 5.867647058823529, |
| "grad_norm": 2.381149520093015, |
| "kl": 0.05126953125, |
| "learning_rate": 7.066176470588236e-07, |
| "loss": 0.0021, |
| "reward": 1.9061532020568848, |
| "reward_std": 0.017387444153428078, |
| "rewards/accuracy_reward": 0.9061532020568848, |
| "rewards/format_reward": 1.0, |
| "step": 399 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.60216522216797, |
| "epoch": 5.882352941176471, |
| "grad_norm": 2.0072217926486235, |
| "kl": 0.048583984375, |
| "learning_rate": 7.058823529411765e-07, |
| "loss": 0.0019, |
| "reward": 1.8860886096954346, |
| "reward_std": 0.01758231595158577, |
| "rewards/accuracy_reward": 0.8860886096954346, |
| "rewards/format_reward": 1.0, |
| "step": 400 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.2908706665039, |
| "epoch": 5.897058823529412, |
| "grad_norm": 3.7976903995467244, |
| "kl": 0.043701171875, |
| "learning_rate": 7.051470588235294e-07, |
| "loss": 0.0017, |
| "reward": 1.9081175327301025, |
| "reward_std": 0.016382873058319092, |
| "rewards/accuracy_reward": 0.9081174731254578, |
| "rewards/format_reward": 1.0, |
| "step": 401 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.97235870361328, |
| "epoch": 5.911764705882353, |
| "grad_norm": 1.6832136746582174, |
| "kl": 0.05322265625, |
| "learning_rate": 7.044117647058824e-07, |
| "loss": 0.0021, |
| "reward": 1.8958063125610352, |
| "reward_std": 0.014682255685329437, |
| "rewards/accuracy_reward": 0.8958064317703247, |
| "rewards/format_reward": 1.0, |
| "step": 402 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.90384674072266, |
| "epoch": 5.926470588235294, |
| "grad_norm": 2.1815402582773618, |
| "kl": 0.0498046875, |
| "learning_rate": 7.036764705882353e-07, |
| "loss": 0.002, |
| "reward": 1.8909062147140503, |
| "reward_std": 0.018609097227454185, |
| "rewards/accuracy_reward": 0.8909061551094055, |
| "rewards/format_reward": 1.0, |
| "step": 403 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.67788696289062, |
| "epoch": 5.9411764705882355, |
| "grad_norm": 1.2061130128497455, |
| "kl": 0.047119140625, |
| "learning_rate": 7.029411764705882e-07, |
| "loss": 0.0019, |
| "reward": 1.8931101560592651, |
| "reward_std": 0.01808551885187626, |
| "rewards/accuracy_reward": 0.8931100368499756, |
| "rewards/format_reward": 1.0, |
| "step": 404 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.68870544433594, |
| "epoch": 5.955882352941177, |
| "grad_norm": 1.9433195416220157, |
| "kl": 0.046630859375, |
| "learning_rate": 7.022058823529411e-07, |
| "loss": 0.0019, |
| "reward": 1.9073554277420044, |
| "reward_std": 0.018337463960051537, |
| "rewards/accuracy_reward": 0.9073554277420044, |
| "rewards/format_reward": 1.0, |
| "step": 405 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.47476196289062, |
| "epoch": 5.970588235294118, |
| "grad_norm": 1.8820064437816768, |
| "kl": 0.060546875, |
| "learning_rate": 7.01470588235294e-07, |
| "loss": 0.0024, |
| "reward": 1.894400715827942, |
| "reward_std": 0.021061552688479424, |
| "rewards/accuracy_reward": 0.8944007158279419, |
| "rewards/format_reward": 1.0, |
| "step": 406 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.52884674072266, |
| "epoch": 5.985294117647059, |
| "grad_norm": 1.4623484883692877, |
| "kl": 0.05859375, |
| "learning_rate": 7.00735294117647e-07, |
| "loss": 0.0024, |
| "reward": 1.9013231992721558, |
| "reward_std": 0.02373763732612133, |
| "rewards/accuracy_reward": 0.9037269949913025, |
| "rewards/format_reward": 0.9975962042808533, |
| "step": 407 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.86758422851562, |
| "epoch": 6.0, |
| "grad_norm": 2.1711922114519124, |
| "kl": 0.048828125, |
| "learning_rate": 7e-07, |
| "loss": 0.002, |
| "reward": 1.9016249179840088, |
| "reward_std": 0.023924626410007477, |
| "rewards/accuracy_reward": 0.9046690464019775, |
| "rewards/format_reward": 0.9969558715820312, |
| "step": 408 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.19110870361328, |
| "epoch": 6.014705882352941, |
| "grad_norm": 1.7310411415632332, |
| "kl": 0.052490234375, |
| "learning_rate": 6.992647058823528e-07, |
| "loss": 0.0021, |
| "reward": 1.8854323625564575, |
| "reward_std": 0.023845864459872246, |
| "rewards/accuracy_reward": 0.8866342902183533, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 409 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.46875762939453, |
| "epoch": 6.029411764705882, |
| "grad_norm": 5.687861204362904, |
| "kl": 0.0908203125, |
| "learning_rate": 6.985294117647058e-07, |
| "loss": 0.0036, |
| "reward": 1.9103180170059204, |
| "reward_std": 0.017014116048812866, |
| "rewards/accuracy_reward": 0.9103179574012756, |
| "rewards/format_reward": 1.0, |
| "step": 410 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.84134674072266, |
| "epoch": 6.044117647058823, |
| "grad_norm": 1.8355942786177852, |
| "kl": 0.048583984375, |
| "learning_rate": 6.977941176470588e-07, |
| "loss": 0.002, |
| "reward": 1.8937817811965942, |
| "reward_std": 0.018505675718188286, |
| "rewards/accuracy_reward": 0.8937817215919495, |
| "rewards/format_reward": 1.0, |
| "step": 411 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.79688262939453, |
| "epoch": 6.0588235294117645, |
| "grad_norm": 1.9759679376426085, |
| "kl": 0.052001953125, |
| "learning_rate": 6.970588235294117e-07, |
| "loss": 0.0021, |
| "reward": 1.8942158222198486, |
| "reward_std": 0.02279289998114109, |
| "rewards/accuracy_reward": 0.8954177498817444, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 412 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.37860870361328, |
| "epoch": 6.073529411764706, |
| "grad_norm": 2.300855046609347, |
| "kl": 0.059814453125, |
| "learning_rate": 6.963235294117647e-07, |
| "loss": 0.0024, |
| "reward": 1.8962349891662598, |
| "reward_std": 0.020851602777838707, |
| "rewards/accuracy_reward": 0.8962349891662598, |
| "rewards/format_reward": 1.0, |
| "step": 413 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.5252456665039, |
| "epoch": 6.088235294117647, |
| "grad_norm": 12.310047514562172, |
| "kl": 0.047607421875, |
| "learning_rate": 6.955882352941176e-07, |
| "loss": 0.0019, |
| "reward": 1.899317979812622, |
| "reward_std": 0.020527588203549385, |
| "rewards/accuracy_reward": 0.9005199074745178, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 414 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.1238021850586, |
| "epoch": 6.102941176470588, |
| "grad_norm": 1.5229186035110283, |
| "kl": 0.05224609375, |
| "learning_rate": 6.948529411764705e-07, |
| "loss": 0.0021, |
| "reward": 1.8821978569030762, |
| "reward_std": 0.022272054105997086, |
| "rewards/accuracy_reward": 0.8833997249603271, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 415 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.74279022216797, |
| "epoch": 6.117647058823529, |
| "grad_norm": 3.2878165148453236, |
| "kl": 0.053955078125, |
| "learning_rate": 6.941176470588235e-07, |
| "loss": 0.0022, |
| "reward": 1.86842679977417, |
| "reward_std": 0.024334656074643135, |
| "rewards/accuracy_reward": 0.8684268593788147, |
| "rewards/format_reward": 1.0, |
| "step": 416 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.48918914794922, |
| "epoch": 6.132352941176471, |
| "grad_norm": 1.2525615714604925, |
| "kl": 0.05029296875, |
| "learning_rate": 6.933823529411765e-07, |
| "loss": 0.002, |
| "reward": 1.914664387702942, |
| "reward_std": 0.01958000287413597, |
| "rewards/accuracy_reward": 0.9146643877029419, |
| "rewards/format_reward": 1.0, |
| "step": 417 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.93269348144531, |
| "epoch": 6.147058823529412, |
| "grad_norm": 2.561139147224278, |
| "kl": 0.052734375, |
| "learning_rate": 6.926470588235293e-07, |
| "loss": 0.0021, |
| "reward": 1.9154963493347168, |
| "reward_std": 0.016698814928531647, |
| "rewards/accuracy_reward": 0.9154961109161377, |
| "rewards/format_reward": 1.0, |
| "step": 418 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.98077392578125, |
| "epoch": 6.161764705882353, |
| "grad_norm": 2.1752639111351666, |
| "kl": 0.058349609375, |
| "learning_rate": 6.919117647058823e-07, |
| "loss": 0.0023, |
| "reward": 1.9047746658325195, |
| "reward_std": 0.018907994031906128, |
| "rewards/accuracy_reward": 0.9047747254371643, |
| "rewards/format_reward": 1.0, |
| "step": 419 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.78606414794922, |
| "epoch": 6.176470588235294, |
| "grad_norm": 2.0292217325556416, |
| "kl": 0.0517578125, |
| "learning_rate": 6.911764705882353e-07, |
| "loss": 0.0021, |
| "reward": 1.9034497737884521, |
| "reward_std": 0.024114146828651428, |
| "rewards/accuracy_reward": 0.9034495949745178, |
| "rewards/format_reward": 1.0, |
| "step": 420 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.4519271850586, |
| "epoch": 6.1911764705882355, |
| "grad_norm": 6.699279802691781, |
| "kl": 0.055419921875, |
| "learning_rate": 6.904411764705882e-07, |
| "loss": 0.0022, |
| "reward": 1.887842059135437, |
| "reward_std": 0.020647389814257622, |
| "rewards/accuracy_reward": 0.887842059135437, |
| "rewards/format_reward": 1.0, |
| "step": 421 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.42188262939453, |
| "epoch": 6.205882352941177, |
| "grad_norm": 1.534235488134726, |
| "kl": 0.050537109375, |
| "learning_rate": 6.897058823529411e-07, |
| "loss": 0.002, |
| "reward": 1.896329402923584, |
| "reward_std": 0.018744876608252525, |
| "rewards/accuracy_reward": 0.8963293433189392, |
| "rewards/format_reward": 1.0, |
| "step": 422 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.9831771850586, |
| "epoch": 6.220588235294118, |
| "grad_norm": 1.5766135493091737, |
| "kl": 0.0478515625, |
| "learning_rate": 6.889705882352941e-07, |
| "loss": 0.0019, |
| "reward": 1.8838109970092773, |
| "reward_std": 0.02198025956749916, |
| "rewards/accuracy_reward": 0.8850129246711731, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 423 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.84616088867188, |
| "epoch": 6.235294117647059, |
| "grad_norm": 2.0708203706815853, |
| "kl": 0.078125, |
| "learning_rate": 6.88235294117647e-07, |
| "loss": 0.0031, |
| "reward": 1.8984771966934204, |
| "reward_std": 0.021190447732806206, |
| "rewards/accuracy_reward": 0.8984770178794861, |
| "rewards/format_reward": 1.0, |
| "step": 424 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.41226196289062, |
| "epoch": 6.25, |
| "grad_norm": 1.7348463789648598, |
| "kl": 0.050048828125, |
| "learning_rate": 6.875e-07, |
| "loss": 0.002, |
| "reward": 1.9097307920455933, |
| "reward_std": 0.019229857251048088, |
| "rewards/accuracy_reward": 0.9097307324409485, |
| "rewards/format_reward": 1.0, |
| "step": 425 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.37500762939453, |
| "epoch": 6.264705882352941, |
| "grad_norm": 15.785742386708213, |
| "kl": 0.05224609375, |
| "learning_rate": 6.86764705882353e-07, |
| "loss": 0.0021, |
| "reward": 1.8983619213104248, |
| "reward_std": 0.018431352451443672, |
| "rewards/accuracy_reward": 0.8983619213104248, |
| "rewards/format_reward": 1.0, |
| "step": 426 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.81009674072266, |
| "epoch": 6.279411764705882, |
| "grad_norm": 2.6324515777384017, |
| "kl": 0.056396484375, |
| "learning_rate": 6.860294117647058e-07, |
| "loss": 0.0023, |
| "reward": 1.8895386457443237, |
| "reward_std": 0.02556595578789711, |
| "rewards/accuracy_reward": 0.889538586139679, |
| "rewards/format_reward": 1.0, |
| "step": 427 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 102.29808044433594, |
| "epoch": 6.294117647058823, |
| "grad_norm": 1.6012975476940077, |
| "kl": 0.04931640625, |
| "learning_rate": 6.852941176470588e-07, |
| "loss": 0.002, |
| "reward": 1.9019221067428589, |
| "reward_std": 0.023045556619763374, |
| "rewards/accuracy_reward": 0.9031239748001099, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 428 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 102.78125762939453, |
| "epoch": 6.3088235294117645, |
| "grad_norm": 2.761466006204092, |
| "kl": 0.0732421875, |
| "learning_rate": 6.845588235294118e-07, |
| "loss": 0.0029, |
| "reward": 1.8907809257507324, |
| "reward_std": 0.021768517792224884, |
| "rewards/accuracy_reward": 0.8907808661460876, |
| "rewards/format_reward": 1.0, |
| "step": 429 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.83413696289062, |
| "epoch": 6.323529411764706, |
| "grad_norm": 1.9867024053824853, |
| "kl": 0.05517578125, |
| "learning_rate": 6.838235294117647e-07, |
| "loss": 0.0022, |
| "reward": 1.8989307880401611, |
| "reward_std": 0.02491842210292816, |
| "rewards/accuracy_reward": 0.9001325964927673, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 430 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.92548370361328, |
| "epoch": 6.338235294117647, |
| "grad_norm": 2.506151416718587, |
| "kl": 0.060546875, |
| "learning_rate": 6.830882352941176e-07, |
| "loss": 0.0024, |
| "reward": 1.8981561660766602, |
| "reward_std": 0.022075144574046135, |
| "rewards/accuracy_reward": 0.8981561064720154, |
| "rewards/format_reward": 1.0, |
| "step": 431 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.0300521850586, |
| "epoch": 6.352941176470588, |
| "grad_norm": 3.453658386620713, |
| "kl": 0.052734375, |
| "learning_rate": 6.823529411764706e-07, |
| "loss": 0.0021, |
| "reward": 1.8945103883743286, |
| "reward_std": 0.02551359124481678, |
| "rewards/accuracy_reward": 0.8969140648841858, |
| "rewards/format_reward": 0.9975962042808533, |
| "step": 432 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.82933044433594, |
| "epoch": 6.367647058823529, |
| "grad_norm": 1.8291376872404717, |
| "kl": 0.049072265625, |
| "learning_rate": 6.816176470588235e-07, |
| "loss": 0.002, |
| "reward": 1.8951138257980347, |
| "reward_std": 0.02293815091252327, |
| "rewards/accuracy_reward": 0.8951138257980347, |
| "rewards/format_reward": 1.0, |
| "step": 433 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.97356414794922, |
| "epoch": 6.382352941176471, |
| "grad_norm": 4.4657449515144965, |
| "kl": 0.051025390625, |
| "learning_rate": 6.808823529411765e-07, |
| "loss": 0.002, |
| "reward": 1.8871821165084839, |
| "reward_std": 0.02225620485842228, |
| "rewards/accuracy_reward": 0.8871821165084839, |
| "rewards/format_reward": 1.0, |
| "step": 434 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.1033706665039, |
| "epoch": 6.397058823529412, |
| "grad_norm": 1.0719524860130538, |
| "kl": 0.05029296875, |
| "learning_rate": 6.801470588235295e-07, |
| "loss": 0.002, |
| "reward": 1.8991506099700928, |
| "reward_std": 0.024187587201595306, |
| "rewards/accuracy_reward": 0.9003525376319885, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 435 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.64063262939453, |
| "epoch": 6.411764705882353, |
| "grad_norm": 1.3740348079978744, |
| "kl": 0.04736328125, |
| "learning_rate": 6.794117647058823e-07, |
| "loss": 0.0019, |
| "reward": 1.8954823017120361, |
| "reward_std": 0.024401018396019936, |
| "rewards/accuracy_reward": 0.8954823017120361, |
| "rewards/format_reward": 1.0, |
| "step": 436 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.45433044433594, |
| "epoch": 6.426470588235294, |
| "grad_norm": 2.5851908946821287, |
| "kl": 0.05126953125, |
| "learning_rate": 6.786764705882353e-07, |
| "loss": 0.002, |
| "reward": 1.9073567390441895, |
| "reward_std": 0.020375726744532585, |
| "rewards/accuracy_reward": 0.9073567390441895, |
| "rewards/format_reward": 1.0, |
| "step": 437 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.93389892578125, |
| "epoch": 6.4411764705882355, |
| "grad_norm": 1.167566426650332, |
| "kl": 0.048828125, |
| "learning_rate": 6.779411764705883e-07, |
| "loss": 0.0019, |
| "reward": 1.8960213661193848, |
| "reward_std": 0.024139244109392166, |
| "rewards/accuracy_reward": 0.8960214257240295, |
| "rewards/format_reward": 1.0, |
| "step": 438 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.06610870361328, |
| "epoch": 6.455882352941177, |
| "grad_norm": 1.1659335106544957, |
| "kl": 0.047607421875, |
| "learning_rate": 6.772058823529412e-07, |
| "loss": 0.0019, |
| "reward": 1.8937411308288574, |
| "reward_std": 0.02366737276315689, |
| "rewards/accuracy_reward": 0.8949430584907532, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 439 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.38341522216797, |
| "epoch": 6.470588235294118, |
| "grad_norm": 3.3358776773497176, |
| "kl": 0.048095703125, |
| "learning_rate": 6.764705882352941e-07, |
| "loss": 0.0019, |
| "reward": 1.8928595781326294, |
| "reward_std": 0.021879835054278374, |
| "rewards/accuracy_reward": 0.8928595781326294, |
| "rewards/format_reward": 1.0, |
| "step": 440 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 114.35216522216797, |
| "epoch": 6.485294117647059, |
| "grad_norm": 1.3906028161781399, |
| "kl": 0.052490234375, |
| "learning_rate": 6.757352941176471e-07, |
| "loss": 0.0021, |
| "reward": 1.8927258253097534, |
| "reward_std": 0.020552638918161392, |
| "rewards/accuracy_reward": 0.8927258849143982, |
| "rewards/format_reward": 1.0, |
| "step": 441 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 114.82091522216797, |
| "epoch": 6.5, |
| "grad_norm": 3.8548633882230567, |
| "kl": 0.059814453125, |
| "learning_rate": 6.75e-07, |
| "loss": 0.0024, |
| "reward": 1.9043219089508057, |
| "reward_std": 0.02147606387734413, |
| "rewards/accuracy_reward": 0.9055235981941223, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 442 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.60456848144531, |
| "epoch": 6.514705882352941, |
| "grad_norm": 2.723744256999097, |
| "kl": 0.05419921875, |
| "learning_rate": 6.74264705882353e-07, |
| "loss": 0.0022, |
| "reward": 1.8985867500305176, |
| "reward_std": 0.017505373805761337, |
| "rewards/accuracy_reward": 0.8985867500305176, |
| "rewards/format_reward": 1.0, |
| "step": 443 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.91466522216797, |
| "epoch": 6.529411764705882, |
| "grad_norm": 2.467770138712217, |
| "kl": 0.05126953125, |
| "learning_rate": 6.735294117647058e-07, |
| "loss": 0.0021, |
| "reward": 1.8991594314575195, |
| "reward_std": 0.0224330872297287, |
| "rewards/accuracy_reward": 0.9003612995147705, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 444 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.12981414794922, |
| "epoch": 6.544117647058823, |
| "grad_norm": 1.235445619018083, |
| "kl": 0.056396484375, |
| "learning_rate": 6.727941176470588e-07, |
| "loss": 0.0023, |
| "reward": 1.8991564512252808, |
| "reward_std": 0.01990494504570961, |
| "rewards/accuracy_reward": 0.900358259677887, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 445 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.72716522216797, |
| "epoch": 6.5588235294117645, |
| "grad_norm": 4.57392237252639, |
| "kl": 0.0576171875, |
| "learning_rate": 6.720588235294118e-07, |
| "loss": 0.0023, |
| "reward": 1.8918448686599731, |
| "reward_std": 0.018118849024176598, |
| "rewards/accuracy_reward": 0.8918446898460388, |
| "rewards/format_reward": 1.0, |
| "step": 446 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.4519271850586, |
| "epoch": 6.573529411764706, |
| "grad_norm": 1.3515959803768587, |
| "kl": 0.05908203125, |
| "learning_rate": 6.713235294117646e-07, |
| "loss": 0.0024, |
| "reward": 1.8976571559906006, |
| "reward_std": 0.019342336803674698, |
| "rewards/accuracy_reward": 0.897657036781311, |
| "rewards/format_reward": 1.0, |
| "step": 447 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.30889892578125, |
| "epoch": 6.588235294117647, |
| "grad_norm": 2.000337423528591, |
| "kl": 0.0654296875, |
| "learning_rate": 6.705882352941176e-07, |
| "loss": 0.0026, |
| "reward": 1.9029864072799683, |
| "reward_std": 0.01870027370750904, |
| "rewards/accuracy_reward": 0.9029862880706787, |
| "rewards/format_reward": 1.0, |
| "step": 448 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.02764892578125, |
| "epoch": 6.602941176470588, |
| "grad_norm": 1.5831041809663697, |
| "kl": 0.05908203125, |
| "learning_rate": 6.698529411764705e-07, |
| "loss": 0.0024, |
| "reward": 1.8981215953826904, |
| "reward_std": 0.023059705272316933, |
| "rewards/accuracy_reward": 0.8993234038352966, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 449 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.4471206665039, |
| "epoch": 6.617647058823529, |
| "grad_norm": 1.7615653292492308, |
| "kl": 0.061279296875, |
| "learning_rate": 6.691176470588234e-07, |
| "loss": 0.0025, |
| "reward": 1.8908604383468628, |
| "reward_std": 0.02007007598876953, |
| "rewards/accuracy_reward": 0.890860378742218, |
| "rewards/format_reward": 1.0, |
| "step": 450 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.1346206665039, |
| "epoch": 6.632352941176471, |
| "grad_norm": 1.6994254050705504, |
| "kl": 0.06298828125, |
| "learning_rate": 6.683823529411764e-07, |
| "loss": 0.0025, |
| "reward": 1.9058598279953003, |
| "reward_std": 0.017709294334053993, |
| "rewards/accuracy_reward": 0.9058597683906555, |
| "rewards/format_reward": 1.0, |
| "step": 451 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.54688262939453, |
| "epoch": 6.647058823529412, |
| "grad_norm": 2.143476953028863, |
| "kl": 0.051513671875, |
| "learning_rate": 6.676470588235294e-07, |
| "loss": 0.0021, |
| "reward": 1.9037673473358154, |
| "reward_std": 0.01912897825241089, |
| "rewards/accuracy_reward": 0.9037670493125916, |
| "rewards/format_reward": 1.0, |
| "step": 452 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.27644348144531, |
| "epoch": 6.661764705882353, |
| "grad_norm": 2.01478887608093, |
| "kl": 0.05029296875, |
| "learning_rate": 6.669117647058822e-07, |
| "loss": 0.002, |
| "reward": 1.8817764520645142, |
| "reward_std": 0.0232061930000782, |
| "rewards/accuracy_reward": 0.8829783201217651, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 453 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.56731414794922, |
| "epoch": 6.676470588235294, |
| "grad_norm": 2.690844672468002, |
| "kl": 0.052978515625, |
| "learning_rate": 6.661764705882352e-07, |
| "loss": 0.0021, |
| "reward": 1.923115849494934, |
| "reward_std": 0.014602301642298698, |
| "rewards/accuracy_reward": 0.9231158494949341, |
| "rewards/format_reward": 1.0, |
| "step": 454 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.59495544433594, |
| "epoch": 6.6911764705882355, |
| "grad_norm": 1.7310069495348166, |
| "kl": 0.048095703125, |
| "learning_rate": 6.654411764705882e-07, |
| "loss": 0.0019, |
| "reward": 1.901757836341858, |
| "reward_std": 0.019638314843177795, |
| "rewards/accuracy_reward": 0.9017577767372131, |
| "rewards/format_reward": 1.0, |
| "step": 455 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.0877456665039, |
| "epoch": 6.705882352941177, |
| "grad_norm": 4.401458379456843, |
| "kl": 0.052978515625, |
| "learning_rate": 6.647058823529411e-07, |
| "loss": 0.0021, |
| "reward": 1.9064425230026245, |
| "reward_std": 0.018589207902550697, |
| "rewards/accuracy_reward": 0.9076444506645203, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 456 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.69231414794922, |
| "epoch": 6.720588235294118, |
| "grad_norm": 2.0195398931401867, |
| "kl": 0.04736328125, |
| "learning_rate": 6.63970588235294e-07, |
| "loss": 0.0019, |
| "reward": 1.8995909690856934, |
| "reward_std": 0.015642177313566208, |
| "rewards/accuracy_reward": 0.8995909094810486, |
| "rewards/format_reward": 1.0, |
| "step": 457 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.81731414794922, |
| "epoch": 6.735294117647059, |
| "grad_norm": 1.5989334696664628, |
| "kl": 0.05224609375, |
| "learning_rate": 6.63235294117647e-07, |
| "loss": 0.0021, |
| "reward": 1.8981314897537231, |
| "reward_std": 0.020380795001983643, |
| "rewards/accuracy_reward": 0.8981314897537231, |
| "rewards/format_reward": 1.0, |
| "step": 458 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.00841522216797, |
| "epoch": 6.75, |
| "grad_norm": 3.993916385712523, |
| "kl": 0.045166015625, |
| "learning_rate": 6.624999999999999e-07, |
| "loss": 0.0018, |
| "reward": 1.9087324142456055, |
| "reward_std": 0.018654441460967064, |
| "rewards/accuracy_reward": 0.9087323546409607, |
| "rewards/format_reward": 1.0, |
| "step": 459 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.8689956665039, |
| "epoch": 6.764705882352941, |
| "grad_norm": 5.917555869011424, |
| "kl": 0.0458984375, |
| "learning_rate": 6.617647058823529e-07, |
| "loss": 0.0018, |
| "reward": 1.9015119075775146, |
| "reward_std": 0.01625223271548748, |
| "rewards/accuracy_reward": 0.9015120267868042, |
| "rewards/format_reward": 1.0, |
| "step": 460 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.99639892578125, |
| "epoch": 6.779411764705882, |
| "grad_norm": 2.4437828911603607, |
| "kl": 0.047607421875, |
| "learning_rate": 6.610294117647059e-07, |
| "loss": 0.0019, |
| "reward": 1.8872531652450562, |
| "reward_std": 0.020600629970431328, |
| "rewards/accuracy_reward": 0.8872530460357666, |
| "rewards/format_reward": 1.0, |
| "step": 461 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.85456848144531, |
| "epoch": 6.794117647058823, |
| "grad_norm": 1.220750465871548, |
| "kl": 0.054931640625, |
| "learning_rate": 6.602941176470587e-07, |
| "loss": 0.0022, |
| "reward": 1.902260661125183, |
| "reward_std": 0.020439889281988144, |
| "rewards/accuracy_reward": 0.9034625291824341, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 462 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.22116088867188, |
| "epoch": 6.8088235294117645, |
| "grad_norm": 2.3522146924761804, |
| "kl": 0.048583984375, |
| "learning_rate": 6.595588235294117e-07, |
| "loss": 0.0019, |
| "reward": 1.8962863683700562, |
| "reward_std": 0.0212849248200655, |
| "rewards/accuracy_reward": 0.8962863087654114, |
| "rewards/format_reward": 1.0, |
| "step": 463 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.49038696289062, |
| "epoch": 6.823529411764706, |
| "grad_norm": 2.653649647480221, |
| "kl": 0.054931640625, |
| "learning_rate": 6.588235294117647e-07, |
| "loss": 0.0022, |
| "reward": 1.907329797744751, |
| "reward_std": 0.01719941571354866, |
| "rewards/accuracy_reward": 0.9073297381401062, |
| "rewards/format_reward": 1.0, |
| "step": 464 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.78606414794922, |
| "epoch": 6.838235294117647, |
| "grad_norm": 1.5113941947142104, |
| "kl": 0.07421875, |
| "learning_rate": 6.580882352941176e-07, |
| "loss": 0.003, |
| "reward": 1.905678153038025, |
| "reward_std": 0.027055159211158752, |
| "rewards/accuracy_reward": 0.9092837572097778, |
| "rewards/format_reward": 0.9963942766189575, |
| "step": 465 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.69831848144531, |
| "epoch": 6.852941176470588, |
| "grad_norm": 2.845621962915759, |
| "kl": 0.054443359375, |
| "learning_rate": 6.573529411764705e-07, |
| "loss": 0.0022, |
| "reward": 1.900158405303955, |
| "reward_std": 0.0176254715770483, |
| "rewards/accuracy_reward": 0.9001582264900208, |
| "rewards/format_reward": 1.0, |
| "step": 466 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.25721740722656, |
| "epoch": 6.867647058823529, |
| "grad_norm": 1.7461784259949287, |
| "kl": 0.060546875, |
| "learning_rate": 6.566176470588235e-07, |
| "loss": 0.0024, |
| "reward": 1.9130123853683472, |
| "reward_std": 0.018750464543700218, |
| "rewards/accuracy_reward": 0.9130123257637024, |
| "rewards/format_reward": 1.0, |
| "step": 467 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.20433044433594, |
| "epoch": 6.882352941176471, |
| "grad_norm": 2.1022701427819346, |
| "kl": 0.051025390625, |
| "learning_rate": 6.558823529411764e-07, |
| "loss": 0.002, |
| "reward": 1.9070162773132324, |
| "reward_std": 0.01999017596244812, |
| "rewards/accuracy_reward": 0.9070160388946533, |
| "rewards/format_reward": 1.0, |
| "step": 468 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.2596206665039, |
| "epoch": 6.897058823529412, |
| "grad_norm": 1.9884301824289226, |
| "kl": 0.05322265625, |
| "learning_rate": 6.551470588235294e-07, |
| "loss": 0.0021, |
| "reward": 1.8830878734588623, |
| "reward_std": 0.025470944121479988, |
| "rewards/accuracy_reward": 0.884289562702179, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 469 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.24639892578125, |
| "epoch": 6.911764705882353, |
| "grad_norm": 2.363006531050007, |
| "kl": 0.060791015625, |
| "learning_rate": 6.544117647058824e-07, |
| "loss": 0.0024, |
| "reward": 1.899465799331665, |
| "reward_std": 0.018820466473698616, |
| "rewards/accuracy_reward": 0.8994657397270203, |
| "rewards/format_reward": 1.0, |
| "step": 470 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.1189956665039, |
| "epoch": 6.926470588235294, |
| "grad_norm": 1.955011720116878, |
| "kl": 0.053466796875, |
| "learning_rate": 6.536764705882352e-07, |
| "loss": 0.0021, |
| "reward": 1.9155514240264893, |
| "reward_std": 0.017393523827195168, |
| "rewards/accuracy_reward": 0.9155513048171997, |
| "rewards/format_reward": 1.0, |
| "step": 471 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.6971206665039, |
| "epoch": 6.9411764705882355, |
| "grad_norm": 1.9269477392518999, |
| "kl": 0.050048828125, |
| "learning_rate": 6.529411764705882e-07, |
| "loss": 0.002, |
| "reward": 1.8984498977661133, |
| "reward_std": 0.019148575142025948, |
| "rewards/accuracy_reward": 0.8996517658233643, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 472 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.8221206665039, |
| "epoch": 6.955882352941177, |
| "grad_norm": 2.9581301695718514, |
| "kl": 0.0537109375, |
| "learning_rate": 6.522058823529412e-07, |
| "loss": 0.0021, |
| "reward": 1.8932065963745117, |
| "reward_std": 0.0210660882294178, |
| "rewards/accuracy_reward": 0.8932065367698669, |
| "rewards/format_reward": 1.0, |
| "step": 473 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.91947174072266, |
| "epoch": 6.970588235294118, |
| "grad_norm": 2.111290033794273, |
| "kl": 0.059326171875, |
| "learning_rate": 6.514705882352941e-07, |
| "loss": 0.0024, |
| "reward": 1.901350498199463, |
| "reward_std": 0.021692074835300446, |
| "rewards/accuracy_reward": 0.9025523662567139, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 474 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.16947174072266, |
| "epoch": 6.985294117647059, |
| "grad_norm": 2.2057304416622108, |
| "kl": 0.060791015625, |
| "learning_rate": 6.50735294117647e-07, |
| "loss": 0.0024, |
| "reward": 1.9076955318450928, |
| "reward_std": 0.021920649334788322, |
| "rewards/accuracy_reward": 0.9088975191116333, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 475 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.17960357666016, |
| "epoch": 7.0, |
| "grad_norm": 1.617597906920766, |
| "kl": 0.056396484375, |
| "learning_rate": 6.5e-07, |
| "loss": 0.0023, |
| "reward": 1.9078389406204224, |
| "reward_std": 0.023063138127326965, |
| "rewards/accuracy_reward": 0.9078390598297119, |
| "rewards/format_reward": 1.0, |
| "step": 476 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.60216522216797, |
| "epoch": 7.014705882352941, |
| "grad_norm": 1.6817279506404212, |
| "kl": 0.0546875, |
| "learning_rate": 6.492647058823529e-07, |
| "loss": 0.0022, |
| "reward": 1.9104756116867065, |
| "reward_std": 0.020941542461514473, |
| "rewards/accuracy_reward": 0.9116773009300232, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 477 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.2752456665039, |
| "epoch": 7.029411764705882, |
| "grad_norm": 2.3781369604321836, |
| "kl": 0.052490234375, |
| "learning_rate": 6.485294117647059e-07, |
| "loss": 0.0021, |
| "reward": 1.8896749019622803, |
| "reward_std": 0.020192930474877357, |
| "rewards/accuracy_reward": 0.8896746635437012, |
| "rewards/format_reward": 1.0, |
| "step": 478 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.77163696289062, |
| "epoch": 7.044117647058823, |
| "grad_norm": 1.276439509093276, |
| "kl": 0.059326171875, |
| "learning_rate": 6.477941176470589e-07, |
| "loss": 0.0024, |
| "reward": 1.8859210014343262, |
| "reward_std": 0.022135620936751366, |
| "rewards/accuracy_reward": 0.8859207630157471, |
| "rewards/format_reward": 1.0, |
| "step": 479 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.27163696289062, |
| "epoch": 7.0588235294117645, |
| "grad_norm": 2.4358423376131815, |
| "kl": 0.047607421875, |
| "learning_rate": 6.470588235294117e-07, |
| "loss": 0.0019, |
| "reward": 1.8972910642623901, |
| "reward_std": 0.020838702097535133, |
| "rewards/accuracy_reward": 0.8972910642623901, |
| "rewards/format_reward": 1.0, |
| "step": 480 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.93991088867188, |
| "epoch": 7.073529411764706, |
| "grad_norm": 2.4539112139463777, |
| "kl": 0.05078125, |
| "learning_rate": 6.463235294117647e-07, |
| "loss": 0.002, |
| "reward": 1.8896875381469727, |
| "reward_std": 0.0205745380371809, |
| "rewards/accuracy_reward": 0.8896873593330383, |
| "rewards/format_reward": 1.0, |
| "step": 481 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.85697174072266, |
| "epoch": 7.088235294117647, |
| "grad_norm": 1.5146047739430706, |
| "kl": 0.054931640625, |
| "learning_rate": 6.455882352941177e-07, |
| "loss": 0.0022, |
| "reward": 1.9029262065887451, |
| "reward_std": 0.020332474261522293, |
| "rewards/accuracy_reward": 0.9029260873794556, |
| "rewards/format_reward": 1.0, |
| "step": 482 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.64543914794922, |
| "epoch": 7.102941176470588, |
| "grad_norm": 1.6263090300209622, |
| "kl": 0.049560546875, |
| "learning_rate": 6.448529411764706e-07, |
| "loss": 0.002, |
| "reward": 1.9050016403198242, |
| "reward_std": 0.020645473152399063, |
| "rewards/accuracy_reward": 0.9050015807151794, |
| "rewards/format_reward": 1.0, |
| "step": 483 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.32452392578125, |
| "epoch": 7.117647058823529, |
| "grad_norm": 2.386969995018498, |
| "kl": 0.04833984375, |
| "learning_rate": 6.441176470588235e-07, |
| "loss": 0.0019, |
| "reward": 1.8980265855789185, |
| "reward_std": 0.02636340633034706, |
| "rewards/accuracy_reward": 0.9016323685646057, |
| "rewards/format_reward": 0.9963942766189575, |
| "step": 484 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.62741088867188, |
| "epoch": 7.132352941176471, |
| "grad_norm": 2.488441055097933, |
| "kl": 0.060302734375, |
| "learning_rate": 6.433823529411765e-07, |
| "loss": 0.0024, |
| "reward": 1.8940205574035645, |
| "reward_std": 0.021218711510300636, |
| "rewards/accuracy_reward": 0.8940205574035645, |
| "rewards/format_reward": 1.0, |
| "step": 485 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.08173370361328, |
| "epoch": 7.147058823529412, |
| "grad_norm": 1.4885294609323232, |
| "kl": 0.055419921875, |
| "learning_rate": 6.426470588235294e-07, |
| "loss": 0.0022, |
| "reward": 1.9081767797470093, |
| "reward_std": 0.021031972020864487, |
| "rewards/accuracy_reward": 0.9093785881996155, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 486 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 114.18029022216797, |
| "epoch": 7.161764705882353, |
| "grad_norm": 2.3558483279167257, |
| "kl": 0.050048828125, |
| "learning_rate": 6.419117647058824e-07, |
| "loss": 0.002, |
| "reward": 1.9089596271514893, |
| "reward_std": 0.021060261875391006, |
| "rewards/accuracy_reward": 0.910161554813385, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 487 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.5300521850586, |
| "epoch": 7.176470588235294, |
| "grad_norm": 2.185868565772857, |
| "kl": 0.051513671875, |
| "learning_rate": 6.411764705882354e-07, |
| "loss": 0.002, |
| "reward": 1.895540475845337, |
| "reward_std": 0.02588125877082348, |
| "rewards/accuracy_reward": 0.8991460204124451, |
| "rewards/format_reward": 0.9963942766189575, |
| "step": 488 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.70313262939453, |
| "epoch": 7.1911764705882355, |
| "grad_norm": 13.781167768252143, |
| "kl": 0.0556640625, |
| "learning_rate": 6.404411764705881e-07, |
| "loss": 0.0022, |
| "reward": 1.900436520576477, |
| "reward_std": 0.018973568454384804, |
| "rewards/accuracy_reward": 0.9004363417625427, |
| "rewards/format_reward": 1.0, |
| "step": 489 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.69231414794922, |
| "epoch": 7.205882352941177, |
| "grad_norm": 1.787139009281532, |
| "kl": 0.055908203125, |
| "learning_rate": 6.397058823529411e-07, |
| "loss": 0.0022, |
| "reward": 1.914228916168213, |
| "reward_std": 0.025157246738672256, |
| "rewards/accuracy_reward": 0.9166327714920044, |
| "rewards/format_reward": 0.9975962042808533, |
| "step": 490 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.53606414794922, |
| "epoch": 7.220588235294118, |
| "grad_norm": 1.3383135392600143, |
| "kl": 0.05322265625, |
| "learning_rate": 6.389705882352941e-07, |
| "loss": 0.0021, |
| "reward": 1.9147520065307617, |
| "reward_std": 0.01828881911933422, |
| "rewards/accuracy_reward": 0.9147518277168274, |
| "rewards/format_reward": 1.0, |
| "step": 491 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.80769348144531, |
| "epoch": 7.235294117647059, |
| "grad_norm": 1.499587887382779, |
| "kl": 0.049560546875, |
| "learning_rate": 6.38235294117647e-07, |
| "loss": 0.002, |
| "reward": 1.9006352424621582, |
| "reward_std": 0.021384483203291893, |
| "rewards/accuracy_reward": 0.901837170124054, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 492 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.85216522216797, |
| "epoch": 7.25, |
| "grad_norm": 9.51747929867123, |
| "kl": 0.052978515625, |
| "learning_rate": 6.374999999999999e-07, |
| "loss": 0.0021, |
| "reward": 1.9037039279937744, |
| "reward_std": 0.017083317041397095, |
| "rewards/accuracy_reward": 0.9037038683891296, |
| "rewards/format_reward": 1.0, |
| "step": 493 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.1863021850586, |
| "epoch": 7.264705882352941, |
| "grad_norm": 2.190318540866678, |
| "kl": 0.052978515625, |
| "learning_rate": 6.367647058823529e-07, |
| "loss": 0.0021, |
| "reward": 1.9090691804885864, |
| "reward_std": 0.016836341470479965, |
| "rewards/accuracy_reward": 0.9090691804885864, |
| "rewards/format_reward": 1.0, |
| "step": 494 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.6658706665039, |
| "epoch": 7.279411764705882, |
| "grad_norm": 16.171603319545223, |
| "kl": 0.0576171875, |
| "learning_rate": 6.360294117647058e-07, |
| "loss": 0.0023, |
| "reward": 1.8776507377624512, |
| "reward_std": 0.018403498455882072, |
| "rewards/accuracy_reward": 0.8776505589485168, |
| "rewards/format_reward": 1.0, |
| "step": 495 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.74519348144531, |
| "epoch": 7.294117647058823, |
| "grad_norm": 1.9202223901107958, |
| "kl": 0.054931640625, |
| "learning_rate": 6.352941176470588e-07, |
| "loss": 0.0022, |
| "reward": 1.883994698524475, |
| "reward_std": 0.018577104434370995, |
| "rewards/accuracy_reward": 0.8839945197105408, |
| "rewards/format_reward": 1.0, |
| "step": 496 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.7752456665039, |
| "epoch": 7.3088235294117645, |
| "grad_norm": 2.7194904114682545, |
| "kl": 0.05908203125, |
| "learning_rate": 6.345588235294117e-07, |
| "loss": 0.0024, |
| "reward": 1.9002341032028198, |
| "reward_std": 0.02195299230515957, |
| "rewards/accuracy_reward": 0.9026379585266113, |
| "rewards/format_reward": 0.9975962042808533, |
| "step": 497 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.41106414794922, |
| "epoch": 7.323529411764706, |
| "grad_norm": 2.818503081927554, |
| "kl": 0.0546875, |
| "learning_rate": 6.338235294117646e-07, |
| "loss": 0.0022, |
| "reward": 1.9031240940093994, |
| "reward_std": 0.021893031895160675, |
| "rewards/accuracy_reward": 0.9055278897285461, |
| "rewards/format_reward": 0.9975962042808533, |
| "step": 498 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.1658706665039, |
| "epoch": 7.338235294117647, |
| "grad_norm": 2.2904375345608683, |
| "kl": 0.054443359375, |
| "learning_rate": 6.330882352941176e-07, |
| "loss": 0.0022, |
| "reward": 1.8997858762741089, |
| "reward_std": 0.01683872379362583, |
| "rewards/accuracy_reward": 0.8997858762741089, |
| "rewards/format_reward": 1.0, |
| "step": 499 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.5877456665039, |
| "epoch": 7.352941176470588, |
| "grad_norm": 1.9874226919695666, |
| "kl": 0.05712890625, |
| "learning_rate": 6.323529411764706e-07, |
| "loss": 0.0023, |
| "reward": 1.9213850498199463, |
| "reward_std": 0.016001444309949875, |
| "rewards/accuracy_reward": 0.9213849306106567, |
| "rewards/format_reward": 1.0, |
| "step": 500 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.45433044433594, |
| "epoch": 7.367647058823529, |
| "grad_norm": 1.4810061564924804, |
| "kl": 0.06005859375, |
| "learning_rate": 6.316176470588234e-07, |
| "loss": 0.0024, |
| "reward": 1.9132663011550903, |
| "reward_std": 0.016456911340355873, |
| "rewards/accuracy_reward": 0.9132663011550903, |
| "rewards/format_reward": 1.0, |
| "step": 501 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.87259674072266, |
| "epoch": 7.382352941176471, |
| "grad_norm": 2.354230850080614, |
| "kl": 0.0537109375, |
| "learning_rate": 6.308823529411764e-07, |
| "loss": 0.0021, |
| "reward": 1.9061925411224365, |
| "reward_std": 0.015117881819605827, |
| "rewards/accuracy_reward": 0.906192421913147, |
| "rewards/format_reward": 1.0, |
| "step": 502 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.88221740722656, |
| "epoch": 7.397058823529412, |
| "grad_norm": 2.702198636610281, |
| "kl": 0.055908203125, |
| "learning_rate": 6.301470588235294e-07, |
| "loss": 0.0023, |
| "reward": 1.8956823348999023, |
| "reward_std": 0.017957165837287903, |
| "rewards/accuracy_reward": 0.8956822752952576, |
| "rewards/format_reward": 1.0, |
| "step": 503 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.91827392578125, |
| "epoch": 7.411764705882353, |
| "grad_norm": 2.707910090355551, |
| "kl": 0.05517578125, |
| "learning_rate": 6.294117647058823e-07, |
| "loss": 0.0022, |
| "reward": 1.8958656787872314, |
| "reward_std": 0.01692720130085945, |
| "rewards/accuracy_reward": 0.8958656787872314, |
| "rewards/format_reward": 1.0, |
| "step": 504 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.41106414794922, |
| "epoch": 7.426470588235294, |
| "grad_norm": 2.732018255754022, |
| "kl": 0.06884765625, |
| "learning_rate": 6.286764705882353e-07, |
| "loss": 0.0028, |
| "reward": 1.907556176185608, |
| "reward_std": 0.016339732334017754, |
| "rewards/accuracy_reward": 0.9075562953948975, |
| "rewards/format_reward": 1.0, |
| "step": 505 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.5252456665039, |
| "epoch": 7.4411764705882355, |
| "grad_norm": 1.9561974612826654, |
| "kl": 0.061279296875, |
| "learning_rate": 6.279411764705882e-07, |
| "loss": 0.0025, |
| "reward": 1.9172346591949463, |
| "reward_std": 0.017858320847153664, |
| "rewards/accuracy_reward": 0.9184365272521973, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 506 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.40745544433594, |
| "epoch": 7.455882352941177, |
| "grad_norm": 4.587975830736582, |
| "kl": 0.07080078125, |
| "learning_rate": 6.272058823529411e-07, |
| "loss": 0.0028, |
| "reward": 1.9027947187423706, |
| "reward_std": 0.014803332276642323, |
| "rewards/accuracy_reward": 0.9027946591377258, |
| "rewards/format_reward": 1.0, |
| "step": 507 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.92909240722656, |
| "epoch": 7.470588235294118, |
| "grad_norm": 3.0606242226177085, |
| "kl": 0.05859375, |
| "learning_rate": 6.264705882352941e-07, |
| "loss": 0.0023, |
| "reward": 1.9170668125152588, |
| "reward_std": 0.01804046705365181, |
| "rewards/accuracy_reward": 0.9182686805725098, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 508 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.75000762939453, |
| "epoch": 7.485294117647059, |
| "grad_norm": 1.565766574508637, |
| "kl": 0.06005859375, |
| "learning_rate": 6.257352941176471e-07, |
| "loss": 0.0024, |
| "reward": 1.9129410982131958, |
| "reward_std": 0.016586054116487503, |
| "rewards/accuracy_reward": 0.914142906665802, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 509 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.11659240722656, |
| "epoch": 7.5, |
| "grad_norm": 2.188031838473879, |
| "kl": 0.06494140625, |
| "learning_rate": 6.249999999999999e-07, |
| "loss": 0.0026, |
| "reward": 1.9109857082366943, |
| "reward_std": 0.01680620200932026, |
| "rewards/accuracy_reward": 0.9109857082366943, |
| "rewards/format_reward": 1.0, |
| "step": 510 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.4206771850586, |
| "epoch": 7.514705882352941, |
| "grad_norm": 2.301709034695236, |
| "kl": 0.05419921875, |
| "learning_rate": 6.242647058823529e-07, |
| "loss": 0.0022, |
| "reward": 1.8962676525115967, |
| "reward_std": 0.01672891154885292, |
| "rewards/accuracy_reward": 0.8962674736976624, |
| "rewards/format_reward": 1.0, |
| "step": 511 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.53726196289062, |
| "epoch": 7.529411764705882, |
| "grad_norm": 2.3427553949456135, |
| "kl": 0.057373046875, |
| "learning_rate": 6.235294117647059e-07, |
| "loss": 0.0023, |
| "reward": 1.9040287733078003, |
| "reward_std": 0.017709048464894295, |
| "rewards/accuracy_reward": 0.9040287137031555, |
| "rewards/format_reward": 1.0, |
| "step": 512 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.12019348144531, |
| "epoch": 7.544117647058823, |
| "grad_norm": 1.665426870174403, |
| "kl": 0.0537109375, |
| "learning_rate": 6.227941176470588e-07, |
| "loss": 0.0022, |
| "reward": 1.88170325756073, |
| "reward_std": 0.017501268535852432, |
| "rewards/accuracy_reward": 0.8817031979560852, |
| "rewards/format_reward": 1.0, |
| "step": 513 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.15144348144531, |
| "epoch": 7.5588235294117645, |
| "grad_norm": 1.6488965895634309, |
| "kl": 0.06591796875, |
| "learning_rate": 6.220588235294118e-07, |
| "loss": 0.0026, |
| "reward": 1.8925837278366089, |
| "reward_std": 0.015000036917626858, |
| "rewards/accuracy_reward": 0.8925836682319641, |
| "rewards/format_reward": 1.0, |
| "step": 514 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.16106414794922, |
| "epoch": 7.573529411764706, |
| "grad_norm": 1.5981543745714246, |
| "kl": 0.05517578125, |
| "learning_rate": 6.213235294117647e-07, |
| "loss": 0.0022, |
| "reward": 1.920210599899292, |
| "reward_std": 0.01567223109304905, |
| "rewards/accuracy_reward": 0.9202104806900024, |
| "rewards/format_reward": 1.0, |
| "step": 515 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.27764892578125, |
| "epoch": 7.588235294117647, |
| "grad_norm": 1.914576473194542, |
| "kl": 0.054931640625, |
| "learning_rate": 6.205882352941176e-07, |
| "loss": 0.0022, |
| "reward": 1.9109389781951904, |
| "reward_std": 0.01758970133960247, |
| "rewards/accuracy_reward": 0.9121409058570862, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 516 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.74639892578125, |
| "epoch": 7.602941176470588, |
| "grad_norm": 2.2056419921026444, |
| "kl": 0.06591796875, |
| "learning_rate": 6.198529411764706e-07, |
| "loss": 0.0026, |
| "reward": 1.909173846244812, |
| "reward_std": 0.0163656584918499, |
| "rewards/accuracy_reward": 0.909173846244812, |
| "rewards/format_reward": 1.0, |
| "step": 517 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.97956848144531, |
| "epoch": 7.617647058823529, |
| "grad_norm": 2.3280532068329505, |
| "kl": 0.0517578125, |
| "learning_rate": 6.191176470588236e-07, |
| "loss": 0.0021, |
| "reward": 1.8901021480560303, |
| "reward_std": 0.01763245277106762, |
| "rewards/accuracy_reward": 0.890101969242096, |
| "rewards/format_reward": 1.0, |
| "step": 518 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.45793914794922, |
| "epoch": 7.632352941176471, |
| "grad_norm": 1.402338629602267, |
| "kl": 0.05712890625, |
| "learning_rate": 6.183823529411764e-07, |
| "loss": 0.0023, |
| "reward": 1.9023847579956055, |
| "reward_std": 0.016608012840151787, |
| "rewards/accuracy_reward": 0.9023846387863159, |
| "rewards/format_reward": 1.0, |
| "step": 519 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.38341522216797, |
| "epoch": 7.647058823529412, |
| "grad_norm": 1.6835653034818958, |
| "kl": 0.058837890625, |
| "learning_rate": 6.176470588235294e-07, |
| "loss": 0.0024, |
| "reward": 1.8985044956207275, |
| "reward_std": 0.01695878617465496, |
| "rewards/accuracy_reward": 0.8985044360160828, |
| "rewards/format_reward": 1.0, |
| "step": 520 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.1346206665039, |
| "epoch": 7.661764705882353, |
| "grad_norm": 9.541516490873528, |
| "kl": 0.05419921875, |
| "learning_rate": 6.169117647058824e-07, |
| "loss": 0.0022, |
| "reward": 1.903184175491333, |
| "reward_std": 0.01812390610575676, |
| "rewards/accuracy_reward": 0.9031841158866882, |
| "rewards/format_reward": 1.0, |
| "step": 521 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.78125762939453, |
| "epoch": 7.676470588235294, |
| "grad_norm": 2.907795347878127, |
| "kl": 0.056396484375, |
| "learning_rate": 6.161764705882353e-07, |
| "loss": 0.0022, |
| "reward": 1.9058642387390137, |
| "reward_std": 0.022106477990746498, |
| "rewards/accuracy_reward": 0.9070661067962646, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 522 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.3425521850586, |
| "epoch": 7.6911764705882355, |
| "grad_norm": 2.122588959449984, |
| "kl": 0.047607421875, |
| "learning_rate": 6.154411764705883e-07, |
| "loss": 0.0019, |
| "reward": 1.892153024673462, |
| "reward_std": 0.017479056492447853, |
| "rewards/accuracy_reward": 0.8921528458595276, |
| "rewards/format_reward": 1.0, |
| "step": 523 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.2019271850586, |
| "epoch": 7.705882352941177, |
| "grad_norm": 2.3564295353752986, |
| "kl": 0.05615234375, |
| "learning_rate": 6.147058823529412e-07, |
| "loss": 0.0022, |
| "reward": 1.9090198278427124, |
| "reward_std": 0.01690274477005005, |
| "rewards/accuracy_reward": 0.9090198874473572, |
| "rewards/format_reward": 1.0, |
| "step": 524 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.92188262939453, |
| "epoch": 7.720588235294118, |
| "grad_norm": 1.574872602292037, |
| "kl": 0.055419921875, |
| "learning_rate": 6.139705882352941e-07, |
| "loss": 0.0022, |
| "reward": 1.901806354522705, |
| "reward_std": 0.020534634590148926, |
| "rewards/accuracy_reward": 0.9030082821846008, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 525 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.71634674072266, |
| "epoch": 7.735294117647059, |
| "grad_norm": 1.4642975468665929, |
| "kl": 0.05615234375, |
| "learning_rate": 6.132352941176471e-07, |
| "loss": 0.0022, |
| "reward": 1.8870961666107178, |
| "reward_std": 0.0190273430198431, |
| "rewards/accuracy_reward": 0.8870960474014282, |
| "rewards/format_reward": 1.0, |
| "step": 526 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.76683044433594, |
| "epoch": 7.75, |
| "grad_norm": 1.64957668506362, |
| "kl": 0.055908203125, |
| "learning_rate": 6.125000000000001e-07, |
| "loss": 0.0022, |
| "reward": 1.888306736946106, |
| "reward_std": 0.018305247649550438, |
| "rewards/accuracy_reward": 0.888306736946106, |
| "rewards/format_reward": 1.0, |
| "step": 527 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.36659240722656, |
| "epoch": 7.764705882352941, |
| "grad_norm": 1.8470414599040466, |
| "kl": 0.054443359375, |
| "learning_rate": 6.117647058823529e-07, |
| "loss": 0.0022, |
| "reward": 1.914186716079712, |
| "reward_std": 0.018072694540023804, |
| "rewards/accuracy_reward": 0.9141866564750671, |
| "rewards/format_reward": 1.0, |
| "step": 528 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.49279022216797, |
| "epoch": 7.779411764705882, |
| "grad_norm": 1.4757794429532862, |
| "kl": 0.05419921875, |
| "learning_rate": 6.110294117647059e-07, |
| "loss": 0.0022, |
| "reward": 1.8795055150985718, |
| "reward_std": 0.022909091785550117, |
| "rewards/accuracy_reward": 0.8807073831558228, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 529 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.40384674072266, |
| "epoch": 7.794117647058823, |
| "grad_norm": 2.0338636064103333, |
| "kl": 0.059326171875, |
| "learning_rate": 6.102941176470589e-07, |
| "loss": 0.0024, |
| "reward": 1.9119820594787598, |
| "reward_std": 0.01673622988164425, |
| "rewards/accuracy_reward": 0.9119820594787598, |
| "rewards/format_reward": 1.0, |
| "step": 530 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.04808044433594, |
| "epoch": 7.8088235294117645, |
| "grad_norm": 1.5646667587468694, |
| "kl": 0.0498046875, |
| "learning_rate": 6.095588235294118e-07, |
| "loss": 0.002, |
| "reward": 1.9045634269714355, |
| "reward_std": 0.019505947828292847, |
| "rewards/accuracy_reward": 0.9045633673667908, |
| "rewards/format_reward": 1.0, |
| "step": 531 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.84495544433594, |
| "epoch": 7.823529411764706, |
| "grad_norm": 1.9910418315918015, |
| "kl": 0.04736328125, |
| "learning_rate": 6.088235294117646e-07, |
| "loss": 0.0019, |
| "reward": 1.9242442846298218, |
| "reward_std": 0.015269236639142036, |
| "rewards/accuracy_reward": 0.9242441058158875, |
| "rewards/format_reward": 1.0, |
| "step": 532 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.67909240722656, |
| "epoch": 7.838235294117647, |
| "grad_norm": 2.1921636670867213, |
| "kl": 0.05126953125, |
| "learning_rate": 6.080882352941175e-07, |
| "loss": 0.0021, |
| "reward": 1.9174097776412964, |
| "reward_std": 0.016234705224633217, |
| "rewards/accuracy_reward": 0.9174097776412964, |
| "rewards/format_reward": 1.0, |
| "step": 533 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.95433044433594, |
| "epoch": 7.852941176470588, |
| "grad_norm": 5.773358773291786, |
| "kl": 0.052734375, |
| "learning_rate": 6.073529411764705e-07, |
| "loss": 0.0021, |
| "reward": 1.904146671295166, |
| "reward_std": 0.018309401348233223, |
| "rewards/accuracy_reward": 0.904146671295166, |
| "rewards/format_reward": 1.0, |
| "step": 534 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.10216522216797, |
| "epoch": 7.867647058823529, |
| "grad_norm": 3.1463228619890646, |
| "kl": 0.0498046875, |
| "learning_rate": 6.066176470588235e-07, |
| "loss": 0.002, |
| "reward": 1.9100762605667114, |
| "reward_std": 0.0184779055416584, |
| "rewards/accuracy_reward": 0.9100762009620667, |
| "rewards/format_reward": 1.0, |
| "step": 535 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.77644348144531, |
| "epoch": 7.882352941176471, |
| "grad_norm": 4.843243565505217, |
| "kl": 0.04736328125, |
| "learning_rate": 6.058823529411763e-07, |
| "loss": 0.0019, |
| "reward": 1.9202395677566528, |
| "reward_std": 0.0161683801561594, |
| "rewards/accuracy_reward": 0.9202395677566528, |
| "rewards/format_reward": 1.0, |
| "step": 536 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.99159240722656, |
| "epoch": 7.897058823529412, |
| "grad_norm": 1.5367043322463625, |
| "kl": 0.0546875, |
| "learning_rate": 6.051470588235293e-07, |
| "loss": 0.0022, |
| "reward": 1.9017425775527954, |
| "reward_std": 0.017361411824822426, |
| "rewards/accuracy_reward": 0.901742696762085, |
| "rewards/format_reward": 1.0, |
| "step": 537 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.72716522216797, |
| "epoch": 7.911764705882353, |
| "grad_norm": 1.6647667695224042, |
| "kl": 0.04736328125, |
| "learning_rate": 6.044117647058823e-07, |
| "loss": 0.0019, |
| "reward": 1.9160679578781128, |
| "reward_std": 0.01760725863277912, |
| "rewards/accuracy_reward": 0.9160678386688232, |
| "rewards/format_reward": 1.0, |
| "step": 538 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.06971740722656, |
| "epoch": 7.926470588235294, |
| "grad_norm": 1.5196339741547849, |
| "kl": 0.04931640625, |
| "learning_rate": 6.036764705882352e-07, |
| "loss": 0.002, |
| "reward": 1.9151060581207275, |
| "reward_std": 0.017051585018634796, |
| "rewards/accuracy_reward": 0.9151060581207275, |
| "rewards/format_reward": 1.0, |
| "step": 539 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.31971740722656, |
| "epoch": 7.9411764705882355, |
| "grad_norm": 2.094866774315946, |
| "kl": 0.04736328125, |
| "learning_rate": 6.029411764705882e-07, |
| "loss": 0.0019, |
| "reward": 1.8982237577438354, |
| "reward_std": 0.017682263627648354, |
| "rewards/accuracy_reward": 0.8982236981391907, |
| "rewards/format_reward": 1.0, |
| "step": 540 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.39543914794922, |
| "epoch": 7.955882352941177, |
| "grad_norm": 3.260878136220598, |
| "kl": 0.04638671875, |
| "learning_rate": 6.022058823529411e-07, |
| "loss": 0.0019, |
| "reward": 1.8989781141281128, |
| "reward_std": 0.016665568575263023, |
| "rewards/accuracy_reward": 0.8989779949188232, |
| "rewards/format_reward": 1.0, |
| "step": 541 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.65625762939453, |
| "epoch": 7.970588235294118, |
| "grad_norm": 2.6615474581638856, |
| "kl": 0.05126953125, |
| "learning_rate": 6.01470588235294e-07, |
| "loss": 0.002, |
| "reward": 1.9191153049468994, |
| "reward_std": 0.016191896051168442, |
| "rewards/accuracy_reward": 0.9191152453422546, |
| "rewards/format_reward": 1.0, |
| "step": 542 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.72116088867188, |
| "epoch": 7.985294117647059, |
| "grad_norm": 2.0899244271104815, |
| "kl": 0.05322265625, |
| "learning_rate": 6.00735294117647e-07, |
| "loss": 0.0021, |
| "reward": 1.9136680364608765, |
| "reward_std": 0.017099877819418907, |
| "rewards/accuracy_reward": 0.9136679768562317, |
| "rewards/format_reward": 1.0, |
| "step": 543 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.3500747680664, |
| "epoch": 8.0, |
| "grad_norm": 3.3285479491467647, |
| "kl": 0.050048828125, |
| "learning_rate": 6e-07, |
| "loss": 0.002, |
| "reward": 1.921190857887268, |
| "reward_std": 0.019811777397990227, |
| "rewards/accuracy_reward": 0.9211909174919128, |
| "rewards/format_reward": 1.0, |
| "step": 544 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.65625762939453, |
| "epoch": 8.014705882352942, |
| "grad_norm": 2.8131573068708047, |
| "kl": 0.0478515625, |
| "learning_rate": 5.992647058823528e-07, |
| "loss": 0.0019, |
| "reward": 1.9018555879592896, |
| "reward_std": 0.017298076301813126, |
| "rewards/accuracy_reward": 0.9018555879592896, |
| "rewards/format_reward": 1.0, |
| "step": 545 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.14423370361328, |
| "epoch": 8.029411764705882, |
| "grad_norm": 1.7926326341266048, |
| "kl": 0.048583984375, |
| "learning_rate": 5.985294117647058e-07, |
| "loss": 0.0019, |
| "reward": 1.902543544769287, |
| "reward_std": 0.018634334206581116, |
| "rewards/accuracy_reward": 0.9025435447692871, |
| "rewards/format_reward": 1.0, |
| "step": 546 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.56370544433594, |
| "epoch": 8.044117647058824, |
| "grad_norm": 1.4449179097965148, |
| "kl": 0.048583984375, |
| "learning_rate": 5.977941176470588e-07, |
| "loss": 0.0019, |
| "reward": 1.916272759437561, |
| "reward_std": 0.01657208427786827, |
| "rewards/accuracy_reward": 0.916272759437561, |
| "rewards/format_reward": 1.0, |
| "step": 547 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.97356414794922, |
| "epoch": 8.058823529411764, |
| "grad_norm": 6.756394605908286, |
| "kl": 0.05126953125, |
| "learning_rate": 5.970588235294117e-07, |
| "loss": 0.0021, |
| "reward": 1.9202964305877686, |
| "reward_std": 0.018207618966698647, |
| "rewards/accuracy_reward": 0.920296311378479, |
| "rewards/format_reward": 1.0, |
| "step": 548 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.40745544433594, |
| "epoch": 8.073529411764707, |
| "grad_norm": 3.5136091347598986, |
| "kl": 0.050537109375, |
| "learning_rate": 5.963235294117647e-07, |
| "loss": 0.002, |
| "reward": 1.9019232988357544, |
| "reward_std": 0.0198956411331892, |
| "rewards/accuracy_reward": 0.9019232392311096, |
| "rewards/format_reward": 1.0, |
| "step": 549 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.43991088867188, |
| "epoch": 8.088235294117647, |
| "grad_norm": 2.2839304418519792, |
| "kl": 0.052001953125, |
| "learning_rate": 5.955882352941176e-07, |
| "loss": 0.0021, |
| "reward": 1.9206691980361938, |
| "reward_std": 0.014682717621326447, |
| "rewards/accuracy_reward": 0.9206691384315491, |
| "rewards/format_reward": 1.0, |
| "step": 550 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.02043914794922, |
| "epoch": 8.102941176470589, |
| "grad_norm": 2.484567856847274, |
| "kl": 0.055419921875, |
| "learning_rate": 5.948529411764705e-07, |
| "loss": 0.0022, |
| "reward": 1.9100967645645142, |
| "reward_std": 0.018312010914087296, |
| "rewards/accuracy_reward": 0.9100965857505798, |
| "rewards/format_reward": 1.0, |
| "step": 551 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 114.34375762939453, |
| "epoch": 8.117647058823529, |
| "grad_norm": 1.6432899488733537, |
| "kl": 0.05419921875, |
| "learning_rate": 5.941176470588235e-07, |
| "loss": 0.0022, |
| "reward": 1.8991518020629883, |
| "reward_std": 0.021620534360408783, |
| "rewards/accuracy_reward": 0.900353729724884, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 552 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.63581848144531, |
| "epoch": 8.132352941176471, |
| "grad_norm": 2.0271204887130416, |
| "kl": 0.05712890625, |
| "learning_rate": 5.933823529411765e-07, |
| "loss": 0.0023, |
| "reward": 1.900107502937317, |
| "reward_std": 0.019040938466787338, |
| "rewards/accuracy_reward": 0.9001075029373169, |
| "rewards/format_reward": 1.0, |
| "step": 553 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 115.09616088867188, |
| "epoch": 8.147058823529411, |
| "grad_norm": 1.5813240400193374, |
| "kl": 0.060791015625, |
| "learning_rate": 5.926470588235293e-07, |
| "loss": 0.0024, |
| "reward": 1.915557622909546, |
| "reward_std": 0.020517727360129356, |
| "rewards/accuracy_reward": 0.9167595505714417, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 554 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 115.71394348144531, |
| "epoch": 8.161764705882353, |
| "grad_norm": 4.637148825036712, |
| "kl": 0.0576171875, |
| "learning_rate": 5.919117647058823e-07, |
| "loss": 0.0023, |
| "reward": 1.9027869701385498, |
| "reward_std": 0.019659575074911118, |
| "rewards/accuracy_reward": 0.9027867317199707, |
| "rewards/format_reward": 1.0, |
| "step": 555 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 114.27404022216797, |
| "epoch": 8.176470588235293, |
| "grad_norm": 1.6509059104235309, |
| "kl": 0.0556640625, |
| "learning_rate": 5.911764705882353e-07, |
| "loss": 0.0022, |
| "reward": 1.898353934288025, |
| "reward_std": 0.019414007663726807, |
| "rewards/accuracy_reward": 0.8983538746833801, |
| "rewards/format_reward": 1.0, |
| "step": 556 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 117.88101196289062, |
| "epoch": 8.191176470588236, |
| "grad_norm": 1.2731867179847607, |
| "kl": 0.0576171875, |
| "learning_rate": 5.904411764705882e-07, |
| "loss": 0.0023, |
| "reward": 1.8892360925674438, |
| "reward_std": 0.025776617228984833, |
| "rewards/accuracy_reward": 0.8916398882865906, |
| "rewards/format_reward": 0.9975962042808533, |
| "step": 557 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 115.42188262939453, |
| "epoch": 8.205882352941176, |
| "grad_norm": 2.6655315489926967, |
| "kl": 0.059326171875, |
| "learning_rate": 5.897058823529412e-07, |
| "loss": 0.0024, |
| "reward": 1.9022037982940674, |
| "reward_std": 0.01893787272274494, |
| "rewards/accuracy_reward": 0.9034057855606079, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 558 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 115.95793914794922, |
| "epoch": 8.220588235294118, |
| "grad_norm": 1.7443518844175923, |
| "kl": 0.053955078125, |
| "learning_rate": 5.889705882352941e-07, |
| "loss": 0.0022, |
| "reward": 1.9148838520050049, |
| "reward_std": 0.016492504626512527, |
| "rewards/accuracy_reward": 0.9148839116096497, |
| "rewards/format_reward": 1.0, |
| "step": 559 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 114.52043914794922, |
| "epoch": 8.235294117647058, |
| "grad_norm": 2.0265041434888853, |
| "kl": 0.050048828125, |
| "learning_rate": 5.88235294117647e-07, |
| "loss": 0.002, |
| "reward": 1.90397047996521, |
| "reward_std": 0.020762760192155838, |
| "rewards/accuracy_reward": 0.90397047996521, |
| "rewards/format_reward": 1.0, |
| "step": 560 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 116.28966522216797, |
| "epoch": 8.25, |
| "grad_norm": 1.5715106635469254, |
| "kl": 0.055419921875, |
| "learning_rate": 5.875e-07, |
| "loss": 0.0022, |
| "reward": 1.911298394203186, |
| "reward_std": 0.019145453348755836, |
| "rewards/accuracy_reward": 0.9112983345985413, |
| "rewards/format_reward": 1.0, |
| "step": 561 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.29808044433594, |
| "epoch": 8.264705882352942, |
| "grad_norm": 1.4932683274744363, |
| "kl": 0.057373046875, |
| "learning_rate": 5.86764705882353e-07, |
| "loss": 0.0023, |
| "reward": 1.9099814891815186, |
| "reward_std": 0.025356175377964973, |
| "rewards/accuracy_reward": 0.9123852252960205, |
| "rewards/format_reward": 0.9975962042808533, |
| "step": 562 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.23197174072266, |
| "epoch": 8.279411764705882, |
| "grad_norm": 7.280740226777266, |
| "kl": 0.055419921875, |
| "learning_rate": 5.860294117647058e-07, |
| "loss": 0.0022, |
| "reward": 1.9085097312927246, |
| "reward_std": 0.018148187547922134, |
| "rewards/accuracy_reward": 0.9085096716880798, |
| "rewards/format_reward": 1.0, |
| "step": 563 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.80048370361328, |
| "epoch": 8.294117647058824, |
| "grad_norm": 2.9076060047905163, |
| "kl": 0.05419921875, |
| "learning_rate": 5.852941176470588e-07, |
| "loss": 0.0022, |
| "reward": 1.909620761871338, |
| "reward_std": 0.018346594646573067, |
| "rewards/accuracy_reward": 0.9096207618713379, |
| "rewards/format_reward": 1.0, |
| "step": 564 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.26081848144531, |
| "epoch": 8.308823529411764, |
| "grad_norm": 3.2289837306279168, |
| "kl": 0.060791015625, |
| "learning_rate": 5.845588235294118e-07, |
| "loss": 0.0024, |
| "reward": 1.9069983959197998, |
| "reward_std": 0.0201736968010664, |
| "rewards/accuracy_reward": 0.9082001447677612, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 565 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.25120544433594, |
| "epoch": 8.323529411764707, |
| "grad_norm": 1.7199349309033736, |
| "kl": 0.05517578125, |
| "learning_rate": 5.838235294117647e-07, |
| "loss": 0.0022, |
| "reward": 1.887500286102295, |
| "reward_std": 0.020135780796408653, |
| "rewards/accuracy_reward": 0.8887021541595459, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 566 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.41226196289062, |
| "epoch": 8.338235294117647, |
| "grad_norm": 6.802575033519044, |
| "kl": 0.0576171875, |
| "learning_rate": 5.830882352941177e-07, |
| "loss": 0.0023, |
| "reward": 1.8974536657333374, |
| "reward_std": 0.020134514197707176, |
| "rewards/accuracy_reward": 0.8974535465240479, |
| "rewards/format_reward": 1.0, |
| "step": 567 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.51683044433594, |
| "epoch": 8.352941176470589, |
| "grad_norm": 36.18405306346145, |
| "kl": 0.054931640625, |
| "learning_rate": 5.823529411764706e-07, |
| "loss": 0.0022, |
| "reward": 1.8917019367218018, |
| "reward_std": 0.01857663132250309, |
| "rewards/accuracy_reward": 0.8929038643836975, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 568 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.5769271850586, |
| "epoch": 8.367647058823529, |
| "grad_norm": 2.3459848810177975, |
| "kl": 0.052734375, |
| "learning_rate": 5.816176470588235e-07, |
| "loss": 0.0021, |
| "reward": 1.9242991209030151, |
| "reward_std": 0.018164673820137978, |
| "rewards/accuracy_reward": 0.9255009293556213, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 569 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.03485870361328, |
| "epoch": 8.382352941176471, |
| "grad_norm": 1.6032225663505273, |
| "kl": 0.06103515625, |
| "learning_rate": 5.808823529411765e-07, |
| "loss": 0.0025, |
| "reward": 1.9081476926803589, |
| "reward_std": 0.019294267520308495, |
| "rewards/accuracy_reward": 0.9081474542617798, |
| "rewards/format_reward": 1.0, |
| "step": 570 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.40866088867188, |
| "epoch": 8.397058823529411, |
| "grad_norm": 2.653883841862032, |
| "kl": 0.05908203125, |
| "learning_rate": 5.801470588235295e-07, |
| "loss": 0.0024, |
| "reward": 1.904270052909851, |
| "reward_std": 0.017766019329428673, |
| "rewards/accuracy_reward": 0.9042699933052063, |
| "rewards/format_reward": 1.0, |
| "step": 571 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.58173370361328, |
| "epoch": 8.411764705882353, |
| "grad_norm": 1.8098644267187556, |
| "kl": 0.05908203125, |
| "learning_rate": 5.794117647058823e-07, |
| "loss": 0.0024, |
| "reward": 1.9107073545455933, |
| "reward_std": 0.017658641561865807, |
| "rewards/accuracy_reward": 0.9107073545455933, |
| "rewards/format_reward": 1.0, |
| "step": 572 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.56250762939453, |
| "epoch": 8.426470588235293, |
| "grad_norm": 3.2259185749098784, |
| "kl": 0.06005859375, |
| "learning_rate": 5.786764705882353e-07, |
| "loss": 0.0024, |
| "reward": 1.899427056312561, |
| "reward_std": 0.016428910195827484, |
| "rewards/accuracy_reward": 0.8994268774986267, |
| "rewards/format_reward": 1.0, |
| "step": 573 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.02764892578125, |
| "epoch": 8.441176470588236, |
| "grad_norm": 3.282308261215477, |
| "kl": 0.05859375, |
| "learning_rate": 5.779411764705882e-07, |
| "loss": 0.0023, |
| "reward": 1.9036839008331299, |
| "reward_std": 0.020171254873275757, |
| "rewards/accuracy_reward": 0.9036839008331299, |
| "rewards/format_reward": 1.0, |
| "step": 574 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.05048370361328, |
| "epoch": 8.455882352941176, |
| "grad_norm": 1.7713296358743136, |
| "kl": 0.061767578125, |
| "learning_rate": 5.772058823529411e-07, |
| "loss": 0.0025, |
| "reward": 1.9190058708190918, |
| "reward_std": 0.018266772851347923, |
| "rewards/accuracy_reward": 0.919005811214447, |
| "rewards/format_reward": 1.0, |
| "step": 575 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.00241088867188, |
| "epoch": 8.470588235294118, |
| "grad_norm": 1.9874750258288356, |
| "kl": 0.055908203125, |
| "learning_rate": 5.76470588235294e-07, |
| "loss": 0.0023, |
| "reward": 1.9230592250823975, |
| "reward_std": 0.013289669528603554, |
| "rewards/accuracy_reward": 0.9230592250823975, |
| "rewards/format_reward": 1.0, |
| "step": 576 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.89063262939453, |
| "epoch": 8.485294117647058, |
| "grad_norm": 2.286169294844814, |
| "kl": 0.0576171875, |
| "learning_rate": 5.75735294117647e-07, |
| "loss": 0.0023, |
| "reward": 1.8973294496536255, |
| "reward_std": 0.01872727833688259, |
| "rewards/accuracy_reward": 0.8973293900489807, |
| "rewards/format_reward": 1.0, |
| "step": 577 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.49279022216797, |
| "epoch": 8.5, |
| "grad_norm": 2.4071803200242416, |
| "kl": 0.0654296875, |
| "learning_rate": 5.749999999999999e-07, |
| "loss": 0.0026, |
| "reward": 1.9316534996032715, |
| "reward_std": 0.017379358410835266, |
| "rewards/accuracy_reward": 0.9316534399986267, |
| "rewards/format_reward": 1.0, |
| "step": 578 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.60216522216797, |
| "epoch": 8.514705882352942, |
| "grad_norm": 1.79762667011906, |
| "kl": 0.064453125, |
| "learning_rate": 5.742647058823529e-07, |
| "loss": 0.0026, |
| "reward": 1.9131649732589722, |
| "reward_std": 0.020887164399027824, |
| "rewards/accuracy_reward": 0.9143666625022888, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 579 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.56971740722656, |
| "epoch": 8.529411764705882, |
| "grad_norm": 2.3602888807544526, |
| "kl": 0.053955078125, |
| "learning_rate": 5.735294117647059e-07, |
| "loss": 0.0022, |
| "reward": 1.9001418352127075, |
| "reward_std": 0.01754673384130001, |
| "rewards/accuracy_reward": 0.9001417756080627, |
| "rewards/format_reward": 1.0, |
| "step": 580 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.32091522216797, |
| "epoch": 8.544117647058824, |
| "grad_norm": 1.8304863518291064, |
| "kl": 0.05615234375, |
| "learning_rate": 5.727941176470587e-07, |
| "loss": 0.0022, |
| "reward": 1.9000111818313599, |
| "reward_std": 0.017091548070311546, |
| "rewards/accuracy_reward": 0.9000111222267151, |
| "rewards/format_reward": 1.0, |
| "step": 581 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.98558044433594, |
| "epoch": 8.558823529411764, |
| "grad_norm": 2.667131501264164, |
| "kl": 0.0517578125, |
| "learning_rate": 5.720588235294117e-07, |
| "loss": 0.0021, |
| "reward": 1.9127933979034424, |
| "reward_std": 0.01638048328459263, |
| "rewards/accuracy_reward": 0.9127933979034424, |
| "rewards/format_reward": 1.0, |
| "step": 582 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.97116088867188, |
| "epoch": 8.573529411764707, |
| "grad_norm": 3.838255993972307, |
| "kl": 0.05322265625, |
| "learning_rate": 5.713235294117647e-07, |
| "loss": 0.0021, |
| "reward": 1.9052388668060303, |
| "reward_std": 0.015317871235311031, |
| "rewards/accuracy_reward": 0.9052388668060303, |
| "rewards/format_reward": 1.0, |
| "step": 583 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.7800521850586, |
| "epoch": 8.588235294117647, |
| "grad_norm": 1.8912826875619027, |
| "kl": 0.0517578125, |
| "learning_rate": 5.705882352941176e-07, |
| "loss": 0.0021, |
| "reward": 1.9118890762329102, |
| "reward_std": 0.016504064202308655, |
| "rewards/accuracy_reward": 0.9118890166282654, |
| "rewards/format_reward": 1.0, |
| "step": 584 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.36538696289062, |
| "epoch": 8.602941176470589, |
| "grad_norm": 1.9564532336203913, |
| "kl": 0.05517578125, |
| "learning_rate": 5.698529411764705e-07, |
| "loss": 0.0022, |
| "reward": 1.9083279371261597, |
| "reward_std": 0.01711435243487358, |
| "rewards/accuracy_reward": 0.9083278775215149, |
| "rewards/format_reward": 1.0, |
| "step": 585 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.89784240722656, |
| "epoch": 8.617647058823529, |
| "grad_norm": 1.411173293415255, |
| "kl": 0.049072265625, |
| "learning_rate": 5.691176470588235e-07, |
| "loss": 0.002, |
| "reward": 1.9171125888824463, |
| "reward_std": 0.016976075246930122, |
| "rewards/accuracy_reward": 0.9183142781257629, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 586 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.58053588867188, |
| "epoch": 8.632352941176471, |
| "grad_norm": 1.8153636392557528, |
| "kl": 0.050537109375, |
| "learning_rate": 5.683823529411764e-07, |
| "loss": 0.002, |
| "reward": 1.9182407855987549, |
| "reward_std": 0.013324742205440998, |
| "rewards/accuracy_reward": 0.9182407855987549, |
| "rewards/format_reward": 1.0, |
| "step": 587 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.02644348144531, |
| "epoch": 8.647058823529411, |
| "grad_norm": 2.991122791360467, |
| "kl": 0.049072265625, |
| "learning_rate": 5.676470588235294e-07, |
| "loss": 0.002, |
| "reward": 1.9190689325332642, |
| "reward_std": 0.01496278028935194, |
| "rewards/accuracy_reward": 0.9190687537193298, |
| "rewards/format_reward": 1.0, |
| "step": 588 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.1971206665039, |
| "epoch": 8.661764705882353, |
| "grad_norm": 1.5660894873733406, |
| "kl": 0.05078125, |
| "learning_rate": 5.669117647058823e-07, |
| "loss": 0.002, |
| "reward": 1.8968241214752197, |
| "reward_std": 0.015287560410797596, |
| "rewards/accuracy_reward": 0.896824061870575, |
| "rewards/format_reward": 1.0, |
| "step": 589 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.95072174072266, |
| "epoch": 8.676470588235293, |
| "grad_norm": 1.7772209705137463, |
| "kl": 0.054443359375, |
| "learning_rate": 5.661764705882352e-07, |
| "loss": 0.0022, |
| "reward": 1.9128403663635254, |
| "reward_std": 0.020346660166978836, |
| "rewards/accuracy_reward": 0.9140422940254211, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 590 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.78726196289062, |
| "epoch": 8.691176470588236, |
| "grad_norm": 1.6721267407160185, |
| "kl": 0.046630859375, |
| "learning_rate": 5.654411764705882e-07, |
| "loss": 0.0019, |
| "reward": 1.9050085544586182, |
| "reward_std": 0.018483325839042664, |
| "rewards/accuracy_reward": 0.9050085544586182, |
| "rewards/format_reward": 1.0, |
| "step": 591 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.97956848144531, |
| "epoch": 8.705882352941176, |
| "grad_norm": 1.6709044867050522, |
| "kl": 0.052490234375, |
| "learning_rate": 5.647058823529412e-07, |
| "loss": 0.0021, |
| "reward": 1.924027442932129, |
| "reward_std": 0.019660327583551407, |
| "rewards/accuracy_reward": 0.9252294301986694, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 592 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.9988021850586, |
| "epoch": 8.720588235294118, |
| "grad_norm": 4.487069114218076, |
| "kl": 0.05029296875, |
| "learning_rate": 5.639705882352941e-07, |
| "loss": 0.002, |
| "reward": 1.8979933261871338, |
| "reward_std": 0.01683800294995308, |
| "rewards/accuracy_reward": 0.897993266582489, |
| "rewards/format_reward": 1.0, |
| "step": 593 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.0769271850586, |
| "epoch": 8.735294117647058, |
| "grad_norm": 2.4615353009495817, |
| "kl": 0.07763671875, |
| "learning_rate": 5.63235294117647e-07, |
| "loss": 0.0031, |
| "reward": 1.8970861434936523, |
| "reward_std": 0.019765671342611313, |
| "rewards/accuracy_reward": 0.8982879519462585, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 594 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.85216522216797, |
| "epoch": 8.75, |
| "grad_norm": 1.735078503716415, |
| "kl": 0.051025390625, |
| "learning_rate": 5.625e-07, |
| "loss": 0.002, |
| "reward": 1.9034225940704346, |
| "reward_std": 0.018201464787125587, |
| "rewards/accuracy_reward": 0.9034226536750793, |
| "rewards/format_reward": 1.0, |
| "step": 595 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.47116088867188, |
| "epoch": 8.764705882352942, |
| "grad_norm": 2.193668601424281, |
| "kl": 0.048095703125, |
| "learning_rate": 5.617647058823529e-07, |
| "loss": 0.0019, |
| "reward": 1.8874914646148682, |
| "reward_std": 0.024996280670166016, |
| "rewards/accuracy_reward": 0.8898953795433044, |
| "rewards/format_reward": 0.9975962042808533, |
| "step": 596 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.53366088867188, |
| "epoch": 8.779411764705882, |
| "grad_norm": 1.5231562376577301, |
| "kl": 0.0498046875, |
| "learning_rate": 5.610294117647059e-07, |
| "loss": 0.002, |
| "reward": 1.910040259361267, |
| "reward_std": 0.016859347000718117, |
| "rewards/accuracy_reward": 0.9100401997566223, |
| "rewards/format_reward": 1.0, |
| "step": 597 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.86058044433594, |
| "epoch": 8.794117647058824, |
| "grad_norm": 6.4583154297757455, |
| "kl": 0.05029296875, |
| "learning_rate": 5.602941176470588e-07, |
| "loss": 0.002, |
| "reward": 1.9111155271530151, |
| "reward_std": 0.016216028481721878, |
| "rewards/accuracy_reward": 0.9111157059669495, |
| "rewards/format_reward": 1.0, |
| "step": 598 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.24279022216797, |
| "epoch": 8.808823529411764, |
| "grad_norm": 1.775836022535222, |
| "kl": 0.05224609375, |
| "learning_rate": 5.595588235294117e-07, |
| "loss": 0.0021, |
| "reward": 1.917399287223816, |
| "reward_std": 0.016161981970071793, |
| "rewards/accuracy_reward": 0.9173993468284607, |
| "rewards/format_reward": 1.0, |
| "step": 599 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.46514892578125, |
| "epoch": 8.823529411764707, |
| "grad_norm": 1.374775331428731, |
| "kl": 0.052001953125, |
| "learning_rate": 5.588235294117647e-07, |
| "loss": 0.0021, |
| "reward": 1.8994345664978027, |
| "reward_std": 0.017103036865592003, |
| "rewards/accuracy_reward": 0.8994343876838684, |
| "rewards/format_reward": 1.0, |
| "step": 600 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.3894271850586, |
| "epoch": 8.838235294117647, |
| "grad_norm": 2.9229408253654365, |
| "kl": 0.05029296875, |
| "learning_rate": 5.580882352941177e-07, |
| "loss": 0.002, |
| "reward": 1.9245597124099731, |
| "reward_std": 0.016064738854765892, |
| "rewards/accuracy_reward": 0.9245598316192627, |
| "rewards/format_reward": 1.0, |
| "step": 601 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.09134674072266, |
| "epoch": 8.852941176470589, |
| "grad_norm": 1.761051200827917, |
| "kl": 0.053955078125, |
| "learning_rate": 5.573529411764706e-07, |
| "loss": 0.0022, |
| "reward": 1.9057958126068115, |
| "reward_std": 0.018393708392977715, |
| "rewards/accuracy_reward": 0.9057958126068115, |
| "rewards/format_reward": 1.0, |
| "step": 602 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.97956848144531, |
| "epoch": 8.867647058823529, |
| "grad_norm": 1.9085242419393635, |
| "kl": 0.05419921875, |
| "learning_rate": 5.566176470588235e-07, |
| "loss": 0.0022, |
| "reward": 1.9057040214538574, |
| "reward_std": 0.02528255060315132, |
| "rewards/accuracy_reward": 0.9069057106971741, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 603 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.55409240722656, |
| "epoch": 8.882352941176471, |
| "grad_norm": 2.2056191207524742, |
| "kl": 0.050537109375, |
| "learning_rate": 5.558823529411765e-07, |
| "loss": 0.002, |
| "reward": 1.9001765251159668, |
| "reward_std": 0.01882914826273918, |
| "rewards/accuracy_reward": 0.9001763463020325, |
| "rewards/format_reward": 1.0, |
| "step": 604 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.1189956665039, |
| "epoch": 8.897058823529411, |
| "grad_norm": 1.6684830952892549, |
| "kl": 0.049072265625, |
| "learning_rate": 5.551470588235294e-07, |
| "loss": 0.002, |
| "reward": 1.9099537134170532, |
| "reward_std": 0.015918085351586342, |
| "rewards/accuracy_reward": 0.9099536538124084, |
| "rewards/format_reward": 1.0, |
| "step": 605 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.1081771850586, |
| "epoch": 8.911764705882353, |
| "grad_norm": 2.437963685533989, |
| "kl": 0.052001953125, |
| "learning_rate": 5.544117647058824e-07, |
| "loss": 0.0021, |
| "reward": 1.9115108251571655, |
| "reward_std": 0.01843283697962761, |
| "rewards/accuracy_reward": 0.911510705947876, |
| "rewards/format_reward": 1.0, |
| "step": 606 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.92188262939453, |
| "epoch": 8.926470588235293, |
| "grad_norm": 5.175318731979124, |
| "kl": 0.053466796875, |
| "learning_rate": 5.536764705882353e-07, |
| "loss": 0.0021, |
| "reward": 1.9014861583709717, |
| "reward_std": 0.016052136197686195, |
| "rewards/accuracy_reward": 0.9014859199523926, |
| "rewards/format_reward": 1.0, |
| "step": 607 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.3377456665039, |
| "epoch": 8.941176470588236, |
| "grad_norm": 1.466199052775766, |
| "kl": 0.055419921875, |
| "learning_rate": 5.529411764705882e-07, |
| "loss": 0.0022, |
| "reward": 1.8980929851531982, |
| "reward_std": 0.020213749259710312, |
| "rewards/accuracy_reward": 0.8992946743965149, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 608 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.5252456665039, |
| "epoch": 8.955882352941176, |
| "grad_norm": 2.4803683445732196, |
| "kl": 0.05224609375, |
| "learning_rate": 5.522058823529412e-07, |
| "loss": 0.0021, |
| "reward": 1.8946930170059204, |
| "reward_std": 0.020943893119692802, |
| "rewards/accuracy_reward": 0.8946928381919861, |
| "rewards/format_reward": 1.0, |
| "step": 609 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.81851196289062, |
| "epoch": 8.970588235294118, |
| "grad_norm": 2.199347709692285, |
| "kl": 0.056396484375, |
| "learning_rate": 5.514705882352942e-07, |
| "loss": 0.0023, |
| "reward": 1.9163153171539307, |
| "reward_std": 0.019964834675192833, |
| "rewards/accuracy_reward": 0.9175171852111816, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 610 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.43389892578125, |
| "epoch": 8.985294117647058, |
| "grad_norm": 1.3598803521986345, |
| "kl": 0.058349609375, |
| "learning_rate": 5.50735294117647e-07, |
| "loss": 0.0023, |
| "reward": 1.9314730167388916, |
| "reward_std": 0.01666920818388462, |
| "rewards/accuracy_reward": 0.9314729571342468, |
| "rewards/format_reward": 1.0, |
| "step": 611 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.38204193115234, |
| "epoch": 9.0, |
| "grad_norm": 2.5292498675314574, |
| "kl": 0.0546875, |
| "learning_rate": 5.5e-07, |
| "loss": 0.0022, |
| "reward": 1.9147766828536987, |
| "reward_std": 0.012550020590424538, |
| "rewards/accuracy_reward": 0.9147767424583435, |
| "rewards/format_reward": 1.0, |
| "step": 612 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.68269348144531, |
| "epoch": 9.014705882352942, |
| "grad_norm": 1.359576415697596, |
| "kl": 0.05419921875, |
| "learning_rate": 5.49264705882353e-07, |
| "loss": 0.0022, |
| "reward": 1.9160057306289673, |
| "reward_std": 0.018689963966608047, |
| "rewards/accuracy_reward": 0.9172075986862183, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 613 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.44831848144531, |
| "epoch": 9.029411764705882, |
| "grad_norm": 1.5142397828555845, |
| "kl": 0.0595703125, |
| "learning_rate": 5.485294117647059e-07, |
| "loss": 0.0024, |
| "reward": 1.8921860456466675, |
| "reward_std": 0.021658875048160553, |
| "rewards/accuracy_reward": 0.8933879137039185, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 614 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.91226196289062, |
| "epoch": 9.044117647058824, |
| "grad_norm": 5.469469688964059, |
| "kl": 0.05615234375, |
| "learning_rate": 5.477941176470589e-07, |
| "loss": 0.0023, |
| "reward": 1.9030665159225464, |
| "reward_std": 0.02024255506694317, |
| "rewards/accuracy_reward": 0.9042685627937317, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 615 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.23197174072266, |
| "epoch": 9.058823529411764, |
| "grad_norm": 2.869773876567515, |
| "kl": 0.07080078125, |
| "learning_rate": 5.470588235294118e-07, |
| "loss": 0.0028, |
| "reward": 1.912828803062439, |
| "reward_std": 0.018641209229826927, |
| "rewards/accuracy_reward": 0.9128286838531494, |
| "rewards/format_reward": 1.0, |
| "step": 616 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.40625762939453, |
| "epoch": 9.073529411764707, |
| "grad_norm": 1.880572098436719, |
| "kl": 0.05712890625, |
| "learning_rate": 5.463235294117646e-07, |
| "loss": 0.0023, |
| "reward": 1.9045672416687012, |
| "reward_std": 0.01760552078485489, |
| "rewards/accuracy_reward": 0.9045671820640564, |
| "rewards/format_reward": 1.0, |
| "step": 617 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.60216522216797, |
| "epoch": 9.088235294117647, |
| "grad_norm": 1.6132861487685206, |
| "kl": 0.056884765625, |
| "learning_rate": 5.455882352941176e-07, |
| "loss": 0.0023, |
| "reward": 1.9064825773239136, |
| "reward_std": 0.02047489397227764, |
| "rewards/accuracy_reward": 0.9064825773239136, |
| "rewards/format_reward": 1.0, |
| "step": 618 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.12139892578125, |
| "epoch": 9.102941176470589, |
| "grad_norm": 1.8352294433327585, |
| "kl": 0.061279296875, |
| "learning_rate": 5.448529411764706e-07, |
| "loss": 0.0024, |
| "reward": 1.9166762828826904, |
| "reward_std": 0.017683234065771103, |
| "rewards/accuracy_reward": 0.9166761636734009, |
| "rewards/format_reward": 1.0, |
| "step": 619 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.82813262939453, |
| "epoch": 9.117647058823529, |
| "grad_norm": 1.9584175345112422, |
| "kl": 0.05810546875, |
| "learning_rate": 5.441176470588234e-07, |
| "loss": 0.0023, |
| "reward": 1.9098379611968994, |
| "reward_std": 0.019358208402991295, |
| "rewards/accuracy_reward": 0.9098379015922546, |
| "rewards/format_reward": 1.0, |
| "step": 620 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.54928588867188, |
| "epoch": 9.132352941176471, |
| "grad_norm": 1.681570036282453, |
| "kl": 0.05908203125, |
| "learning_rate": 5.433823529411764e-07, |
| "loss": 0.0024, |
| "reward": 1.903365135192871, |
| "reward_std": 0.018463198095560074, |
| "rewards/accuracy_reward": 0.9045670032501221, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 621 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.81370544433594, |
| "epoch": 9.147058823529411, |
| "grad_norm": 2.175735273009701, |
| "kl": 0.06787109375, |
| "learning_rate": 5.426470588235294e-07, |
| "loss": 0.0027, |
| "reward": 1.9245383739471436, |
| "reward_std": 0.015885451808571815, |
| "rewards/accuracy_reward": 0.9245381355285645, |
| "rewards/format_reward": 1.0, |
| "step": 622 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.20313262939453, |
| "epoch": 9.161764705882353, |
| "grad_norm": 2.20406419419079, |
| "kl": 0.05810546875, |
| "learning_rate": 5.419117647058823e-07, |
| "loss": 0.0023, |
| "reward": 1.9137063026428223, |
| "reward_std": 0.018180424347519875, |
| "rewards/accuracy_reward": 0.9137064218521118, |
| "rewards/format_reward": 1.0, |
| "step": 623 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.13221740722656, |
| "epoch": 9.176470588235293, |
| "grad_norm": 1.7531001551644594, |
| "kl": 0.057373046875, |
| "learning_rate": 5.411764705882353e-07, |
| "loss": 0.0023, |
| "reward": 1.9103367328643799, |
| "reward_std": 0.01696370542049408, |
| "rewards/accuracy_reward": 0.9103366136550903, |
| "rewards/format_reward": 1.0, |
| "step": 624 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.02163696289062, |
| "epoch": 9.191176470588236, |
| "grad_norm": 1.9594882556521003, |
| "kl": 0.06201171875, |
| "learning_rate": 5.404411764705882e-07, |
| "loss": 0.0025, |
| "reward": 1.9130682945251465, |
| "reward_std": 0.014644093811511993, |
| "rewards/accuracy_reward": 0.9130681157112122, |
| "rewards/format_reward": 1.0, |
| "step": 625 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.65866088867188, |
| "epoch": 9.205882352941176, |
| "grad_norm": 2.6647032908164303, |
| "kl": 0.0595703125, |
| "learning_rate": 5.397058823529411e-07, |
| "loss": 0.0024, |
| "reward": 1.9141494035720825, |
| "reward_std": 0.013022142462432384, |
| "rewards/accuracy_reward": 0.9141494035720825, |
| "rewards/format_reward": 1.0, |
| "step": 626 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.36298370361328, |
| "epoch": 9.220588235294118, |
| "grad_norm": 2.5371748740188753, |
| "kl": 0.060791015625, |
| "learning_rate": 5.389705882352941e-07, |
| "loss": 0.0025, |
| "reward": 1.91664719581604, |
| "reward_std": 0.017335418611764908, |
| "rewards/accuracy_reward": 0.9166471362113953, |
| "rewards/format_reward": 1.0, |
| "step": 627 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 114.4627456665039, |
| "epoch": 9.235294117647058, |
| "grad_norm": 1.7615752856014142, |
| "kl": 0.05712890625, |
| "learning_rate": 5.382352941176471e-07, |
| "loss": 0.0023, |
| "reward": 1.8938990831375122, |
| "reward_std": 0.016765251755714417, |
| "rewards/accuracy_reward": 0.893899142742157, |
| "rewards/format_reward": 1.0, |
| "step": 628 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.77404022216797, |
| "epoch": 9.25, |
| "grad_norm": 2.0562724198467914, |
| "kl": 0.0634765625, |
| "learning_rate": 5.374999999999999e-07, |
| "loss": 0.0026, |
| "reward": 1.9214463233947754, |
| "reward_std": 0.014558774419128895, |
| "rewards/accuracy_reward": 0.9214461445808411, |
| "rewards/format_reward": 1.0, |
| "step": 629 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.44351196289062, |
| "epoch": 9.264705882352942, |
| "grad_norm": 2.1195056229903555, |
| "kl": 0.0576171875, |
| "learning_rate": 5.367647058823529e-07, |
| "loss": 0.0023, |
| "reward": 1.9065834283828735, |
| "reward_std": 0.016932621598243713, |
| "rewards/accuracy_reward": 0.906583309173584, |
| "rewards/format_reward": 1.0, |
| "step": 630 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.56610870361328, |
| "epoch": 9.279411764705882, |
| "grad_norm": 1.5239442402080228, |
| "kl": 0.0546875, |
| "learning_rate": 5.360294117647058e-07, |
| "loss": 0.0022, |
| "reward": 1.9177383184432983, |
| "reward_std": 0.014962482266128063, |
| "rewards/accuracy_reward": 0.9177382588386536, |
| "rewards/format_reward": 1.0, |
| "step": 631 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.86538696289062, |
| "epoch": 9.294117647058824, |
| "grad_norm": 2.027623877286765, |
| "kl": 0.05517578125, |
| "learning_rate": 5.352941176470588e-07, |
| "loss": 0.0022, |
| "reward": 1.9050103425979614, |
| "reward_std": 0.014742089435458183, |
| "rewards/accuracy_reward": 0.9050101637840271, |
| "rewards/format_reward": 1.0, |
| "step": 632 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.48558044433594, |
| "epoch": 9.308823529411764, |
| "grad_norm": 1.9258858099537597, |
| "kl": 0.060546875, |
| "learning_rate": 5.345588235294117e-07, |
| "loss": 0.0024, |
| "reward": 1.9041670560836792, |
| "reward_std": 0.017647940665483475, |
| "rewards/accuracy_reward": 0.9053689241409302, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 633 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.88221740722656, |
| "epoch": 9.323529411764707, |
| "grad_norm": 5.481626944028736, |
| "kl": 0.05419921875, |
| "learning_rate": 5.338235294117646e-07, |
| "loss": 0.0022, |
| "reward": 1.8989014625549316, |
| "reward_std": 0.017911560833454132, |
| "rewards/accuracy_reward": 0.8989012837409973, |
| "rewards/format_reward": 1.0, |
| "step": 634 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 114.16947174072266, |
| "epoch": 9.338235294117647, |
| "grad_norm": 1.5838252460154945, |
| "kl": 0.052734375, |
| "learning_rate": 5.330882352941176e-07, |
| "loss": 0.0021, |
| "reward": 1.9077017307281494, |
| "reward_std": 0.01683455891907215, |
| "rewards/accuracy_reward": 0.9089035987854004, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 635 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.02764892578125, |
| "epoch": 9.352941176470589, |
| "grad_norm": 20.072264259712355, |
| "kl": 0.05908203125, |
| "learning_rate": 5.323529411764706e-07, |
| "loss": 0.0024, |
| "reward": 1.9148465394973755, |
| "reward_std": 0.01663627289235592, |
| "rewards/accuracy_reward": 0.9160483479499817, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 636 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 114.4158706665039, |
| "epoch": 9.367647058823529, |
| "grad_norm": 1.5736397571374754, |
| "kl": 0.05712890625, |
| "learning_rate": 5.316176470588235e-07, |
| "loss": 0.0023, |
| "reward": 1.9175165891647339, |
| "reward_std": 0.017536688596010208, |
| "rewards/accuracy_reward": 0.9175165891647339, |
| "rewards/format_reward": 1.0, |
| "step": 637 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.81971740722656, |
| "epoch": 9.382352941176471, |
| "grad_norm": 6.15992463353084, |
| "kl": 0.054931640625, |
| "learning_rate": 5.308823529411764e-07, |
| "loss": 0.0022, |
| "reward": 1.9007567167282104, |
| "reward_std": 0.015532239340245724, |
| "rewards/accuracy_reward": 0.9007567167282104, |
| "rewards/format_reward": 1.0, |
| "step": 638 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.92668914794922, |
| "epoch": 9.397058823529411, |
| "grad_norm": 1.9267289588312566, |
| "kl": 0.05712890625, |
| "learning_rate": 5.301470588235294e-07, |
| "loss": 0.0023, |
| "reward": 1.910290241241455, |
| "reward_std": 0.017649637535214424, |
| "rewards/accuracy_reward": 0.9114922285079956, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 639 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.70553588867188, |
| "epoch": 9.411764705882353, |
| "grad_norm": 2.331055894294996, |
| "kl": 0.057861328125, |
| "learning_rate": 5.294117647058823e-07, |
| "loss": 0.0023, |
| "reward": 1.9117798805236816, |
| "reward_std": 0.014000487513840199, |
| "rewards/accuracy_reward": 0.9117798209190369, |
| "rewards/format_reward": 1.0, |
| "step": 640 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 114.7644271850586, |
| "epoch": 9.426470588235293, |
| "grad_norm": 2.9782765384820302, |
| "kl": 0.05908203125, |
| "learning_rate": 5.286764705882353e-07, |
| "loss": 0.0024, |
| "reward": 1.8985432386398315, |
| "reward_std": 0.016746515408158302, |
| "rewards/accuracy_reward": 0.8985430002212524, |
| "rewards/format_reward": 1.0, |
| "step": 641 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.16106414794922, |
| "epoch": 9.441176470588236, |
| "grad_norm": 3.0263333302618585, |
| "kl": 0.0576171875, |
| "learning_rate": 5.279411764705882e-07, |
| "loss": 0.0023, |
| "reward": 1.9077398777008057, |
| "reward_std": 0.016522472724318504, |
| "rewards/accuracy_reward": 0.9089417457580566, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 642 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.0252456665039, |
| "epoch": 9.455882352941176, |
| "grad_norm": 3.6088229354945858, |
| "kl": 0.05615234375, |
| "learning_rate": 5.272058823529411e-07, |
| "loss": 0.0023, |
| "reward": 1.912189245223999, |
| "reward_std": 0.01710669696331024, |
| "rewards/accuracy_reward": 0.9133911728858948, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 643 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.2908706665039, |
| "epoch": 9.470588235294118, |
| "grad_norm": 8.188981831182572, |
| "kl": 0.053466796875, |
| "learning_rate": 5.264705882352941e-07, |
| "loss": 0.0021, |
| "reward": 1.9111279249191284, |
| "reward_std": 0.0145622119307518, |
| "rewards/accuracy_reward": 0.9111279249191284, |
| "rewards/format_reward": 1.0, |
| "step": 644 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.99639892578125, |
| "epoch": 9.485294117647058, |
| "grad_norm": 3.3433584553752866, |
| "kl": 0.054931640625, |
| "learning_rate": 5.257352941176471e-07, |
| "loss": 0.0022, |
| "reward": 1.9080722332000732, |
| "reward_std": 0.01804247312247753, |
| "rewards/accuracy_reward": 0.9092740416526794, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 645 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.66346740722656, |
| "epoch": 9.5, |
| "grad_norm": 1.3581128929115824, |
| "kl": 0.05908203125, |
| "learning_rate": 5.25e-07, |
| "loss": 0.0024, |
| "reward": 1.9173364639282227, |
| "reward_std": 0.0154867609962821, |
| "rewards/accuracy_reward": 0.9173362851142883, |
| "rewards/format_reward": 1.0, |
| "step": 646 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.97956848144531, |
| "epoch": 9.514705882352942, |
| "grad_norm": 3.583185511646481, |
| "kl": 0.0517578125, |
| "learning_rate": 5.242647058823529e-07, |
| "loss": 0.0021, |
| "reward": 1.913078784942627, |
| "reward_std": 0.01793815754354, |
| "rewards/accuracy_reward": 0.9142807126045227, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 647 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.81971740722656, |
| "epoch": 9.529411764705882, |
| "grad_norm": 2.2219404022193494, |
| "kl": 0.058349609375, |
| "learning_rate": 5.235294117647059e-07, |
| "loss": 0.0023, |
| "reward": 1.9022469520568848, |
| "reward_std": 0.019146382808685303, |
| "rewards/accuracy_reward": 0.9046508073806763, |
| "rewards/format_reward": 0.9975962042808533, |
| "step": 648 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.07813262939453, |
| "epoch": 9.544117647058824, |
| "grad_norm": 2.825487501905413, |
| "kl": 0.0625, |
| "learning_rate": 5.227941176470588e-07, |
| "loss": 0.0025, |
| "reward": 1.916589617729187, |
| "reward_std": 0.016064953058958054, |
| "rewards/accuracy_reward": 0.9177915453910828, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 649 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.50601196289062, |
| "epoch": 9.558823529411764, |
| "grad_norm": 5.158018889820331, |
| "kl": 0.051513671875, |
| "learning_rate": 5.220588235294118e-07, |
| "loss": 0.0021, |
| "reward": 1.9119791984558105, |
| "reward_std": 0.015298433601856232, |
| "rewards/accuracy_reward": 0.9119791388511658, |
| "rewards/format_reward": 1.0, |
| "step": 650 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.58413696289062, |
| "epoch": 9.573529411764707, |
| "grad_norm": 2.329896463825327, |
| "kl": 0.047119140625, |
| "learning_rate": 5.213235294117647e-07, |
| "loss": 0.0019, |
| "reward": 1.9101742506027222, |
| "reward_std": 0.015595993027091026, |
| "rewards/accuracy_reward": 0.9101741909980774, |
| "rewards/format_reward": 1.0, |
| "step": 651 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.80769348144531, |
| "epoch": 9.588235294117647, |
| "grad_norm": 1.93503535510301, |
| "kl": 0.05322265625, |
| "learning_rate": 5.205882352941176e-07, |
| "loss": 0.0021, |
| "reward": 1.9048891067504883, |
| "reward_std": 0.014783341437578201, |
| "rewards/accuracy_reward": 0.904888927936554, |
| "rewards/format_reward": 1.0, |
| "step": 652 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.08534240722656, |
| "epoch": 9.602941176470589, |
| "grad_norm": 2.84495334769509, |
| "kl": 0.060791015625, |
| "learning_rate": 5.198529411764706e-07, |
| "loss": 0.0024, |
| "reward": 1.9240138530731201, |
| "reward_std": 0.014201764948666096, |
| "rewards/accuracy_reward": 0.9240137338638306, |
| "rewards/format_reward": 1.0, |
| "step": 653 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.84856414794922, |
| "epoch": 9.617647058823529, |
| "grad_norm": 3.012251558930328, |
| "kl": 0.05419921875, |
| "learning_rate": 5.191176470588236e-07, |
| "loss": 0.0022, |
| "reward": 1.905051350593567, |
| "reward_std": 0.013432993553578854, |
| "rewards/accuracy_reward": 0.9050513505935669, |
| "rewards/format_reward": 1.0, |
| "step": 654 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.21394348144531, |
| "epoch": 9.632352941176471, |
| "grad_norm": 2.1316420678211956, |
| "kl": 0.056640625, |
| "learning_rate": 5.183823529411764e-07, |
| "loss": 0.0023, |
| "reward": 1.9092304706573486, |
| "reward_std": 0.01678118109703064, |
| "rewards/accuracy_reward": 0.9092304706573486, |
| "rewards/format_reward": 1.0, |
| "step": 655 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.33654022216797, |
| "epoch": 9.647058823529411, |
| "grad_norm": 2.2956788687873826, |
| "kl": 0.06298828125, |
| "learning_rate": 5.176470588235294e-07, |
| "loss": 0.0025, |
| "reward": 1.92042076587677, |
| "reward_std": 0.016655659303069115, |
| "rewards/accuracy_reward": 0.92042076587677, |
| "rewards/format_reward": 1.0, |
| "step": 656 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.62259674072266, |
| "epoch": 9.661764705882353, |
| "grad_norm": 2.467082612046906, |
| "kl": 0.054931640625, |
| "learning_rate": 5.169117647058824e-07, |
| "loss": 0.0022, |
| "reward": 1.9160475730895996, |
| "reward_std": 0.01348184235394001, |
| "rewards/accuracy_reward": 0.9160475134849548, |
| "rewards/format_reward": 1.0, |
| "step": 657 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.80529022216797, |
| "epoch": 9.676470588235293, |
| "grad_norm": 7.66545161952161, |
| "kl": 0.06396484375, |
| "learning_rate": 5.161764705882353e-07, |
| "loss": 0.0026, |
| "reward": 1.9231846332550049, |
| "reward_std": 0.016615498811006546, |
| "rewards/accuracy_reward": 0.9231847524642944, |
| "rewards/format_reward": 1.0, |
| "step": 658 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.1189956665039, |
| "epoch": 9.691176470588236, |
| "grad_norm": 2.341042431156855, |
| "kl": 0.057373046875, |
| "learning_rate": 5.154411764705882e-07, |
| "loss": 0.0023, |
| "reward": 1.912707805633545, |
| "reward_std": 0.018813416361808777, |
| "rewards/accuracy_reward": 0.9127077460289001, |
| "rewards/format_reward": 1.0, |
| "step": 659 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.10938262939453, |
| "epoch": 9.705882352941176, |
| "grad_norm": 2.1614288925748966, |
| "kl": 0.0654296875, |
| "learning_rate": 5.147058823529411e-07, |
| "loss": 0.0026, |
| "reward": 1.9185718297958374, |
| "reward_std": 0.01388857513666153, |
| "rewards/accuracy_reward": 0.9185718894004822, |
| "rewards/format_reward": 1.0, |
| "step": 660 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.92788696289062, |
| "epoch": 9.720588235294118, |
| "grad_norm": 2.80121444579991, |
| "kl": 0.06396484375, |
| "learning_rate": 5.13970588235294e-07, |
| "loss": 0.0026, |
| "reward": 1.9083857536315918, |
| "reward_std": 0.01726207509636879, |
| "rewards/accuracy_reward": 0.9083855152130127, |
| "rewards/format_reward": 1.0, |
| "step": 661 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.50721740722656, |
| "epoch": 9.735294117647058, |
| "grad_norm": 2.7135290225206936, |
| "kl": 0.060546875, |
| "learning_rate": 5.13235294117647e-07, |
| "loss": 0.0024, |
| "reward": 1.8957988023757935, |
| "reward_std": 0.017370108515024185, |
| "rewards/accuracy_reward": 0.8957986235618591, |
| "rewards/format_reward": 1.0, |
| "step": 662 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.86298370361328, |
| "epoch": 9.75, |
| "grad_norm": 1.6206042221923098, |
| "kl": 0.054931640625, |
| "learning_rate": 5.125e-07, |
| "loss": 0.0022, |
| "reward": 1.9014039039611816, |
| "reward_std": 0.0170334093272686, |
| "rewards/accuracy_reward": 0.9014038443565369, |
| "rewards/format_reward": 1.0, |
| "step": 663 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.10577392578125, |
| "epoch": 9.764705882352942, |
| "grad_norm": 2.1238141084640287, |
| "kl": 0.056396484375, |
| "learning_rate": 5.117647058823528e-07, |
| "loss": 0.0022, |
| "reward": 1.9057307243347168, |
| "reward_std": 0.017774611711502075, |
| "rewards/accuracy_reward": 0.9057307243347168, |
| "rewards/format_reward": 1.0, |
| "step": 664 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.60096740722656, |
| "epoch": 9.779411764705882, |
| "grad_norm": 1.6218363054606357, |
| "kl": 0.052490234375, |
| "learning_rate": 5.110294117647058e-07, |
| "loss": 0.0021, |
| "reward": 1.892624855041504, |
| "reward_std": 0.018729185685515404, |
| "rewards/accuracy_reward": 0.8926247954368591, |
| "rewards/format_reward": 1.0, |
| "step": 665 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.73798370361328, |
| "epoch": 9.794117647058824, |
| "grad_norm": 4.41894162926749, |
| "kl": 0.061279296875, |
| "learning_rate": 5.102941176470588e-07, |
| "loss": 0.0024, |
| "reward": 1.8940479755401611, |
| "reward_std": 0.019051702693104744, |
| "rewards/accuracy_reward": 0.8940479159355164, |
| "rewards/format_reward": 1.0, |
| "step": 666 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.1706771850586, |
| "epoch": 9.808823529411764, |
| "grad_norm": 2.5204373524492754, |
| "kl": 0.0625, |
| "learning_rate": 5.095588235294117e-07, |
| "loss": 0.0025, |
| "reward": 1.9303815364837646, |
| "reward_std": 0.016049683094024658, |
| "rewards/accuracy_reward": 0.9303815364837646, |
| "rewards/format_reward": 1.0, |
| "step": 667 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.2596206665039, |
| "epoch": 9.823529411764707, |
| "grad_norm": 2.247808117324336, |
| "kl": 0.062255859375, |
| "learning_rate": 5.088235294117646e-07, |
| "loss": 0.0025, |
| "reward": 1.905713677406311, |
| "reward_std": 0.0260589811950922, |
| "rewards/accuracy_reward": 0.9069154858589172, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 668 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.00360870361328, |
| "epoch": 9.838235294117647, |
| "grad_norm": 2.9378173829289542, |
| "kl": 0.05517578125, |
| "learning_rate": 5.080882352941176e-07, |
| "loss": 0.0022, |
| "reward": 1.9220634698867798, |
| "reward_std": 0.018444770947098732, |
| "rewards/accuracy_reward": 0.923265278339386, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 669 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.38341522216797, |
| "epoch": 9.852941176470589, |
| "grad_norm": 3.1381468730562396, |
| "kl": 0.055908203125, |
| "learning_rate": 5.073529411764705e-07, |
| "loss": 0.0022, |
| "reward": 1.9113494157791138, |
| "reward_std": 0.017814362421631813, |
| "rewards/accuracy_reward": 0.91255122423172, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 670 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.08654022216797, |
| "epoch": 9.867647058823529, |
| "grad_norm": 1.8503806680078807, |
| "kl": 0.053466796875, |
| "learning_rate": 5.066176470588235e-07, |
| "loss": 0.0021, |
| "reward": 1.9079095125198364, |
| "reward_std": 0.01949225552380085, |
| "rewards/accuracy_reward": 0.9091113209724426, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 671 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.08654022216797, |
| "epoch": 9.882352941176471, |
| "grad_norm": 1.6795640500896751, |
| "kl": 0.051025390625, |
| "learning_rate": 5.058823529411765e-07, |
| "loss": 0.002, |
| "reward": 1.9217894077301025, |
| "reward_std": 0.01603555493056774, |
| "rewards/accuracy_reward": 0.9217892289161682, |
| "rewards/format_reward": 1.0, |
| "step": 672 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.29327392578125, |
| "epoch": 9.897058823529411, |
| "grad_norm": 1.9285682708702305, |
| "kl": 0.05126953125, |
| "learning_rate": 5.051470588235293e-07, |
| "loss": 0.002, |
| "reward": 1.9053161144256592, |
| "reward_std": 0.01650179550051689, |
| "rewards/accuracy_reward": 0.9053158760070801, |
| "rewards/format_reward": 1.0, |
| "step": 673 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.87981414794922, |
| "epoch": 9.911764705882353, |
| "grad_norm": 2.038439014531085, |
| "kl": 0.05126953125, |
| "learning_rate": 5.044117647058823e-07, |
| "loss": 0.002, |
| "reward": 1.9242396354675293, |
| "reward_std": 0.016753748059272766, |
| "rewards/accuracy_reward": 0.9242396354675293, |
| "rewards/format_reward": 1.0, |
| "step": 674 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.26202392578125, |
| "epoch": 9.926470588235293, |
| "grad_norm": 1.6234604098074579, |
| "kl": 0.053955078125, |
| "learning_rate": 5.036764705882353e-07, |
| "loss": 0.0022, |
| "reward": 1.914339542388916, |
| "reward_std": 0.01580924354493618, |
| "rewards/accuracy_reward": 0.9143396019935608, |
| "rewards/format_reward": 1.0, |
| "step": 675 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.72116088867188, |
| "epoch": 9.941176470588236, |
| "grad_norm": 5.62812065845287, |
| "kl": 0.0537109375, |
| "learning_rate": 5.029411764705882e-07, |
| "loss": 0.0021, |
| "reward": 1.905260443687439, |
| "reward_std": 0.018505161628127098, |
| "rewards/accuracy_reward": 0.9052603244781494, |
| "rewards/format_reward": 1.0, |
| "step": 676 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.61538696289062, |
| "epoch": 9.955882352941176, |
| "grad_norm": 1.7952493866544998, |
| "kl": 0.048095703125, |
| "learning_rate": 5.022058823529411e-07, |
| "loss": 0.0019, |
| "reward": 1.9118279218673706, |
| "reward_std": 0.01776190474629402, |
| "rewards/accuracy_reward": 0.9118278622627258, |
| "rewards/format_reward": 1.0, |
| "step": 677 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.45433044433594, |
| "epoch": 9.970588235294118, |
| "grad_norm": 1.7192070770567416, |
| "kl": 0.059814453125, |
| "learning_rate": 5.014705882352941e-07, |
| "loss": 0.0024, |
| "reward": 1.9066780805587769, |
| "reward_std": 0.01788855344057083, |
| "rewards/accuracy_reward": 0.9066779613494873, |
| "rewards/format_reward": 1.0, |
| "step": 678 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.34856414794922, |
| "epoch": 9.985294117647058, |
| "grad_norm": 2.906612944219061, |
| "kl": 0.06005859375, |
| "learning_rate": 5.00735294117647e-07, |
| "loss": 0.0024, |
| "reward": 1.9021302461624146, |
| "reward_std": 0.022055475041270256, |
| "rewards/accuracy_reward": 0.9021300673484802, |
| "rewards/format_reward": 1.0, |
| "step": 679 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.98021697998047, |
| "epoch": 10.0, |
| "grad_norm": 1.6562395375859271, |
| "kl": 0.047607421875, |
| "learning_rate": 5e-07, |
| "loss": 0.0019, |
| "reward": 1.9186270236968994, |
| "reward_std": 0.019354015588760376, |
| "rewards/accuracy_reward": 0.9201489090919495, |
| "rewards/format_reward": 0.9984779357910156, |
| "step": 680 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.89423370361328, |
| "epoch": 10.014705882352942, |
| "grad_norm": 2.9068911534041764, |
| "kl": 0.05712890625, |
| "learning_rate": 4.99264705882353e-07, |
| "loss": 0.0023, |
| "reward": 1.907537817955017, |
| "reward_std": 0.01922653429210186, |
| "rewards/accuracy_reward": 0.9075377583503723, |
| "rewards/format_reward": 1.0, |
| "step": 681 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.22356414794922, |
| "epoch": 10.029411764705882, |
| "grad_norm": 3.5541747826461805, |
| "kl": 0.055419921875, |
| "learning_rate": 4.985294117647058e-07, |
| "loss": 0.0022, |
| "reward": 1.917733073234558, |
| "reward_std": 0.01717350259423256, |
| "rewards/accuracy_reward": 0.9177329540252686, |
| "rewards/format_reward": 1.0, |
| "step": 682 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.38341522216797, |
| "epoch": 10.044117647058824, |
| "grad_norm": 1.903386359782145, |
| "kl": 0.057373046875, |
| "learning_rate": 4.977941176470588e-07, |
| "loss": 0.0023, |
| "reward": 1.9117200374603271, |
| "reward_std": 0.01899932697415352, |
| "rewards/accuracy_reward": 0.9129219651222229, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 683 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.29688262939453, |
| "epoch": 10.058823529411764, |
| "grad_norm": 1.8861273273154469, |
| "kl": 0.05810546875, |
| "learning_rate": 4.970588235294118e-07, |
| "loss": 0.0023, |
| "reward": 1.9124337434768677, |
| "reward_std": 0.02022276073694229, |
| "rewards/accuracy_reward": 0.9136357307434082, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 684 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.86058044433594, |
| "epoch": 10.073529411764707, |
| "grad_norm": 2.8430111341562823, |
| "kl": 0.05078125, |
| "learning_rate": 4.963235294117647e-07, |
| "loss": 0.002, |
| "reward": 1.9203444719314575, |
| "reward_std": 0.016688454896211624, |
| "rewards/accuracy_reward": 0.9203445315361023, |
| "rewards/format_reward": 1.0, |
| "step": 685 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.33053588867188, |
| "epoch": 10.088235294117647, |
| "grad_norm": 2.803964635402021, |
| "kl": 0.05859375, |
| "learning_rate": 4.955882352941176e-07, |
| "loss": 0.0023, |
| "reward": 1.9130260944366455, |
| "reward_std": 0.019971825182437897, |
| "rewards/accuracy_reward": 0.9142280220985413, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 686 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.25481414794922, |
| "epoch": 10.102941176470589, |
| "grad_norm": 2.1290481897859066, |
| "kl": 0.05517578125, |
| "learning_rate": 4.948529411764706e-07, |
| "loss": 0.0022, |
| "reward": 1.9154521226882935, |
| "reward_std": 0.019178370013833046, |
| "rewards/accuracy_reward": 0.9166539311408997, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 687 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 115.15264892578125, |
| "epoch": 10.117647058823529, |
| "grad_norm": 2.2315042906400944, |
| "kl": 0.0556640625, |
| "learning_rate": 4.941176470588235e-07, |
| "loss": 0.0022, |
| "reward": 1.9171024560928345, |
| "reward_std": 0.015508743934333324, |
| "rewards/accuracy_reward": 0.9171023368835449, |
| "rewards/format_reward": 1.0, |
| "step": 688 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 114.18991088867188, |
| "epoch": 10.132352941176471, |
| "grad_norm": 8.899062168348006, |
| "kl": 0.05859375, |
| "learning_rate": 4.933823529411765e-07, |
| "loss": 0.0023, |
| "reward": 1.918565034866333, |
| "reward_std": 0.016680248081684113, |
| "rewards/accuracy_reward": 0.9185649156570435, |
| "rewards/format_reward": 1.0, |
| "step": 689 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 114.71154022216797, |
| "epoch": 10.147058823529411, |
| "grad_norm": 1.684126014415895, |
| "kl": 0.05712890625, |
| "learning_rate": 4.926470588235295e-07, |
| "loss": 0.0023, |
| "reward": 1.8880369663238525, |
| "reward_std": 0.01872415281832218, |
| "rewards/accuracy_reward": 0.8880367875099182, |
| "rewards/format_reward": 1.0, |
| "step": 690 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 114.60697174072266, |
| "epoch": 10.161764705882353, |
| "grad_norm": 2.3449846328839596, |
| "kl": 0.05712890625, |
| "learning_rate": 4.919117647058823e-07, |
| "loss": 0.0023, |
| "reward": 1.910075306892395, |
| "reward_std": 0.024183494970202446, |
| "rewards/accuracy_reward": 0.9124791622161865, |
| "rewards/format_reward": 0.9975962042808533, |
| "step": 691 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.82572174072266, |
| "epoch": 10.176470588235293, |
| "grad_norm": 3.23331569595653, |
| "kl": 0.06298828125, |
| "learning_rate": 4.911764705882352e-07, |
| "loss": 0.0025, |
| "reward": 1.9134478569030762, |
| "reward_std": 0.01637120544910431, |
| "rewards/accuracy_reward": 0.9134476184844971, |
| "rewards/format_reward": 1.0, |
| "step": 692 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.63702392578125, |
| "epoch": 10.191176470588236, |
| "grad_norm": 2.3313697919069027, |
| "kl": 0.054931640625, |
| "learning_rate": 4.904411764705882e-07, |
| "loss": 0.0022, |
| "reward": 1.905639410018921, |
| "reward_std": 0.016094274818897247, |
| "rewards/accuracy_reward": 0.9056394100189209, |
| "rewards/format_reward": 1.0, |
| "step": 693 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.92188262939453, |
| "epoch": 10.205882352941176, |
| "grad_norm": 2.6050118805934894, |
| "kl": 0.061767578125, |
| "learning_rate": 4.897058823529412e-07, |
| "loss": 0.0025, |
| "reward": 1.9092425107955933, |
| "reward_std": 0.017230795696377754, |
| "rewards/accuracy_reward": 0.9092425107955933, |
| "rewards/format_reward": 1.0, |
| "step": 694 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.80409240722656, |
| "epoch": 10.220588235294118, |
| "grad_norm": 2.25480395310822, |
| "kl": 0.05615234375, |
| "learning_rate": 4.88970588235294e-07, |
| "loss": 0.0023, |
| "reward": 1.9153361320495605, |
| "reward_std": 0.01601433753967285, |
| "rewards/accuracy_reward": 0.9153360724449158, |
| "rewards/format_reward": 1.0, |
| "step": 695 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.41226196289062, |
| "epoch": 10.235294117647058, |
| "grad_norm": 1.8257275879557004, |
| "kl": 0.05712890625, |
| "learning_rate": 4.88235294117647e-07, |
| "loss": 0.0023, |
| "reward": 1.9105335474014282, |
| "reward_std": 0.014649370685219765, |
| "rewards/accuracy_reward": 0.9105334877967834, |
| "rewards/format_reward": 1.0, |
| "step": 696 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.64063262939453, |
| "epoch": 10.25, |
| "grad_norm": 1.904503451542197, |
| "kl": 0.05712890625, |
| "learning_rate": 4.875e-07, |
| "loss": 0.0023, |
| "reward": 1.9262598752975464, |
| "reward_std": 0.01650756038725376, |
| "rewards/accuracy_reward": 0.9262598752975464, |
| "rewards/format_reward": 1.0, |
| "step": 697 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.5144271850586, |
| "epoch": 10.264705882352942, |
| "grad_norm": 2.0347260430337584, |
| "kl": 0.05712890625, |
| "learning_rate": 4.867647058823529e-07, |
| "loss": 0.0023, |
| "reward": 1.9039766788482666, |
| "reward_std": 0.01813569664955139, |
| "rewards/accuracy_reward": 0.9051785469055176, |
| "rewards/format_reward": 0.998798131942749, |
| "step": 698 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.98798370361328, |
| "epoch": 10.279411764705882, |
| "grad_norm": 3.903638499409672, |
| "kl": 0.055908203125, |
| "learning_rate": 4.860294117647058e-07, |
| "loss": 0.0022, |
| "reward": 1.9150954484939575, |
| "reward_std": 0.014970256946980953, |
| "rewards/accuracy_reward": 0.9150952696800232, |
| "rewards/format_reward": 1.0, |
| "step": 699 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.92308044433594, |
| "epoch": 10.294117647058824, |
| "grad_norm": 2.18266269445753, |
| "kl": 0.0595703125, |
| "learning_rate": 4.852941176470588e-07, |
| "loss": 0.0024, |
| "reward": 1.9265812635421753, |
| "reward_std": 0.013946098275482655, |
| "rewards/accuracy_reward": 0.9265811443328857, |
| "rewards/format_reward": 1.0, |
| "step": 700 |
| } |
| ], |
| "logging_steps": 1.0, |
| "max_steps": 1360, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 20, |
| "save_steps": 100, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 52, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
