{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.11428571428571428, "eval_steps": 500, "global_step": 100, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "completion_length": 2700.4271850585938, "cov_mean": -6.0587970438064076e-05, "cov_std": 0.35307812318205833, "entropy": 0.3681640625, "epoch": 0.001142857142857143, "grad_norm": 0.45492634177207947, "kl": 0.0, "learning_rate": 1e-07, "loss": -0.0382, "reward": 0.7604166893288493, "reward_std": 0.4268697127699852, "rewards/accuracy_reward": 0.25000001303851604, "rewards/format_reward": 0.5104166669771075, "step": 1, "w_high_ratio": 0.2200421690940857, "w_low_ratio": 0.03663695091381669, "w_max": 2.1593789756298065, "w_mean": 1.4711343348026276, "w_min": 6.525355682266089e-35, "w_std": 0.2659660503268242 }, { "completion_length": 3127.3958435058594, "cov_mean": -2.155053698515985e-05, "cov_std": 0.310540571808815, "entropy": 0.353515625, "epoch": 0.002285714285714286, "grad_norm": 0.5142282843589783, "kl": 0.0, "learning_rate": 2e-07, "loss": 0.0049, "reward": 0.6458333637565374, "reward_std": 0.4249730706214905, "rewards/accuracy_reward": 0.2812500102445483, "rewards/format_reward": 0.3645833386108279, "step": 2, "w_high_ratio": 0.05183619633316994, "w_low_ratio": 0.036958135198801756, "w_max": 1.8325217366218567, "w_mean": 1.2113382518291473, "w_min": 0.0, "w_std": 0.20957503467798233 }, { "completion_length": 3733.9583740234375, "cov_mean": -2.010253456319333e-05, "cov_std": 0.2787674851715565, "entropy": 0.451171875, "epoch": 0.0034285714285714284, "grad_norm": 0.28526851534843445, "kl": 4.315376281738281e-05, "learning_rate": 3e-07, "loss": 0.0391, "reward": 0.16666667070239782, "reward_std": 0.31075509265065193, "rewards/accuracy_reward": 0.031250000931322575, "rewards/format_reward": 0.13541666697710752, "step": 3, "w_high_ratio": 0.0, "w_low_ratio": 0.03727753087878227, "w_max": 1.426508367061615, "w_mean": 1.069275826215744, "w_min": 0.0, "w_std": 0.17276715114712715 }, { "completion_length": 2456.5104370117188, "cov_mean": 6.050579213479068e-05, "cov_std": 0.4746721386909485, "entropy": 0.404296875, "epoch": 0.004571428571428572, "grad_norm": 0.4348958432674408, "kl": 3.764033317565918e-05, "learning_rate": 4e-07, "loss": -0.0128, "reward": 0.8125000223517418, "reward_std": 0.4816601872444153, "rewards/accuracy_reward": 0.1354166679084301, "rewards/format_reward": 0.677083358168602, "step": 4, "w_high_ratio": 0.09904679656028748, "w_low_ratio": 0.03708651335909963, "w_max": 2.0615443289279938, "w_mean": 1.430301010608673, "w_min": 2.2357144385761694e-24, "w_std": 0.25941959768533707 }, { "completion_length": 3495.9584350585938, "cov_mean": -0.00011635808687060489, "cov_std": 0.43240153789520264, "entropy": 0.4658203125, "epoch": 0.005714285714285714, "grad_norm": 0.44890686869621277, "kl": 4.1797757148742676e-05, "learning_rate": 5e-07, "loss": -0.077, "reward": 0.38541667722165585, "reward_std": 0.41880372166633606, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.34375001303851604, "step": 5, "w_high_ratio": 0.057075776159763336, "w_low_ratio": 0.04613090865314007, "w_max": 1.8747033178806305, "w_mean": 1.2025023698806763, "w_min": 2.0185204017920427e-32, "w_std": 0.24863697960972786 }, { "completion_length": 3429.7604370117188, "cov_mean": 7.305060944418074e-05, "cov_std": 0.3747733160853386, "entropy": 0.4658203125, "epoch": 0.006857142857142857, "grad_norm": 0.323794424533844, "kl": 4.607439041137695e-05, "learning_rate": 6e-07, "loss": 0.0289, "reward": 0.4791666828095913, "reward_std": 0.4919559434056282, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.3125000111758709, "step": 6, "w_high_ratio": 0.0, "w_low_ratio": 0.045196511782705784, "w_max": 1.4718832075595856, "w_mean": 1.11675626039505, "w_min": 0.25, "w_std": 0.20970172435045242 }, { "completion_length": 3298.1875610351562, "cov_mean": 1.544893893878907e-05, "cov_std": 0.5214613378047943, "entropy": 0.3837890625, "epoch": 0.008, "grad_norm": 0.3830799460411072, "kl": 2.524256706237793e-05, "learning_rate": 7e-07, "loss": -0.0429, "reward": 0.802083358168602, "reward_std": 0.5995434895157814, "rewards/accuracy_reward": 0.2083333432674408, "rewards/format_reward": 0.5937500223517418, "step": 7, "w_high_ratio": 0.10538750886917114, "w_low_ratio": 0.0529888030141592, "w_max": 1.5837838649749756, "w_mean": 1.266415923833847, "w_min": 1.0439673559219887e-43, "w_std": 0.28801921010017395 }, { "completion_length": 2903.697998046875, "cov_mean": -8.183626050595194e-06, "cov_std": 0.2509700208902359, "entropy": 0.355712890625, "epoch": 0.009142857142857144, "grad_norm": 0.2550622820854187, "kl": 1.93864107131958e-05, "learning_rate": 8e-07, "loss": -0.0211, "reward": 0.7812500298023224, "reward_std": 0.3037422299385071, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.4479166865348816, "step": 8, "w_high_ratio": 0.152647502720356, "w_low_ratio": 0.020135570783168077, "w_max": 1.6893351376056671, "w_mean": 1.325122356414795, "w_min": 0.5, "w_std": 0.13287343829870224 }, { "completion_length": 3427.0001220703125, "cov_mean": 2.1584107344096992e-05, "cov_std": 0.41377831995487213, "entropy": 0.4404296875, "epoch": 0.010285714285714285, "grad_norm": 0.36070960760116577, "kl": 4.297494888305664e-05, "learning_rate": 9e-07, "loss": -0.0144, "reward": 0.4375000149011612, "reward_std": 0.5150812789797783, "rewards/accuracy_reward": 0.11458333395421505, "rewards/format_reward": 0.322916679084301, "step": 9, "w_high_ratio": 0.0, "w_low_ratio": 0.053231239318847656, "w_max": 1.665610283613205, "w_mean": 1.168578028678894, "w_min": 0.0, "w_std": 0.2691279575228691 }, { "completion_length": 2921.260498046875, "cov_mean": -0.00011126142680950579, "cov_std": 0.37009984999895096, "entropy": 0.34130859375, "epoch": 0.011428571428571429, "grad_norm": 0.463008314371109, "kl": 2.7805566787719727e-05, "learning_rate": 1e-06, "loss": -0.0304, "reward": 0.604166679084301, "reward_std": 0.49994633346796036, "rewards/accuracy_reward": 0.19791667349636555, "rewards/format_reward": 0.4062500111758709, "step": 10, "w_high_ratio": 0.10105682164430618, "w_low_ratio": 0.039817056618630886, "w_max": 2.203812062740326, "w_mean": 1.3322690725326538, "w_min": 0.0, "w_std": 0.24996469356119633 }, { "completion_length": 3739.4583740234375, "cov_mean": 1.6300052948281518e-05, "cov_std": 0.22599393874406815, "entropy": 0.3876953125, "epoch": 0.012571428571428572, "grad_norm": 0.2399490475654602, "kl": 2.893805503845215e-05, "learning_rate": 9.997258721585931e-07, "loss": 0.0375, "reward": 0.1979166716337204, "reward_std": 0.3496348336338997, "rewards/accuracy_reward": 0.07291666697710752, "rewards/format_reward": 0.12500000558793545, "step": 11, "w_high_ratio": 0.0, "w_low_ratio": 0.03310772683471441, "w_max": 1.3216401040554047, "w_mean": 1.05658358335495, "w_min": 0.25, "w_std": 0.13504411466419697 }, { "completion_length": 2672.3959045410156, "cov_mean": -1.72590916918125e-05, "cov_std": 0.4432706832885742, "entropy": 0.38525390625, "epoch": 0.013714285714285714, "grad_norm": 0.5923348665237427, "kl": 3.87728214263916e-05, "learning_rate": 9.989038226169207e-07, "loss": -0.0398, "reward": 0.7916666865348816, "reward_std": 0.44830071553587914, "rewards/accuracy_reward": 0.15625, "rewards/format_reward": 0.6354166865348816, "step": 12, "w_high_ratio": 0.18827838450670242, "w_low_ratio": 0.039877891540527344, "w_max": 2.278617113828659, "w_mean": 1.533039778470993, "w_min": 0.0, "w_std": 0.30229373276233673 }, { "completion_length": 3129.9168090820312, "cov_mean": 5.8586666455084924e-05, "cov_std": 0.3728199526667595, "entropy": 0.3896484375, "epoch": 0.014857142857142857, "grad_norm": 0.7794428467750549, "kl": 2.4527311325073242e-05, "learning_rate": 9.975348529157229e-07, "loss": -0.0268, "reward": 0.6145833507180214, "reward_std": 0.3987155184149742, "rewards/accuracy_reward": 0.1979166716337204, "rewards/format_reward": 0.4166666716337204, "step": 13, "w_high_ratio": 0.1489243432879448, "w_low_ratio": 0.03552113357000053, "w_max": 1.913963943719864, "w_mean": 1.3093808591365814, "w_min": 0.0, "w_std": 0.22065079398453236 }, { "completion_length": 2925.8750610351562, "cov_mean": 7.129182449716609e-06, "cov_std": 0.3760656714439392, "entropy": 0.353515625, "epoch": 0.016, "grad_norm": 0.6446069478988647, "kl": 2.3916363716125488e-05, "learning_rate": 9.956206309337066e-07, "loss": -0.0123, "reward": 0.6562500298023224, "reward_std": 0.41464143246412277, "rewards/accuracy_reward": 0.17708334233611822, "rewards/format_reward": 0.4791666716337204, "step": 14, "w_high_ratio": 0.04337232932448387, "w_low_ratio": 0.04529819544404745, "w_max": 1.892830729484558, "w_mean": 1.2675927579402924, "w_min": 7.264961823368798e-41, "w_std": 0.26389942318201065 }, { "completion_length": 2977.6458740234375, "cov_mean": 2.068816957034869e-05, "cov_std": 0.2604588344693184, "entropy": 0.3759765625, "epoch": 0.017142857142857144, "grad_norm": 0.4150949716567993, "kl": 3.0860304832458496e-05, "learning_rate": 9.931634888554935e-07, "loss": -0.0032, "reward": 0.6041666902601719, "reward_std": 0.25453949719667435, "rewards/accuracy_reward": 0.20833333861082792, "rewards/format_reward": 0.3958333358168602, "step": 15, "w_high_ratio": 0.0, "w_low_ratio": 0.027457581600174308, "w_max": 1.3896198272705078, "w_mean": 1.1120553016662598, "w_min": 0.0, "w_std": 0.1376371644437313 }, { "completion_length": 3860.1771850585938, "cov_mean": 3.589540312987083e-05, "cov_std": 0.1900031939148903, "entropy": 0.45751953125, "epoch": 0.018285714285714287, "grad_norm": 0.25400859117507935, "kl": 3.0994415283203125e-05, "learning_rate": 9.901664203302124e-07, "loss": 0.0468, "reward": 0.1458333395421505, "reward_std": 0.24955713748931885, "rewards/accuracy_reward": 0.06250000279396772, "rewards/format_reward": 0.08333333674818277, "step": 16, "w_high_ratio": 0.0, "w_low_ratio": 0.028868926223367453, "w_max": 1.2315952479839325, "w_mean": 1.0204393565654755, "w_min": 0.25, "w_std": 0.10705284960567951 }, { "completion_length": 2540.4895935058594, "cov_mean": -0.0001341435610129338, "cov_std": 0.47027114033699036, "entropy": 0.45166015625, "epoch": 0.019428571428571427, "grad_norm": 0.46505773067474365, "kl": 5.486607551574707e-05, "learning_rate": 9.866330768241983e-07, "loss": -0.0001, "reward": 0.812500037252903, "reward_std": 0.49661170691251755, "rewards/accuracy_reward": 0.2291666716337204, "rewards/format_reward": 0.5833333507180214, "step": 17, "w_high_ratio": 0.15490177273750305, "w_low_ratio": 0.04533949866890907, "w_max": 2.4269833266735077, "w_mean": 1.407887727022171, "w_min": 6.344169701078957e-36, "w_std": 0.2766948938369751 }, { "completion_length": 3100.604248046875, "cov_mean": 9.316997420683037e-06, "cov_std": 0.345074113458395, "entropy": 0.36865234375, "epoch": 0.02057142857142857, "grad_norm": 0.26780965924263, "kl": 3.306567668914795e-05, "learning_rate": 9.825677631722435e-07, "loss": 0.0093, "reward": 0.5208333507180214, "reward_std": 0.4629998616874218, "rewards/accuracy_reward": 0.11458333861082792, "rewards/format_reward": 0.4062500074505806, "step": 18, "w_high_ratio": 0.12114312499761581, "w_low_ratio": 0.046439004596322775, "w_max": 2.0130829215049744, "w_mean": 1.2991048991680145, "w_min": 0.0, "w_std": 0.22966066375374794 }, { "completion_length": 3111.229248046875, "cov_mean": -1.3287149158713873e-10, "cov_std": 0.305859737098217, "entropy": 0.37939453125, "epoch": 0.021714285714285714, "grad_norm": 0.2747480869293213, "kl": 4.106760025024414e-05, "learning_rate": 9.779754323328192e-07, "loss": -0.0265, "reward": 0.8437500251457095, "reward_std": 0.44304458051919937, "rewards/accuracy_reward": 0.3854166818782687, "rewards/format_reward": 0.4583333535119891, "step": 19, "w_high_ratio": 0.007512019481509924, "w_low_ratio": 0.03380460711196065, "w_max": 1.7184137105941772, "w_mean": 1.2587754130363464, "w_min": 0.0, "w_std": 0.17892248183488846 }, { "completion_length": 2602.0000610351562, "cov_mean": -4.431823981576599e-06, "cov_std": 0.3394951745867729, "entropy": 0.334716796875, "epoch": 0.022857142857142857, "grad_norm": 0.2722320854663849, "kl": 7.30752944946289e-05, "learning_rate": 9.728616793536587e-07, "loss": -0.016, "reward": 0.989583358168602, "reward_std": 0.41970477253198624, "rewards/accuracy_reward": 0.2812500074505806, "rewards/format_reward": 0.708333358168602, "step": 20, "w_high_ratio": 0.053037188947200775, "w_low_ratio": 0.03638318553566933, "w_max": 1.8807734847068787, "w_mean": 1.3341735899448395, "w_min": 0.28736552596092224, "w_std": 0.21230074390769005 }, { "completion_length": 2938.9896240234375, "cov_mean": 1.115940267482074e-05, "cov_std": 0.3257220685482025, "entropy": 0.42919921875, "epoch": 0.024, "grad_norm": 0.9565598964691162, "kl": 0.00023573637008666992, "learning_rate": 9.672327345550543e-07, "loss": 0.0067, "reward": 0.604166692122817, "reward_std": 0.4190382584929466, "rewards/accuracy_reward": 0.17708333674818277, "rewards/format_reward": 0.4270833460614085, "step": 21, "w_high_ratio": 0.2210865318775177, "w_low_ratio": 0.03242270671762526, "w_max": 1.8820680975914001, "w_mean": 1.4674345552921295, "w_min": 2.9579121420725567e-37, "w_std": 0.23061984032392502 }, { "completion_length": 1817.8125305175781, "cov_mean": 3.471795207588002e-05, "cov_std": 0.4554404318332672, "entropy": 0.37548828125, "epoch": 0.025142857142857144, "grad_norm": 0.6403908729553223, "kl": 0.0001913309097290039, "learning_rate": 9.610954559391704e-07, "loss": -0.0476, "reward": 1.1041667014360428, "reward_std": 0.36877378076314926, "rewards/accuracy_reward": 0.28125000558793545, "rewards/format_reward": 0.8229166716337204, "step": 22, "w_high_ratio": 0.3237294889986515, "w_low_ratio": 0.03283138480037451, "w_max": 3.265183061361313, "w_mean": 1.977644294500351, "w_min": 5.22007475166266e-30, "w_std": 0.34112248569726944 }, { "completion_length": 2787.4896850585938, "cov_mean": 3.710087821673369e-05, "cov_std": 0.3974616304039955, "entropy": 0.38525390625, "epoch": 0.026285714285714287, "grad_norm": 0.6307149529457092, "kl": 0.00013697147369384766, "learning_rate": 9.54457320834625e-07, "loss": -0.0959, "reward": 0.6041666716337204, "reward_std": 0.4507869631052017, "rewards/accuracy_reward": 0.14583333395421505, "rewards/format_reward": 0.4583333358168602, "step": 23, "w_high_ratio": 0.18567950278520584, "w_low_ratio": 0.042684963904321194, "w_max": 2.490270048379898, "w_mean": 1.4617709517478943, "w_min": 1.9980297628321344e-34, "w_std": 0.30704520270228386 }, { "completion_length": 3070.3125610351562, "cov_mean": 5.4191557865124196e-05, "cov_std": 0.5094943791627884, "entropy": 0.367431640625, "epoch": 0.027428571428571427, "grad_norm": 0.5826663374900818, "kl": 0.0001633763313293457, "learning_rate": 9.473264167865171e-07, "loss": -0.02, "reward": 0.8229166939854622, "reward_std": 0.5960981249809265, "rewards/accuracy_reward": 0.2708333358168602, "rewards/format_reward": 0.5520833507180214, "step": 24, "w_high_ratio": 0.16660767421126366, "w_low_ratio": 0.05264063738286495, "w_max": 2.2747649252414703, "w_mean": 1.4129657745361328, "w_min": 0.0, "w_std": 0.3425128310918808 }, { "completion_length": 3037.8125, "cov_mean": -3.0258611332101282e-05, "cov_std": 0.2974156364798546, "entropy": 0.4462890625, "epoch": 0.02857142857142857, "grad_norm": 0.33458662033081055, "kl": 0.00022605061531066895, "learning_rate": 9.397114317029974e-07, "loss": 0.0202, "reward": 0.6875000149011612, "reward_std": 0.4009811282157898, "rewards/accuracy_reward": 0.2395833432674408, "rewards/format_reward": 0.4479166865348816, "step": 25, "w_high_ratio": 0.1122930608689785, "w_low_ratio": 0.034513187827542424, "w_max": 1.991903692483902, "w_mean": 1.2860748171806335, "w_min": 0.25, "w_std": 0.2155727706849575 }, { "completion_length": 3117.4584350585938, "cov_mean": -8.455652073280362e-05, "cov_std": 0.23132415860891342, "entropy": 0.42626953125, "epoch": 0.029714285714285714, "grad_norm": 0.41856279969215393, "kl": 6.198883056640625e-05, "learning_rate": 9.316216432703916e-07, "loss": -0.053, "reward": 0.6562500074505806, "reward_std": 0.20796867460012436, "rewards/accuracy_reward": 0.1979166716337204, "rewards/format_reward": 0.4583333432674408, "step": 26, "w_high_ratio": 0.04478337988257408, "w_low_ratio": 0.011211750796064734, "w_max": 1.7470227479934692, "w_mean": 1.224214792251587, "w_min": 0.25, "w_std": 0.10643218830227852 }, { "completion_length": 3314.1041870117188, "cov_mean": 2.3687035081820795e-05, "cov_std": 0.44449782371520996, "entropy": 0.44921875, "epoch": 0.030857142857142857, "grad_norm": 0.3447716236114502, "kl": 0.0005403757095336914, "learning_rate": 9.230669076497687e-07, "loss": -0.0255, "reward": 0.6354166865348816, "reward_std": 0.5543549209833145, "rewards/accuracy_reward": 0.1875000037252903, "rewards/format_reward": 0.447916679084301, "step": 27, "w_high_ratio": 0.04345417395234108, "w_low_ratio": 0.047908838372677565, "w_max": 1.680276095867157, "w_mean": 1.1971288323402405, "w_min": 0.0, "w_std": 0.23072999715805054 }, { "completion_length": 3026.5626220703125, "cov_mean": 4.19466396124335e-05, "cov_std": 0.4268927723169327, "entropy": 0.4091796875, "epoch": 0.032, "grad_norm": 0.4165506958961487, "kl": 0.00021666288375854492, "learning_rate": 9.140576474687263e-07, "loss": -0.0277, "reward": 0.8437500298023224, "reward_std": 0.4840777963399887, "rewards/accuracy_reward": 0.3750000149011612, "rewards/format_reward": 0.4687500149011612, "step": 28, "w_high_ratio": 0.09859117493033409, "w_low_ratio": 0.04558967752382159, "w_max": 2.2461937069892883, "w_mean": 1.3320654332637787, "w_min": 0.0, "w_std": 0.2779119908809662 }, { "completion_length": 3620.1563110351562, "cov_mean": -3.5699086538443225e-05, "cov_std": 0.34214310348033905, "entropy": 0.43017578125, "epoch": 0.03314285714285714, "grad_norm": 0.3975992202758789, "kl": 0.0004680156707763672, "learning_rate": 9.046048391230247e-07, "loss": 0.0814, "reward": 0.2291666716337204, "reward_std": 0.39027639478445053, "rewards/accuracy_reward": 0.0625, "rewards/format_reward": 0.1666666716337204, "step": 29, "w_high_ratio": 0.0, "w_low_ratio": 0.04788992367684841, "w_max": 1.3650976717472076, "w_mean": 1.0994165241718292, "w_min": 0.25, "w_std": 0.22099098190665245 }, { "completion_length": 3238.17724609375, "cov_mean": 5.147313459019642e-06, "cov_std": 0.466504842042923, "entropy": 0.38134765625, "epoch": 0.03428571428571429, "grad_norm": 0.3571697175502777, "kl": 0.0008358955383300781, "learning_rate": 8.9471999940354e-07, "loss": 0.0066, "reward": 0.7395833693444729, "reward_std": 0.6347565650939941, "rewards/accuracy_reward": 0.2708333423361182, "rewards/format_reward": 0.4687500074505806, "step": 30, "w_high_ratio": 0.09309378266334534, "w_low_ratio": 0.04863026458770037, "w_max": 1.7810659110546112, "w_mean": 1.2216298282146454, "w_min": 0.0, "w_std": 0.26985964179039 }, { "completion_length": 3252.6771240234375, "cov_mean": -2.1283684873196762e-05, "cov_std": 0.31266890093684196, "entropy": 0.376953125, "epoch": 0.03542857142857143, "grad_norm": 0.28267669677734375, "kl": 0.00033402442932128906, "learning_rate": 8.844151714648274e-07, "loss": 0.0136, "reward": 0.53125, "reward_std": 0.3965660147368908, "rewards/accuracy_reward": 0.1979166679084301, "rewards/format_reward": 0.3333333358168602, "step": 31, "w_high_ratio": 0.13101918064057827, "w_low_ratio": 0.03200887469574809, "w_max": 1.9720379710197449, "w_mean": 1.2918123006820679, "w_min": 0.25, "w_std": 0.22479910403490067 }, { "completion_length": 3411.0208740234375, "cov_mean": -2.1311116142896935e-06, "cov_std": 0.37892643362283707, "entropy": 0.439453125, "epoch": 0.036571428571428574, "grad_norm": 0.245382159948349, "kl": 0.00054168701171875, "learning_rate": 8.737029101523929e-07, "loss": 0.0361, "reward": 0.6666666716337204, "reward_std": 0.5180360972881317, "rewards/accuracy_reward": 0.2916666669771075, "rewards/format_reward": 0.3750000037252903, "step": 32, "w_high_ratio": 0.0408792570233345, "w_low_ratio": 0.0450265696272254, "w_max": 1.5775762796401978, "w_mean": 1.1778713762760162, "w_min": 0.0, "w_std": 0.22701822221279144 }, { "completion_length": 3525.760498046875, "cov_mean": -4.6630036194983404e-05, "cov_std": 0.3494578003883362, "entropy": 0.37939453125, "epoch": 0.037714285714285714, "grad_norm": 0.2607223689556122, "kl": 0.0014134645462036133, "learning_rate": 8.625962667065487e-07, "loss": 0.0251, "reward": 0.47916669212281704, "reward_std": 0.5007270351052284, "rewards/accuracy_reward": 0.18750001024454832, "rewards/format_reward": 0.2916666818782687, "step": 33, "w_high_ratio": 0.038857411593198776, "w_low_ratio": 0.04762307275086641, "w_max": 1.5454095602035522, "w_mean": 1.1393149495124817, "w_min": 0.0, "w_std": 0.22297161258757114 }, { "completion_length": 2453.9791870117188, "cov_mean": -9.226416841556784e-05, "cov_std": 0.4420240595936775, "entropy": 0.43701171875, "epoch": 0.038857142857142854, "grad_norm": 0.5145274996757507, "kl": 0.0017969608306884766, "learning_rate": 8.511087728614862e-07, "loss": -0.018, "reward": 0.9583333684131503, "reward_std": 0.4463650956749916, "rewards/accuracy_reward": 0.375000006519258, "rewards/format_reward": 0.5833333432674408, "step": 34, "w_high_ratio": 0.13110801950097084, "w_low_ratio": 0.052741190418601036, "w_max": 2.1099403500556946, "w_mean": 1.343121200799942, "w_min": 0.0, "w_std": 0.31580859050154686 }, { "completion_length": 3289.4376220703125, "cov_mean": 5.6672024584258907e-05, "cov_std": 0.3820243999361992, "entropy": 0.42529296875, "epoch": 0.04, "grad_norm": 0.35655274987220764, "kl": 0.0010710954666137695, "learning_rate": 8.392544243589427e-07, "loss": 0.0185, "reward": 0.6458333432674408, "reward_std": 0.5987429097294807, "rewards/accuracy_reward": 0.2708333395421505, "rewards/format_reward": 0.3750000037252903, "step": 35, "w_high_ratio": 0.0, "w_low_ratio": 0.05319366045296192, "w_max": 1.4592802226543427, "w_mean": 1.0992730259895325, "w_min": 3.853570776893247e-45, "w_std": 0.22482672333717346 }, { "completion_length": 3621.78125, "cov_mean": 7.363205986621324e-05, "cov_std": 0.19353903084993362, "entropy": 0.4765625, "epoch": 0.04114285714285714, "grad_norm": 0.30453404784202576, "kl": 0.001312553882598877, "learning_rate": 8.270476638965461e-07, "loss": -0.0147, "reward": 0.1666666716337204, "reward_std": 0.17689276114106178, "rewards/accuracy_reward": 0.010416666977107525, "rewards/format_reward": 0.15625, "step": 36, "w_high_ratio": 0.006425275932997465, "w_low_ratio": 0.022115232422947884, "w_max": 1.4147455394268036, "w_mean": 1.117747575044632, "w_min": 0.5, "w_std": 0.1310933344066143 }, { "completion_length": 3511.2813110351562, "cov_mean": -8.677186542627169e-06, "cov_std": 0.2782720513641834, "entropy": 0.43408203125, "epoch": 0.04228571428571429, "grad_norm": 0.357716828584671, "kl": 0.0007710456848144531, "learning_rate": 8.145033635316128e-07, "loss": -0.0037, "reward": 0.21875000279396772, "reward_std": 0.20087094232439995, "rewards/accuracy_reward": 0.010416666977107525, "rewards/format_reward": 0.20833333861082792, "step": 37, "w_high_ratio": 0.07502041757106781, "w_low_ratio": 0.02649637684226036, "w_max": 1.848203331232071, "w_mean": 1.2203471958637238, "w_min": 0.25, "w_std": 0.18464426696300507 }, { "completion_length": 3627.8438110351562, "cov_mean": -6.837155524408445e-05, "cov_std": 0.14785446226596832, "entropy": 0.44921875, "epoch": 0.04342857142857143, "grad_norm": 0.28619271516799927, "kl": 0.0005115270614624023, "learning_rate": 8.01636806561836e-07, "loss": 0.0051, "reward": 0.3645833358168602, "reward_std": 0.25648824870586395, "rewards/accuracy_reward": 0.1770833395421505, "rewards/format_reward": 0.1875000111758709, "step": 38, "w_high_ratio": 0.051015257835388184, "w_low_ratio": 0.01901705004274845, "w_max": 1.441689670085907, "w_mean": 1.1396986544132233, "w_min": 0.5, "w_std": 0.09881949052214622 }, { "completion_length": 3050.979248046875, "cov_mean": 4.009085705547477e-05, "cov_std": 0.20474716648459435, "entropy": 0.366943359375, "epoch": 0.044571428571428574, "grad_norm": 0.2579488158226013, "kl": 0.0016760826110839844, "learning_rate": 7.884636689049422e-07, "loss": -0.0486, "reward": 0.7083333358168602, "reward_std": 0.2621734067797661, "rewards/accuracy_reward": 0.21875, "rewards/format_reward": 0.4895833358168602, "step": 39, "w_high_ratio": 0.04251348227262497, "w_low_ratio": 0.022377901477739215, "w_max": 1.5512709319591522, "w_mean": 1.1656133532524109, "w_min": 0.25, "w_std": 0.12140309438109398 }, { "completion_length": 2983.6876220703125, "cov_mean": -2.4692089027666952e-05, "cov_std": 0.3223262317478657, "entropy": 0.39697265625, "epoch": 0.045714285714285714, "grad_norm": 0.3634110987186432, "kl": 0.003978729248046875, "learning_rate": 7.75e-07, "loss": 0.0235, "reward": 0.7083333507180214, "reward_std": 0.39669927954673767, "rewards/accuracy_reward": 0.20833334140479565, "rewards/format_reward": 0.5000000149011612, "step": 40, "w_high_ratio": 0.023237863555550575, "w_low_ratio": 0.03736424632370472, "w_max": 1.7330412566661835, "w_mean": 1.253759890794754, "w_min": 0.0, "w_std": 0.2265719212591648 }, { "completion_length": 3342.916748046875, "cov_mean": 7.446667950716801e-06, "cov_std": 0.36464616656303406, "entropy": 0.38232421875, "epoch": 0.046857142857142854, "grad_norm": 0.3088231682777405, "kl": 0.0006549358367919922, "learning_rate": 7.612622032536507e-07, "loss": 0.01, "reward": 0.416666679084301, "reward_std": 0.49482717365026474, "rewards/accuracy_reward": 0.10416667070239782, "rewards/format_reward": 0.3125000037252903, "step": 41, "w_high_ratio": 0.02740619331598282, "w_low_ratio": 0.045960213989019394, "w_max": 1.735625982284546, "w_mean": 1.2091110348701477, "w_min": 0.0, "w_std": 0.24962139129638672 }, { "completion_length": 3026.229248046875, "cov_mean": 2.7950623916694894e-05, "cov_std": 0.22710799425840378, "entropy": 0.46923828125, "epoch": 0.048, "grad_norm": 0.188913494348526, "kl": 0.0005321502685546875, "learning_rate": 7.472670160550848e-07, "loss": 0.0293, "reward": 0.36458334140479565, "reward_std": 0.2087036520242691, "rewards/accuracy_reward": 0.010416666977107525, "rewards/format_reward": 0.3541666744276881, "step": 42, "w_high_ratio": 0.0, "w_low_ratio": 0.03042191034182906, "w_max": 1.318351686000824, "w_mean": 1.0878586322069168, "w_min": 0.25, "w_std": 0.1288738138973713 }, { "completion_length": 3391.8333740234375, "cov_mean": 1.0224231118627358e-05, "cov_std": 0.3281352184712887, "entropy": 0.421875, "epoch": 0.04914285714285714, "grad_norm": 0.3027113378047943, "kl": 0.0014162063598632812, "learning_rate": 7.330314893841101e-07, "loss": 0.0293, "reward": 0.4479166716337204, "reward_std": 0.4176873341202736, "rewards/accuracy_reward": 0.18750000558793545, "rewards/format_reward": 0.2604166716337204, "step": 43, "w_high_ratio": 0.0, "w_low_ratio": 0.0375382611528039, "w_max": 1.406785786151886, "w_mean": 1.0772785246372223, "w_min": 0.0, "w_std": 0.17778386548161507 }, { "completion_length": 2956.250030517578, "cov_mean": -4.8457440243510064e-05, "cov_std": 0.4899379573762417, "entropy": 0.3984375, "epoch": 0.05028571428571429, "grad_norm": 0.4807187020778656, "kl": 0.001889944076538086, "learning_rate": 7.185729670371604e-07, "loss": 0.0382, "reward": 0.7395833684131503, "reward_std": 0.4934317544102669, "rewards/accuracy_reward": 0.2395833432674408, "rewards/format_reward": 0.5000000176951289, "step": 44, "w_high_ratio": 0.09932601824402809, "w_low_ratio": 0.052049466874450445, "w_max": 2.26826411485672, "w_mean": 1.3322851359844208, "w_min": 0.0, "w_std": 0.30705559998750687 }, { "completion_length": 3707.854248046875, "cov_mean": -4.304830235923873e-05, "cov_std": 0.290540874004364, "entropy": 0.4033203125, "epoch": 0.05142857142857143, "grad_norm": 0.16171735525131226, "kl": 0.002239227294921875, "learning_rate": 7.039090644965509e-07, "loss": 0.0065, "reward": 0.5000000279396772, "reward_std": 0.4716969132423401, "rewards/accuracy_reward": 0.21875000279396772, "rewards/format_reward": 0.2812500102445483, "step": 45, "w_high_ratio": 0.0, "w_low_ratio": 0.03661782294511795, "w_max": 1.3389239609241486, "w_mean": 1.0591736733913422, "w_min": 0.0, "w_std": 0.1518448255956173 }, { "completion_length": 3539.2188110351562, "cov_mean": 0.00013026446868025232, "cov_std": 0.34402996301651, "entropy": 0.5166015625, "epoch": 0.052571428571428575, "grad_norm": 0.28467056155204773, "kl": 0.0012564659118652344, "learning_rate": 6.890576474687263e-07, "loss": 0.0542, "reward": 0.2395833395421505, "reward_std": 0.31017982587218285, "rewards/accuracy_reward": 0.041666666977107525, "rewards/format_reward": 0.1979166716337204, "step": 46, "w_high_ratio": 0.038347791880369186, "w_low_ratio": 0.04659654130227864, "w_max": 1.562936246395111, "w_mean": 1.0923520624637604, "w_min": 6.825020352992891e-31, "w_std": 0.19784706830978394 }, { "completion_length": 3168.5313110351562, "cov_mean": -1.4116008060227614e-05, "cov_std": 0.3409600518643856, "entropy": 0.4072265625, "epoch": 0.053714285714285714, "grad_norm": 0.21553435921669006, "kl": 0.0014462471008300781, "learning_rate": 6.740368101176495e-07, "loss": 0.0063, "reward": 0.7083333432674408, "reward_std": 0.505032941699028, "rewards/accuracy_reward": 0.28125, "rewards/format_reward": 0.4270833358168602, "step": 47, "w_high_ratio": 0.1824447140097618, "w_low_ratio": 0.0349124651402235, "w_max": 1.8283918797969818, "w_mean": 1.3520298600196838, "w_min": 0.25, "w_std": 0.17590370774269104 }, { "completion_length": 3045.5209350585938, "cov_mean": 1.4324002222565468e-05, "cov_std": 0.41333913803100586, "entropy": 0.40966796875, "epoch": 0.054857142857142854, "grad_norm": 0.514603316783905, "kl": 0.0032389163970947266, "learning_rate": 6.588648530198504e-07, "loss": -0.0291, "reward": 0.5729166902601719, "reward_std": 0.4265441298484802, "rewards/accuracy_reward": 0.20833334419876337, "rewards/format_reward": 0.3645833460614085, "step": 48, "w_high_ratio": 0.14455852285027504, "w_low_ratio": 0.04336892580613494, "w_max": 2.106689363718033, "w_mean": 1.3398694097995758, "w_min": 0.0, "w_std": 0.29295507818460464 }, { "completion_length": 2561.3646850585938, "cov_mean": -5.681005814039963e-05, "cov_std": 0.3279518857598305, "entropy": 0.36865234375, "epoch": 0.056, "grad_norm": 0.25349730253219604, "kl": 0.0052852630615234375, "learning_rate": 6.435602608679916e-07, "loss": 0.0033, "reward": 0.833333358168602, "reward_std": 0.46440815925598145, "rewards/accuracy_reward": 0.21875001303851604, "rewards/format_reward": 0.6145833432674408, "step": 49, "w_high_ratio": 0.02398517169058323, "w_low_ratio": 0.04019799269735813, "w_max": 1.7071016132831573, "w_mean": 1.2629328966140747, "w_min": 0.25, "w_std": 0.19682494550943375 }, { "completion_length": 3245.2083740234375, "cov_mean": 4.621147024863603e-05, "cov_std": 0.35216138139367104, "entropy": 0.35205078125, "epoch": 0.05714285714285714, "grad_norm": 0.20161262154579163, "kl": 0.0010981559753417969, "learning_rate": 6.281416799501187e-07, "loss": 0.014, "reward": 0.6354166967794299, "reward_std": 0.420846089720726, "rewards/accuracy_reward": 0.2812500149011612, "rewards/format_reward": 0.3541666818782687, "step": 50, "w_high_ratio": 0.0, "w_low_ratio": 0.042036826722323895, "w_max": 1.5023921430110931, "w_mean": 1.146658033132553, "w_min": 0.0, "w_std": 0.206118730828166 }, { "completion_length": 2383.6980590820312, "cov_mean": -2.697939362406032e-05, "cov_std": 0.38119756057858467, "entropy": 0.42822265625, "epoch": 0.05828571428571429, "grad_norm": 0.4689660966396332, "kl": 0.0037221908569335938, "learning_rate": 6.126278954320294e-07, "loss": 0.011, "reward": 0.7604166865348816, "reward_std": 0.39414726570248604, "rewards/accuracy_reward": 0.20833333861082792, "rewards/format_reward": 0.5520833432674408, "step": 51, "w_high_ratio": 0.0406239777803421, "w_low_ratio": 0.04253132501617074, "w_max": 1.5954174101352692, "w_mean": 1.217557042837143, "w_min": 0.0, "w_std": 0.24302981421351433 }, { "completion_length": 3188.3021240234375, "cov_mean": -1.9340616745466832e-05, "cov_std": 0.42421338707208633, "entropy": 0.41259765625, "epoch": 0.05942857142857143, "grad_norm": 0.43333640694618225, "kl": 0.0021028518676757812, "learning_rate": 5.97037808470444e-07, "loss": -0.0265, "reward": 0.7500000298023224, "reward_std": 0.56574647128582, "rewards/accuracy_reward": 0.3229166716337204, "rewards/format_reward": 0.4270833507180214, "step": 52, "w_high_ratio": 0.0, "w_low_ratio": 0.04618662968277931, "w_max": 1.4080191552639008, "w_mean": 1.1331122517585754, "w_min": 0.25, "w_std": 0.21432508528232574 }, { "completion_length": 2943.291748046875, "cov_mean": 1.8985145061378716e-05, "cov_std": 0.42384568601846695, "entropy": 0.41259765625, "epoch": 0.060571428571428575, "grad_norm": 0.28105950355529785, "kl": 0.001844644546508789, "learning_rate": 5.813904131848564e-07, "loss": 0.0064, "reward": 0.895833358168602, "reward_std": 0.5318989902734756, "rewards/accuracy_reward": 0.30208334885537624, "rewards/format_reward": 0.5937500149011612, "step": 53, "w_high_ratio": 0.0625, "w_low_ratio": 0.04191893618553877, "w_max": 1.6293854713439941, "w_mean": 1.2208082973957062, "w_min": 0.25, "w_std": 0.20713179931044579 }, { "completion_length": 3035.8438110351562, "cov_mean": 6.566583215317223e-05, "cov_std": 0.39858507737517357, "entropy": 0.3798828125, "epoch": 0.061714285714285715, "grad_norm": 0.38175392150878906, "kl": 0.0011749267578125, "learning_rate": 5.657047735161255e-07, "loss": -0.0521, "reward": 0.9791667088866234, "reward_std": 0.5917666852474213, "rewards/accuracy_reward": 0.42708334885537624, "rewards/format_reward": 0.552083358168602, "step": 54, "w_high_ratio": 0.044216301292181015, "w_low_ratio": 0.03513730876147747, "w_max": 1.7533048391342163, "w_mean": 1.3249602317810059, "w_min": 0.0, "w_std": 0.22004129365086555 }, { "completion_length": 3354.666748046875, "cov_mean": -3.56279870175058e-05, "cov_std": 0.30405301228165627, "entropy": 0.419921875, "epoch": 0.06285714285714286, "grad_norm": 0.3018040060997009, "kl": 0.0012395381927490234, "learning_rate": 5.5e-07, "loss": -0.0185, "reward": 0.5208333469927311, "reward_std": 0.3702617585659027, "rewards/accuracy_reward": 0.1979166716337204, "rewards/format_reward": 0.3229166679084301, "step": 55, "w_high_ratio": 0.0, "w_low_ratio": 0.03356400621123612, "w_max": 1.4872715771198273, "w_mean": 1.1252524256706238, "w_min": 0.0, "w_std": 0.17813249677419662 }, { "completion_length": 3199.7188720703125, "cov_mean": -1.4941413155611372e-05, "cov_std": 0.25671473518013954, "entropy": 0.40185546875, "epoch": 0.064, "grad_norm": 0.21876569092273712, "kl": 0.0006089210510253906, "learning_rate": 5.342952264838747e-07, "loss": -0.0325, "reward": 0.6458333395421505, "reward_std": 0.27983931452035904, "rewards/accuracy_reward": 0.1979166716337204, "rewards/format_reward": 0.4479166828095913, "step": 56, "w_high_ratio": 0.0, "w_low_ratio": 0.028304541716352105, "w_max": 1.453190177679062, "w_mean": 1.1167612671852112, "w_min": 1.400437147815929e-36, "w_std": 0.14085076563060284 }, { "completion_length": 3545.4271850585938, "cov_mean": 1.6042108654801268e-05, "cov_std": 0.19959648326039314, "entropy": 0.31787109375, "epoch": 0.06514285714285714, "grad_norm": 0.15603993833065033, "kl": 0.0003676414489746094, "learning_rate": 5.186095868151436e-07, "loss": 0.0067, "reward": 0.4270833395421505, "reward_std": 0.26762784644961357, "rewards/accuracy_reward": 0.13541667256504297, "rewards/format_reward": 0.29166667722165585, "step": 57, "w_high_ratio": 0.0, "w_low_ratio": 0.028780631721019745, "w_max": 1.3290077447891235, "w_mean": 1.1172043681144714, "w_min": 0.25, "w_std": 0.12668126448988914 }, { "completion_length": 2554.1458740234375, "cov_mean": -8.6602381998091e-05, "cov_std": 0.4178408235311508, "entropy": 0.333740234375, "epoch": 0.06628571428571428, "grad_norm": 0.4292741119861603, "kl": 0.0062389373779296875, "learning_rate": 5.02962191529556e-07, "loss": -0.045, "reward": 0.9375000447034836, "reward_std": 0.5022420063614845, "rewards/accuracy_reward": 0.28125000558793545, "rewards/format_reward": 0.6562500223517418, "step": 58, "w_high_ratio": 0.08225274085998535, "w_low_ratio": 0.038829353637993336, "w_max": 1.8886817693710327, "w_mean": 1.3429239392280579, "w_min": 0.0, "w_std": 0.24752679839730263 }, { "completion_length": 3169.0625610351562, "cov_mean": 4.032192100567045e-05, "cov_std": 0.30262381583452225, "entropy": 0.34619140625, "epoch": 0.06742857142857143, "grad_norm": 0.38411930203437805, "kl": 0.0004928112030029297, "learning_rate": 4.873721045679706e-07, "loss": 0.0209, "reward": 0.5000000102445483, "reward_std": 0.27465011551976204, "rewards/accuracy_reward": 0.1770833358168602, "rewards/format_reward": 0.3229166669771075, "step": 59, "w_high_ratio": 0.15833117440342903, "w_low_ratio": 0.027412693481892347, "w_max": 2.2743676006793976, "w_mean": 1.3636261224746704, "w_min": 0.25, "w_std": 0.22530828416347504 }, { "completion_length": 3205.885498046875, "cov_mean": 5.204351282372954e-06, "cov_std": 0.21868624165654182, "entropy": 0.35986328125, "epoch": 0.06857142857142857, "grad_norm": 0.2159898579120636, "kl": 0.0013399124145507812, "learning_rate": 4.7185832004988133e-07, "loss": 0.0012, "reward": 0.552083358168602, "reward_std": 0.3438131511211395, "rewards/accuracy_reward": 0.13541667070239782, "rewards/format_reward": 0.416666679084301, "step": 60, "w_high_ratio": 0.0, "w_low_ratio": 0.0294471038505435, "w_max": 1.4707800149917603, "w_mean": 1.1660968363285065, "w_min": 0.25, "w_std": 0.13705550134181976 }, { "completion_length": 3322.7084350585938, "cov_mean": 3.926075260096695e-05, "cov_std": 0.3601963371038437, "entropy": 0.3564453125, "epoch": 0.06971428571428571, "grad_norm": 0.39628517627716064, "kl": 0.0009188652038574219, "learning_rate": 4.5643973913200837e-07, "loss": -0.0714, "reward": 0.5833333414047956, "reward_std": 0.515319935977459, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.41666666977107525, "step": 61, "w_high_ratio": 0.03236403688788414, "w_low_ratio": 0.03914234112016857, "w_max": 1.7150523364543915, "w_mean": 1.2098670601844788, "w_min": 0.0, "w_std": 0.2282575462013483 }, { "completion_length": 3006.885498046875, "cov_mean": 2.1906609163124813e-05, "cov_std": 0.46718820184469223, "entropy": 0.33935546875, "epoch": 0.07085714285714285, "grad_norm": 0.30309396982192993, "kl": 0.0017642974853515625, "learning_rate": 4.4113514698014953e-07, "loss": -0.035, "reward": 0.8229167014360428, "reward_std": 0.6976551413536072, "rewards/accuracy_reward": 0.27083334140479565, "rewards/format_reward": 0.5520833432674408, "step": 62, "w_high_ratio": 0.0, "w_low_ratio": 0.05965683329850435, "w_max": 1.593172937631607, "w_mean": 1.1934809386730194, "w_min": 1.0754965713692971e-43, "w_std": 0.28461313247680664 }, { "completion_length": 2580.229248046875, "cov_mean": 5.573407906922512e-05, "cov_std": 0.3945640027523041, "entropy": 0.39111328125, "epoch": 0.072, "grad_norm": 0.45718619227409363, "kl": 0.001735687255859375, "learning_rate": 4.2596318988235037e-07, "loss": -0.0807, "reward": 0.958333358168602, "reward_std": 0.5150914192199707, "rewards/accuracy_reward": 0.31250001303851604, "rewards/format_reward": 0.6458333432674408, "step": 63, "w_high_ratio": 0.08918174356222153, "w_low_ratio": 0.03547394182533026, "w_max": 2.0218554735183716, "w_mean": 1.3539214432239532, "w_min": 0.0, "w_std": 0.23917281255126 }, { "completion_length": 3340.0313110351562, "cov_mean": 7.548666690126993e-05, "cov_std": 0.31813300400972366, "entropy": 0.4248046875, "epoch": 0.07314285714285715, "grad_norm": 0.5389249920845032, "kl": 0.0018744468688964844, "learning_rate": 4.1094235253127374e-07, "loss": -0.0757, "reward": 0.5104166865348816, "reward_std": 0.4255366772413254, "rewards/accuracy_reward": 0.1979166753590107, "rewards/format_reward": 0.3125000074505806, "step": 64, "w_high_ratio": 0.15633241832256317, "w_low_ratio": 0.0320257218554616, "w_max": 1.8200311660766602, "w_mean": 1.3009037971496582, "w_min": 0.25, "w_std": 0.21954189240932465 }, { "completion_length": 2969.541748046875, "cov_mean": -2.4962878342194017e-05, "cov_std": 0.2727944627404213, "entropy": 0.353759765625, "epoch": 0.07428571428571429, "grad_norm": 0.2332458198070526, "kl": 0.0015840530395507812, "learning_rate": 3.9609093550344907e-07, "loss": 0.012, "reward": 0.666666679084301, "reward_std": 0.357291292399168, "rewards/accuracy_reward": 0.2083333358168602, "rewards/format_reward": 0.4583333432674408, "step": 65, "w_high_ratio": 0.0, "w_low_ratio": 0.03699250426143408, "w_max": 1.5260689854621887, "w_mean": 1.164745956659317, "w_min": 1.0748096548620851e-35, "w_std": 0.14831626787781715 }, { "completion_length": 2311.2709197998047, "cov_mean": 2.157548601644521e-05, "cov_std": 0.23550356552004814, "entropy": 0.31787109375, "epoch": 0.07542857142857143, "grad_norm": 0.5557933449745178, "kl": 0.0011758804321289062, "learning_rate": 3.8142703296283953e-07, "loss": -0.0116, "reward": 0.9062500260770321, "reward_std": 0.29818330332636833, "rewards/accuracy_reward": 0.38541667722165585, "rewards/format_reward": 0.520833333954215, "step": 66, "w_high_ratio": 0.10277381911873817, "w_low_ratio": 0.03059569653123617, "w_max": 1.97393798828125, "w_mean": 1.278551459312439, "w_min": 0.0, "w_std": 0.20061488449573517 }, { "completion_length": 3726.135498046875, "cov_mean": 8.957304089562967e-07, "cov_std": 0.12987565249204636, "entropy": 0.35205078125, "epoch": 0.07657142857142857, "grad_norm": 0.11042757332324982, "kl": 0.0013402700424194336, "learning_rate": 3.6696851061588994e-07, "loss": -0.0059, "reward": 0.1770833358168602, "reward_std": 0.15668470412492752, "rewards/accuracy_reward": 0.03125, "rewards/format_reward": 0.1458333358168602, "step": 67, "w_high_ratio": 0.0, "w_low_ratio": 0.01588357798755169, "w_max": 1.2971015870571136, "w_mean": 1.0769366472959518, "w_min": 0.5, "w_std": 0.08244858682155609 }, { "completion_length": 2420.979232788086, "cov_mean": 1.4521759112540167e-05, "cov_std": 0.2692869156599045, "entropy": 0.37353515625, "epoch": 0.07771428571428571, "grad_norm": 0.3267475962638855, "kl": 0.0064449310302734375, "learning_rate": 3.5273298394491515e-07, "loss": -0.0214, "reward": 0.822916679084301, "reward_std": 0.39656912535429, "rewards/accuracy_reward": 0.2187500074505806, "rewards/format_reward": 0.6041666716337204, "step": 68, "w_high_ratio": 0.028063589707016945, "w_low_ratio": 0.029913824051618576, "w_max": 1.8040810525417328, "w_mean": 1.316351056098938, "w_min": 0.25, "w_std": 0.17742961645126343 }, { "completion_length": 2790.5730590820312, "cov_mean": 3.09187362290686e-05, "cov_std": 0.42568420618772507, "entropy": 0.4453125, "epoch": 0.07885714285714286, "grad_norm": 0.7317398190498352, "kl": 0.006710052490234375, "learning_rate": 3.387377967463493e-07, "loss": -0.0192, "reward": 0.4895833358168602, "reward_std": 0.41857269406318665, "rewards/accuracy_reward": 0.08333333674818277, "rewards/format_reward": 0.4062500074505806, "step": 69, "w_high_ratio": 0.19593259878456593, "w_low_ratio": 0.036022237967699766, "w_max": 3.16190242767334, "w_mean": 1.5170519053936005, "w_min": 0.0, "w_std": 0.3000602051615715 }, { "completion_length": 3255.7709350585938, "cov_mean": -6.321061573544284e-05, "cov_std": 0.31040842086076736, "entropy": 0.34765625, "epoch": 0.08, "grad_norm": 0.2253309190273285, "kl": 0.0019707679748535156, "learning_rate": 3.250000000000001e-07, "loss": 0.038, "reward": 0.5104166679084301, "reward_std": 0.3943600133061409, "rewards/accuracy_reward": 0.09375000093132257, "rewards/format_reward": 0.4166666716337204, "step": 70, "w_high_ratio": 0.0, "w_low_ratio": 0.04103634599596262, "w_max": 1.2795763909816742, "w_mean": 1.0765459537506104, "w_min": 0.0, "w_std": 0.15424982458353043 }, { "completion_length": 2893.093795776367, "cov_mean": -5.907902482249483e-05, "cov_std": 0.28848100267350674, "entropy": 0.39111328125, "epoch": 0.08114285714285714, "grad_norm": 0.4800524115562439, "kl": 0.0017595291137695312, "learning_rate": 3.115363310950578e-07, "loss": 0.0284, "reward": 0.562500030733645, "reward_std": 0.3276159390807152, "rewards/accuracy_reward": 0.18750000651925802, "rewards/format_reward": 0.37500000558793545, "step": 71, "w_high_ratio": 0.07317740470170975, "w_low_ratio": 0.03526141308248043, "w_max": 2.095757156610489, "w_mean": 1.278771072626114, "w_min": 4.3955273748636295e-36, "w_std": 0.23910818807780743 }, { "completion_length": 3460.354248046875, "cov_mean": 6.12462517892709e-05, "cov_std": 0.36787012219429016, "entropy": 0.4931640625, "epoch": 0.08228571428571428, "grad_norm": 0.39541420340538025, "kl": 0.0038938522338867188, "learning_rate": 2.9836319343816397e-07, "loss": -0.0245, "reward": 0.3333333469927311, "reward_std": 0.4593120887875557, "rewards/accuracy_reward": 0.06250000093132257, "rewards/format_reward": 0.2708333395421505, "step": 72, "w_high_ratio": 0.040276188403367996, "w_low_ratio": 0.046780284494161606, "w_max": 1.6123959422111511, "w_mean": 1.1790215075016022, "w_min": 0.0, "w_std": 0.24864742159843445 }, { "completion_length": 3808.166748046875, "cov_mean": 2.382581669735373e-05, "cov_std": 0.26704882457852364, "entropy": 0.4677734375, "epoch": 0.08342857142857144, "grad_norm": 0.39250648021698, "kl": 0.0005078315734863281, "learning_rate": 2.854966364683872e-07, "loss": 0.0333, "reward": 0.21875001024454832, "reward_std": 0.34505901485681534, "rewards/accuracy_reward": 0.0729166679084301, "rewards/format_reward": 0.14583333488553762, "step": 73, "w_high_ratio": 0.0, "w_low_ratio": 0.031687673181295395, "w_max": 1.3024969696998596, "w_mean": 1.03611221909523, "w_min": 0.0, "w_std": 0.13818097487092018 }, { "completion_length": 3584.8438110351562, "cov_mean": -4.775809429702349e-05, "cov_std": 0.3642261065542698, "entropy": 0.42138671875, "epoch": 0.08457142857142858, "grad_norm": 0.22599299252033234, "kl": 0.002000570297241211, "learning_rate": 2.729523361034538e-07, "loss": 0.0465, "reward": 0.552083358168602, "reward_std": 0.47496967762708664, "rewards/accuracy_reward": 0.2604166679084301, "rewards/format_reward": 0.2916666753590107, "step": 74, "w_high_ratio": 0.0, "w_low_ratio": 0.04971347749233246, "w_max": 1.3916961252689362, "w_mean": 1.079592913389206, "w_min": 0.0, "w_std": 0.20160867273807526 }, { "completion_length": 3403.7188720703125, "cov_mean": 4.775901288667228e-06, "cov_std": 0.31741658970713615, "entropy": 0.39208984375, "epoch": 0.08571428571428572, "grad_norm": 0.28842759132385254, "kl": 0.002251148223876953, "learning_rate": 2.6074557564105724e-07, "loss": -0.0149, "reward": 0.5312500074505806, "reward_std": 0.3348959684371948, "rewards/accuracy_reward": 0.1770833358168602, "rewards/format_reward": 0.3541666744276881, "step": 75, "w_high_ratio": 0.05775141716003418, "w_low_ratio": 0.04404309671372175, "w_max": 1.7450213432312012, "w_mean": 1.203758329153061, "w_min": 0.25, "w_std": 0.19945883378386497 }, { "completion_length": 3129.885498046875, "cov_mean": 1.2703612810582854e-05, "cov_std": 0.2526189871132374, "entropy": 0.396484375, "epoch": 0.08685714285714285, "grad_norm": 0.15439313650131226, "kl": 0.0004563331604003906, "learning_rate": 2.488912271385139e-07, "loss": 0.0167, "reward": 0.572916679084301, "reward_std": 0.2542962171137333, "rewards/accuracy_reward": 0.1041666716337204, "rewards/format_reward": 0.4687500074505806, "step": 76, "w_high_ratio": 0.0, "w_low_ratio": 0.03176691988483071, "w_max": 1.4051032662391663, "w_mean": 1.1220950484275818, "w_min": 0.25, "w_std": 0.1384560540318489 }, { "completion_length": 3342.510498046875, "cov_mean": 4.0359717786486726e-05, "cov_std": 0.2773597203195095, "entropy": 0.3876953125, "epoch": 0.088, "grad_norm": 0.39251646399497986, "kl": 0.0005764961242675781, "learning_rate": 2.374037332934512e-07, "loss": 0.0041, "reward": 0.552083333954215, "reward_std": 0.26390674337744713, "rewards/accuracy_reward": 0.1458333432674408, "rewards/format_reward": 0.40625000558793545, "step": 77, "w_high_ratio": 0.0, "w_low_ratio": 0.035081148613244295, "w_max": 1.4559779465198517, "w_mean": 1.158086121082306, "w_min": 0.0, "w_std": 0.16358545422554016 }, { "completion_length": 3630.3438720703125, "cov_mean": 2.0063042029505596e-05, "cov_std": 0.3748911917209625, "entropy": 0.39599609375, "epoch": 0.08914285714285715, "grad_norm": 0.50286865234375, "kl": 0.013364076614379883, "learning_rate": 2.2629708984760706e-07, "loss": 0.0002, "reward": 0.4479166939854622, "reward_std": 0.5592645406723022, "rewards/accuracy_reward": 0.1770833432674408, "rewards/format_reward": 0.2708333469927311, "step": 78, "w_high_ratio": 0.04053416848182678, "w_low_ratio": 0.047788419760763645, "w_max": 1.5594379305839539, "w_mean": 1.1108087003231049, "w_min": 0.0, "w_std": 0.2227778285741806 }, { "completion_length": 2405.6354598999023, "cov_mean": 9.158174270851305e-05, "cov_std": 0.34057875350117683, "entropy": 0.326171875, "epoch": 0.09028571428571429, "grad_norm": 0.4699649214744568, "kl": 0.0014543533325195312, "learning_rate": 2.1558482853517253e-07, "loss": 0.0159, "reward": 0.8750000223517418, "reward_std": 0.34952742606401443, "rewards/accuracy_reward": 0.250000006519258, "rewards/format_reward": 0.6250000298023224, "step": 79, "w_high_ratio": 0.11352726817131042, "w_low_ratio": 0.03820635436568409, "w_max": 1.81002277135849, "w_mean": 1.309929370880127, "w_min": 0.0, "w_std": 0.2257107077166438 }, { "completion_length": 3596.3750610351562, "cov_mean": 7.016459858277813e-05, "cov_std": 0.21266964823007584, "entropy": 0.46337890625, "epoch": 0.09142857142857143, "grad_norm": 0.224221870303154, "kl": 0.0014083385467529297, "learning_rate": 2.0528000059645995e-07, "loss": 0.0048, "reward": 0.5520833348855376, "reward_std": 0.33429770916700363, "rewards/accuracy_reward": 0.20833333395421505, "rewards/format_reward": 0.3437500009313226, "step": 80, "w_high_ratio": 0.0, "w_low_ratio": 0.029426061548292637, "w_max": 1.2386472523212433, "w_mean": 1.0539521872997284, "w_min": 0.25, "w_std": 0.11237771809101105 }, { "completion_length": 3377.5521240234375, "cov_mean": 1.5553408047708217e-05, "cov_std": 0.22893034294247627, "entropy": 0.52197265625, "epoch": 0.09257142857142857, "grad_norm": 0.27431702613830566, "kl": 0.0027017593383789062, "learning_rate": 1.9539516087697517e-07, "loss": 0.0442, "reward": 0.3125000027939677, "reward_std": 0.3139677047729492, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.22916666697710752, "step": 81, "w_high_ratio": 0.0, "w_low_ratio": 0.03366563096642494, "w_max": 1.3624085187911987, "w_mean": 1.1045191884040833, "w_min": 0.25, "w_std": 0.13904165104031563 }, { "completion_length": 3133.4583740234375, "cov_mean": -1.667316132625274e-05, "cov_std": 0.27454316802322865, "entropy": 0.408203125, "epoch": 0.09371428571428571, "grad_norm": 0.38514095544815063, "kl": 0.0017423629760742188, "learning_rate": 1.8594235253127372e-07, "loss": -0.0213, "reward": 0.5937500149011612, "reward_std": 0.35102832317352295, "rewards/accuracy_reward": 0.18750000465661287, "rewards/format_reward": 0.4062500074505806, "step": 82, "w_high_ratio": 0.0, "w_low_ratio": 0.034213229548186064, "w_max": 1.6117319464683533, "w_mean": 1.1832448840141296, "w_min": 9.42250974980716e-31, "w_std": 0.1813708432018757 }, { "completion_length": 3142.1771240234375, "cov_mean": 4.071597049914999e-05, "cov_std": 0.17724395170807838, "entropy": 0.45751953125, "epoch": 0.09485714285714286, "grad_norm": 0.3409688472747803, "kl": 0.001071929931640625, "learning_rate": 1.7693309235023127e-07, "loss": 0.0102, "reward": 0.38541667722165585, "reward_std": 0.20087094232439995, "rewards/accuracy_reward": 0.11458333395421505, "rewards/format_reward": 0.2708333386108279, "step": 83, "w_high_ratio": 0.0, "w_low_ratio": 0.023759393487125635, "w_max": 1.2598195374011993, "w_mean": 1.0758444666862488, "w_min": 0.25, "w_std": 0.10355101153254509 }, { "completion_length": 3299.4271850585938, "cov_mean": 1.610395520401653e-05, "cov_std": 0.3381447046995163, "entropy": 0.43505859375, "epoch": 0.096, "grad_norm": 0.3248639404773712, "kl": 0.0011587142944335938, "learning_rate": 1.6837835672960831e-07, "loss": -0.0241, "reward": 0.7500000298023224, "reward_std": 0.4175455719232559, "rewards/accuracy_reward": 0.3020833358168602, "rewards/format_reward": 0.4479166865348816, "step": 84, "w_high_ratio": 0.0, "w_low_ratio": 0.034003185108304024, "w_max": 1.4201359450817108, "w_mean": 1.1546694040298462, "w_min": 0.25, "w_std": 0.19938229769468307 }, { "completion_length": 3368.1146240234375, "cov_mean": 1.1272636584180873e-05, "cov_std": 0.3433619923889637, "entropy": 0.353515625, "epoch": 0.09714285714285714, "grad_norm": 0.2291695475578308, "kl": 0.0006651878356933594, "learning_rate": 1.6028856829700258e-07, "loss": 0.0156, "reward": 0.552083333954215, "reward_std": 0.5098650455474854, "rewards/accuracy_reward": 0.17708334233611822, "rewards/format_reward": 0.3750000102445483, "step": 85, "w_high_ratio": 0.05620652809739113, "w_low_ratio": 0.04038320202380419, "w_max": 1.5647819340229034, "w_mean": 1.2357110977172852, "w_min": 1.9246834407501362e-42, "w_std": 0.2121213562786579 }, { "completion_length": 3338.4479370117188, "cov_mean": 6.590973612219386e-06, "cov_std": 0.2853589281439781, "entropy": 0.45654296875, "epoch": 0.09828571428571428, "grad_norm": 0.3797212839126587, "kl": 0.0021305084228515625, "learning_rate": 1.5267358321348285e-07, "loss": -0.0236, "reward": 0.5208333432674408, "reward_std": 0.299550324678421, "rewards/accuracy_reward": 0.1770833395421505, "rewards/format_reward": 0.34375, "step": 86, "w_high_ratio": 0.05717267841100693, "w_low_ratio": 0.02882188046351075, "w_max": 1.963057816028595, "w_mean": 1.2316765785217285, "w_min": 0.25, "w_std": 0.1924368217587471 }, { "completion_length": 3156.2188110351562, "cov_mean": 8.500453259330243e-05, "cov_std": 0.5130489468574524, "entropy": 0.5185546875, "epoch": 0.09942857142857142, "grad_norm": 0.46258026361465454, "kl": 0.00295257568359375, "learning_rate": 1.4554267916537495e-07, "loss": -0.0212, "reward": 0.5833333432674408, "reward_std": 0.5054471418261528, "rewards/accuracy_reward": 0.14583333674818277, "rewards/format_reward": 0.4375000074505806, "step": 87, "w_high_ratio": 0.13361193798482418, "w_low_ratio": 0.05879375524818897, "w_max": 1.8866381645202637, "w_mean": 1.3355560600757599, "w_min": 0.0, "w_std": 0.33218564838171005 }, { "completion_length": 2856.8959350585938, "cov_mean": -5.945291923126206e-05, "cov_std": 0.5394940003752708, "entropy": 0.44140625, "epoch": 0.10057142857142858, "grad_norm": 0.890120804309845, "kl": 0.02105236053466797, "learning_rate": 1.3890454406082956e-07, "loss": -0.0362, "reward": 0.8958333730697632, "reward_std": 0.7090653479099274, "rewards/accuracy_reward": 0.3645833432674408, "rewards/format_reward": 0.5312500149011612, "step": 88, "w_high_ratio": 0.21772565320134163, "w_low_ratio": 0.049323974177241325, "w_max": 2.266294479370117, "w_mean": 1.4745216369628906, "w_min": 0.0, "w_std": 0.3573007620871067 }, { "completion_length": 3663.3021850585938, "cov_mean": -5.7599154160925536e-05, "cov_std": 0.2993681877851486, "entropy": 0.43505859375, "epoch": 0.10171428571428572, "grad_norm": 0.3626137971878052, "kl": 0.00238800048828125, "learning_rate": 1.3276726544494571e-07, "loss": 0.0099, "reward": 0.3645833469927311, "reward_std": 0.47493190318346024, "rewards/accuracy_reward": 0.12500000651925802, "rewards/format_reward": 0.23958334513008595, "step": 89, "w_high_ratio": 0.0, "w_low_ratio": 0.0442591137252748, "w_max": 1.4542400240898132, "w_mean": 1.079471230506897, "w_min": 0.0, "w_std": 0.18644139170646667 }, { "completion_length": 2692.8125610351562, "cov_mean": 4.647310163363727e-06, "cov_std": 0.3419715017080307, "entropy": 0.5390625, "epoch": 0.10285714285714286, "grad_norm": 0.36119693517684937, "kl": 0.0059909820556640625, "learning_rate": 1.2713832064634125e-07, "loss": 0.0442, "reward": 0.5729166697710752, "reward_std": 0.3239624537527561, "rewards/accuracy_reward": 0.06250000279396772, "rewards/format_reward": 0.5104166744276881, "step": 90, "w_high_ratio": 0.14126155525445938, "w_low_ratio": 0.02876184810884297, "w_max": 2.2300167977809906, "w_mean": 1.42653489112854, "w_min": 1.471363387541058e-44, "w_std": 0.21830591186881065 }, { "completion_length": 3451.5313110351562, "cov_mean": 1.8957917745865416e-05, "cov_std": 0.4000442400574684, "entropy": 0.45654296875, "epoch": 0.104, "grad_norm": 0.4046555459499359, "kl": 0.0024518966674804688, "learning_rate": 1.220245676671809e-07, "loss": -0.0665, "reward": 0.6770833730697632, "reward_std": 0.40345774590969086, "rewards/accuracy_reward": 0.260416679084301, "rewards/format_reward": 0.4166666865348816, "step": 91, "w_high_ratio": 0.06697376817464828, "w_low_ratio": 0.0342584103345871, "w_max": 1.4452155828475952, "w_mean": 1.1833300292491913, "w_min": 0.25, "w_std": 0.2148173823952675 }, { "completion_length": 3021.2188110351562, "cov_mean": -6.440309425670421e-05, "cov_std": 0.45011676102876663, "entropy": 0.38720703125, "epoch": 0.10514285714285715, "grad_norm": 0.44917768239974976, "kl": 0.007047891616821289, "learning_rate": 1.1743223682775649e-07, "loss": 0.0015, "reward": 0.7291666865348816, "reward_std": 0.5440528020262718, "rewards/accuracy_reward": 0.19791667256504297, "rewards/format_reward": 0.5312500074505806, "step": 92, "w_high_ratio": 0.060439030872657895, "w_low_ratio": 0.046312456484884024, "w_max": 2.021807760000229, "w_mean": 1.2982419431209564, "w_min": 0.0, "w_std": 0.2590548172593117 }, { "completion_length": 3865.6250610351562, "cov_mean": 1.7783165276341606e-05, "cov_std": 0.23176611587405205, "entropy": 0.52880859375, "epoch": 0.10628571428571429, "grad_norm": 0.2683483958244324, "kl": 0.002590179443359375, "learning_rate": 1.1336692317580158e-07, "loss": 0.0487, "reward": 0.1145833358168602, "reward_std": 0.27467070147395134, "rewards/accuracy_reward": 0.031250000931322575, "rewards/format_reward": 0.08333333488553762, "step": 93, "w_high_ratio": 0.0, "w_low_ratio": 0.03534638648852706, "w_max": 1.232116162776947, "w_mean": 1.0145132839679718, "w_min": 4.369921683443164e-37, "w_std": 0.12999964877963066 }, { "completion_length": 3234.9271850585938, "cov_mean": 1.053587311616866e-05, "cov_std": 0.20603487268090248, "entropy": 0.52490234375, "epoch": 0.10742857142857143, "grad_norm": 0.25439760088920593, "kl": 0.0029878616333007812, "learning_rate": 1.0983357966978745e-07, "loss": 0.0214, "reward": 0.4270833348855376, "reward_std": 0.17300797998905182, "rewards/accuracy_reward": 0.14583333395421505, "rewards/format_reward": 0.281250006519258, "step": 94, "w_high_ratio": 0.0574149414896965, "w_low_ratio": 0.021458564791828394, "w_max": 1.955292284488678, "w_mean": 1.259522944688797, "w_min": 0.25, "w_std": 0.14511701837182045 }, { "completion_length": 3710.2396240234375, "cov_mean": 1.0393607226433232e-05, "cov_std": 0.3523143231868744, "entropy": 0.42578125, "epoch": 0.10857142857142857, "grad_norm": 0.20859070122241974, "kl": 0.0004982948303222656, "learning_rate": 1.068365111445064e-07, "loss": 0.0209, "reward": 0.34375, "reward_std": 0.45749304443597794, "rewards/accuracy_reward": 0.11458333488553762, "rewards/format_reward": 0.2291666716337204, "step": 95, "w_high_ratio": 0.0, "w_low_ratio": 0.04463845491409302, "w_max": 1.3612166047096252, "w_mean": 1.0816101431846619, "w_min": 0.25, "w_std": 0.20336205884814262 }, { "completion_length": 3114.041748046875, "cov_mean": -6.73702224958106e-05, "cov_std": 0.36486806720495224, "entropy": 0.4013671875, "epoch": 0.10971428571428571, "grad_norm": 0.3484714925289154, "kl": 0.002678394317626953, "learning_rate": 1.0437936906629334e-07, "loss": -0.0171, "reward": 0.6979166865348816, "reward_std": 0.46633855253458023, "rewards/accuracy_reward": 0.26041666977107525, "rewards/format_reward": 0.4375000074505806, "step": 96, "w_high_ratio": 0.1971779614686966, "w_low_ratio": 0.04042420396581292, "w_max": 2.1471228301525116, "w_mean": 1.42051500082016, "w_min": 1.9142914218395184e-30, "w_std": 0.24729400128126144 }, { "completion_length": 3457.760498046875, "cov_mean": -1.2597635304700816e-05, "cov_std": 0.2245815545320511, "entropy": 0.41552734375, "epoch": 0.11085714285714286, "grad_norm": 0.24951718747615814, "kl": 0.0011386871337890625, "learning_rate": 1.0246514708427701e-07, "loss": -0.0394, "reward": 0.5625000055879354, "reward_std": 0.30785489082336426, "rewards/accuracy_reward": 0.23958333395421505, "rewards/format_reward": 0.32291666977107525, "step": 97, "w_high_ratio": 0.0, "w_low_ratio": 0.02458228124305606, "w_max": 1.5016373097896576, "w_mean": 1.1687426269054413, "w_min": 0.25, "w_std": 0.14521248266100883 }, { "completion_length": 3134.1251220703125, "cov_mean": 8.062847064138623e-05, "cov_std": 0.3362714499235153, "entropy": 0.384765625, "epoch": 0.112, "grad_norm": 0.3256186544895172, "kl": 0.0006456375122070312, "learning_rate": 1.0109617738307911e-07, "loss": -0.0102, "reward": 0.572916679084301, "reward_std": 0.3148062378168106, "rewards/accuracy_reward": 0.14583333395421505, "rewards/format_reward": 0.4270833432674408, "step": 98, "w_high_ratio": 0.00467883562669158, "w_low_ratio": 0.039873864501714706, "w_max": 1.795419156551361, "w_mean": 1.266855150461197, "w_min": 0.25, "w_std": 0.22055331617593765 }, { "completion_length": 3086.7395935058594, "cov_mean": 1.7768036741472315e-05, "cov_std": 0.15420139580965042, "entropy": 0.35205078125, "epoch": 0.11314285714285714, "grad_norm": 0.3351471722126007, "kl": 0.0013794898986816406, "learning_rate": 1.002741278414069e-07, "loss": 0.0477, "reward": 0.5000000223517418, "reward_std": 0.22891659289598465, "rewards/accuracy_reward": 0.1875000074505806, "rewards/format_reward": 0.3125, "step": 99, "w_high_ratio": 0.055527687072753906, "w_low_ratio": 0.02032439224421978, "w_max": 1.390455573797226, "w_mean": 1.1158559024333954, "w_min": 0.5, "w_std": 0.11466998234391212 }, { "completion_length": 3004.6354370117188, "cov_mean": 2.6663584918651395e-05, "cov_std": 0.3363041579723358, "entropy": 0.36181640625, "epoch": 0.11428571428571428, "grad_norm": 0.40786388516426086, "kl": 0.002788543701171875, "learning_rate": 1e-07, "loss": -0.0227, "reward": 0.7187500149011612, "reward_std": 0.4770490266382694, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.4687500149011612, "step": 100, "w_high_ratio": 0.01440071314573288, "w_low_ratio": 0.04455719655379653, "w_max": 1.8046407103538513, "w_mean": 1.2536596357822418, "w_min": 2.0318827732709848e-44, "w_std": 0.2585573047399521 }, { "epoch": 0.11428571428571428, "step": 100, "total_flos": 0.0, "train_loss": -0.004574703625403345, "train_runtime": 8361.7209, "train_samples_per_second": 1.148, "train_steps_per_second": 0.012 } ], "logging_steps": 1, "max_steps": 100, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 6, "trial_name": null, "trial_params": null }