{ "best_metric": null, "best_model_checkpoint": null, "epoch": 0.9996101364522417, "eval_steps": 500, "global_step": 1282, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio": 0.0, "completion_length": 72.97917175292969, "epoch": 0.0007797270955165692, "grad_norm": 2.0360797534837407, "kl": 0.0, "learning_rate": 7.751937984496123e-09, "loss": 0.1572, "reward": 0.034821840934455395, "reward_std": 0.006848668563179672, "rewards/accuracy_reward": 0.034821840934455395, "step": 1 }, { "clip_ratio": 0.0, "completion_length": 74.54166793823242, "epoch": 0.0015594541910331384, "grad_norm": 1.9195591077510774, "kl": 0.0, "learning_rate": 1.5503875968992246e-08, "loss": 0.1859, "reward": 0.035166485235095024, "reward_std": 0.007111588958650827, "rewards/accuracy_reward": 0.035166485235095024, "step": 2 }, { "clip_ratio": 0.0, "completion_length": 85.10416793823242, "epoch": 0.0023391812865497076, "grad_norm": 2.077424101765198, "kl": 2.440810203552246e-05, "learning_rate": 2.3255813953488372e-08, "loss": 0.3084, "reward": 0.030526814982295036, "reward_std": 0.005166596965864301, "rewards/accuracy_reward": 0.030526814982295036, "step": 3 }, { "clip_ratio": 0.0, "completion_length": 78.72916793823242, "epoch": 0.0031189083820662767, "grad_norm": 2.0593942152096165, "kl": 4.89652156829834e-05, "learning_rate": 3.100775193798449e-08, "loss": 0.2624, "reward": 0.031168147921562195, "reward_std": 0.004581201123073697, "rewards/accuracy_reward": 0.031168147921562195, "step": 4 }, { "clip_ratio": 0.0, "completion_length": 69.10416984558105, "epoch": 0.003898635477582846, "grad_norm": 2.4255141009351355, "kl": 5.245208740234375e-06, "learning_rate": 3.8759689922480615e-08, "loss": 0.3318, "reward": 0.037738995626568794, "reward_std": 0.00937926722690463, "rewards/accuracy_reward": 0.037738995626568794, "step": 5 }, { "clip_ratio": 0.0, "completion_length": 77.20833587646484, "epoch": 0.004678362573099415, "grad_norm": 2.277346720017906, "kl": 4.00543212890625e-05, "learning_rate": 4.6511627906976744e-08, "loss": 0.1688, "reward": 0.039112428203225136, "reward_std": 0.011630565393716097, "rewards/accuracy_reward": 0.039112428203225136, "step": 6 }, { "clip_ratio": 0.0, "completion_length": 63.812503814697266, "epoch": 0.005458089668615985, "grad_norm": 2.49297928409578, "kl": -2.3618340492248535e-06, "learning_rate": 5.426356589147287e-08, "loss": 0.1724, "reward": 0.04318097606301308, "reward_std": 0.009315338917076588, "rewards/accuracy_reward": 0.04318097606301308, "step": 7 }, { "clip_ratio": 0.0, "completion_length": 73.39583587646484, "epoch": 0.006237816764132553, "grad_norm": 2.452193481562994, "kl": 2.4437904357910156e-05, "learning_rate": 6.201550387596898e-08, "loss": 0.3233, "reward": 0.03827681764960289, "reward_std": 0.006922724889591336, "rewards/accuracy_reward": 0.03827681764960289, "step": 8 }, { "clip_ratio": 0.0, "completion_length": 84.62500381469727, "epoch": 0.007017543859649123, "grad_norm": 1.9369529915399666, "kl": -2.384185791015625e-07, "learning_rate": 6.976744186046511e-08, "loss": 0.1277, "reward": 0.033217170275747776, "reward_std": 0.006850299425423145, "rewards/accuracy_reward": 0.033217170275747776, "step": 9 }, { "clip_ratio": 0.0, "completion_length": 72.45833587646484, "epoch": 0.007797270955165692, "grad_norm": 2.165737547300236, "kl": 1.2695789337158203e-05, "learning_rate": 7.751937984496123e-08, "loss": 0.3101, "reward": 0.033010080456733704, "reward_std": 0.004679354839026928, "rewards/accuracy_reward": 0.033010080456733704, "step": 10 }, { "clip_ratio": 0.0, "completion_length": 42.43750190734863, "epoch": 0.008576998050682261, "grad_norm": 3.9467304654500217, "kl": 8.845329284667969e-05, "learning_rate": 8.527131782945737e-08, "loss": 0.2326, "reward": 0.05289607308804989, "reward_std": 0.011021225713193417, "rewards/accuracy_reward": 0.05289607308804989, "step": 11 }, { "clip_ratio": 0.0, "completion_length": 98.04166793823242, "epoch": 0.00935672514619883, "grad_norm": 3.435577459737531, "kl": 5.418062210083008e-05, "learning_rate": 9.302325581395349e-08, "loss": 0.4666, "reward": 0.04114745929837227, "reward_std": 0.010439948178827763, "rewards/accuracy_reward": 0.04114745929837227, "step": 12 }, { "clip_ratio": 0.0, "completion_length": 74.72916793823242, "epoch": 0.0101364522417154, "grad_norm": 1.8829822654527015, "kl": -7.525086402893066e-06, "learning_rate": 1.0077519379844961e-07, "loss": 0.1082, "reward": 0.0373533945530653, "reward_std": 0.010985687375068665, "rewards/accuracy_reward": 0.0373533945530653, "step": 13 }, { "clip_ratio": 0.0, "completion_length": 46.5, "epoch": 0.01091617933723197, "grad_norm": 3.9183025177944577, "kl": 4.0650367736816406e-05, "learning_rate": 1.0852713178294573e-07, "loss": 0.2163, "reward": 0.048170048743486404, "reward_std": 0.009852790739387274, "rewards/accuracy_reward": 0.048170048743486404, "step": 14 }, { "clip_ratio": 0.0, "completion_length": 79.58333587646484, "epoch": 0.011695906432748537, "grad_norm": 2.052544767124884, "kl": 6.449222564697266e-05, "learning_rate": 1.1627906976744186e-07, "loss": 0.2347, "reward": 0.036619529128074646, "reward_std": 0.008851302787661552, "rewards/accuracy_reward": 0.036619529128074646, "step": 15 }, { "clip_ratio": 0.0, "completion_length": 59.89583396911621, "epoch": 0.012475633528265107, "grad_norm": 2.656217708142409, "kl": -8.165836334228516e-06, "learning_rate": 1.2403100775193797e-07, "loss": 0.2568, "reward": 0.03845728747546673, "reward_std": 0.00811865832656622, "rewards/accuracy_reward": 0.03845728747546673, "step": 16 }, { "clip_ratio": 0.0, "completion_length": 80.22917175292969, "epoch": 0.013255360623781676, "grad_norm": 1.9427871138959392, "kl": 3.3020973205566406e-05, "learning_rate": 1.317829457364341e-07, "loss": 0.2086, "reward": 0.03203662671148777, "reward_std": 0.003987034084275365, "rewards/accuracy_reward": 0.03203662671148777, "step": 17 }, { "clip_ratio": 0.0, "completion_length": 63.416669845581055, "epoch": 0.014035087719298246, "grad_norm": 3.33750190352222, "kl": 3.084540367126465e-05, "learning_rate": 1.3953488372093021e-07, "loss": 0.1715, "reward": 0.04148095287382603, "reward_std": 0.014872618019580841, "rewards/accuracy_reward": 0.04148095287382603, "step": 18 }, { "clip_ratio": 0.0, "completion_length": 69.85417175292969, "epoch": 0.014814814814814815, "grad_norm": 2.0649953692424075, "kl": 1.9073486328125e-05, "learning_rate": 1.4728682170542635e-07, "loss": 0.2053, "reward": 0.04008662328124046, "reward_std": 0.008591718040406704, "rewards/accuracy_reward": 0.04008662328124046, "step": 19 }, { "clip_ratio": 0.0, "completion_length": 78.79167175292969, "epoch": 0.015594541910331383, "grad_norm": 2.0297255573747295, "kl": -1.74790620803833e-05, "learning_rate": 1.5503875968992246e-07, "loss": 0.322, "reward": 0.03438205271959305, "reward_std": 0.007812638068571687, "rewards/accuracy_reward": 0.03438205271959305, "step": 20 }, { "clip_ratio": 0.0, "completion_length": 65.52083587646484, "epoch": 0.016374269005847954, "grad_norm": 2.4261350923804264, "kl": 1.4513731002807617e-05, "learning_rate": 1.6279069767441862e-07, "loss": 0.2003, "reward": 0.03825371153652668, "reward_std": 0.008155276766046882, "rewards/accuracy_reward": 0.03825371153652668, "step": 21 }, { "clip_ratio": 0.0, "completion_length": 71.625, "epoch": 0.017153996101364522, "grad_norm": 1.9250078658310887, "kl": -2.5033950805664062e-06, "learning_rate": 1.7054263565891473e-07, "loss": 0.0647, "reward": 0.03269452229142189, "reward_std": 0.008812714833766222, "rewards/accuracy_reward": 0.03269452229142189, "step": 22 }, { "clip_ratio": 0.0, "completion_length": 79.29166793823242, "epoch": 0.01793372319688109, "grad_norm": 2.010657233810007, "kl": 2.331542782485485e-05, "learning_rate": 1.7829457364341087e-07, "loss": 0.2563, "reward": 0.03494575526565313, "reward_std": 0.006035172613337636, "rewards/accuracy_reward": 0.03494575526565313, "step": 23 }, { "clip_ratio": 0.0, "completion_length": 76.20833587646484, "epoch": 0.01871345029239766, "grad_norm": 2.0027894508025343, "kl": 8.577108383178711e-05, "learning_rate": 1.8604651162790698e-07, "loss": 0.1875, "reward": 0.030015477910637856, "reward_std": 0.0032516615465283394, "rewards/accuracy_reward": 0.030015477910637856, "step": 24 }, { "clip_ratio": 0.0, "completion_length": 68.52083587646484, "epoch": 0.01949317738791423, "grad_norm": 2.444697061646816, "kl": 7.224082946777344e-05, "learning_rate": 1.9379844961240311e-07, "loss": 0.312, "reward": 0.03344830125570297, "reward_std": 0.004560418426990509, "rewards/accuracy_reward": 0.03344830125570297, "step": 25 }, { "clip_ratio": 0.0, "completion_length": 64.97916793823242, "epoch": 0.0202729044834308, "grad_norm": 2.532369903498374, "kl": 8.785724639892578e-05, "learning_rate": 2.0155038759689922e-07, "loss": 0.0783, "reward": 0.04169601574540138, "reward_std": 0.015225842595100403, "rewards/accuracy_reward": 0.04169601574540138, "step": 26 }, { "clip_ratio": 0.0, "completion_length": 70.77083587646484, "epoch": 0.021052631578947368, "grad_norm": 2.3095745232275755, "kl": 5.412101745605469e-05, "learning_rate": 2.0930232558139536e-07, "loss": 0.1652, "reward": 0.03223975747823715, "reward_std": 0.0025580759393051267, "rewards/accuracy_reward": 0.03223975747823715, "step": 27 }, { "clip_ratio": 0.0, "completion_length": 68.93750190734863, "epoch": 0.02183235867446394, "grad_norm": 2.788800470549753, "kl": 0.0001437664031982422, "learning_rate": 2.1705426356589147e-07, "loss": 0.2741, "reward": 0.036526650190353394, "reward_std": 0.005832177121192217, "rewards/accuracy_reward": 0.036526650190353394, "step": 28 }, { "clip_ratio": 0.0, "completion_length": 74.87500190734863, "epoch": 0.022612085769980507, "grad_norm": 2.282049473570082, "kl": 0.00024366378784179688, "learning_rate": 2.248062015503876e-07, "loss": 0.4466, "reward": 0.03732151258736849, "reward_std": 0.0073881205171346664, "rewards/accuracy_reward": 0.03732151258736849, "step": 29 }, { "clip_ratio": 0.0, "completion_length": 65.12500190734863, "epoch": 0.023391812865497075, "grad_norm": 2.8771857181708316, "kl": 0.00025177001953125, "learning_rate": 2.3255813953488372e-07, "loss": 0.3322, "reward": 0.0398801788687706, "reward_std": 0.011091248597949743, "rewards/accuracy_reward": 0.0398801788687706, "step": 30 }, { "clip_ratio": 0.0, "completion_length": 68.22916793823242, "epoch": 0.024171539961013646, "grad_norm": 2.9801638513714446, "kl": 0.0003724098205566406, "learning_rate": 2.403100775193798e-07, "loss": 0.1957, "reward": 0.04014836438000202, "reward_std": 0.010948444716632366, "rewards/accuracy_reward": 0.04014836438000202, "step": 31 }, { "clip_ratio": 0.0, "completion_length": 71.02083587646484, "epoch": 0.024951267056530214, "grad_norm": 2.7228841256045686, "kl": 0.00022363662719726562, "learning_rate": 2.4806201550387593e-07, "loss": 0.1496, "reward": 0.037018182687461376, "reward_std": 0.00551977613940835, "rewards/accuracy_reward": 0.037018182687461376, "step": 32 }, { "clip_ratio": 0.0, "completion_length": 72.875, "epoch": 0.025730994152046785, "grad_norm": 2.1519240668442303, "kl": 0.00014019012451171875, "learning_rate": 2.558139534883721e-07, "loss": 0.1798, "reward": 0.031542438082396984, "reward_std": 0.006069755996577442, "rewards/accuracy_reward": 0.031542438082396984, "step": 33 }, { "clip_ratio": 0.0, "completion_length": 122.20833969116211, "epoch": 0.026510721247563353, "grad_norm": 1.9690755699819855, "kl": 0.00036907196044921875, "learning_rate": 2.635658914728682e-07, "loss": 0.4417, "reward": 0.032262321561574936, "reward_std": 0.008104364853352308, "rewards/accuracy_reward": 0.032262321561574936, "step": 34 }, { "clip_ratio": 0.0, "completion_length": 58.35416793823242, "epoch": 0.02729044834307992, "grad_norm": 2.3964490155178697, "kl": 0.0004863739013671875, "learning_rate": 2.713178294573643e-07, "loss": 0.2393, "reward": 0.04383615590631962, "reward_std": 0.010008457116782665, "rewards/accuracy_reward": 0.04383615590631962, "step": 35 }, { "clip_ratio": 0.0, "completion_length": 60.625003814697266, "epoch": 0.028070175438596492, "grad_norm": 2.486430271357229, "kl": 0.0010223388671875, "learning_rate": 2.7906976744186043e-07, "loss": 0.2161, "reward": 0.0347980335354805, "reward_std": 0.006134419701993465, "rewards/accuracy_reward": 0.0347980335354805, "step": 36 }, { "clip_ratio": 0.0, "completion_length": 49.39583396911621, "epoch": 0.02884990253411306, "grad_norm": 3.782970773046541, "kl": 0.001277923583984375, "learning_rate": 2.868217054263566e-07, "loss": 0.3321, "reward": 0.04106608219444752, "reward_std": 0.006041216081939638, "rewards/accuracy_reward": 0.04106608219444752, "step": 37 }, { "clip_ratio": 0.0, "completion_length": 70.0625, "epoch": 0.02962962962962963, "grad_norm": 2.6091415664520716, "kl": 0.001514434814453125, "learning_rate": 2.945736434108527e-07, "loss": 0.2164, "reward": 0.04012400843203068, "reward_std": 0.010114745702594519, "rewards/accuracy_reward": 0.04012400843203068, "step": 38 }, { "clip_ratio": 0.0, "completion_length": 62.72916793823242, "epoch": 0.0304093567251462, "grad_norm": 2.2160406971755457, "kl": 0.0016937255859375, "learning_rate": 3.023255813953488e-07, "loss": 0.1534, "reward": 0.03779347240924835, "reward_std": 0.005149615230038762, "rewards/accuracy_reward": 0.03779347240924835, "step": 39 }, { "clip_ratio": 0.0, "completion_length": 75.89583587646484, "epoch": 0.031189083820662766, "grad_norm": 2.12251220530095, "kl": 0.001483917236328125, "learning_rate": 3.100775193798449e-07, "loss": 0.0937, "reward": 0.03355565946549177, "reward_std": 0.005913240602239966, "rewards/accuracy_reward": 0.03355565946549177, "step": 40 }, { "clip_ratio": 0.0, "completion_length": 55.770835876464844, "epoch": 0.031968810916179334, "grad_norm": 2.628803783097872, "kl": 0.001811981201171875, "learning_rate": 3.178294573643411e-07, "loss": 0.1487, "reward": 0.04058343544602394, "reward_std": 0.007635206216946244, "rewards/accuracy_reward": 0.04058343544602394, "step": 41 }, { "clip_ratio": 0.0, "completion_length": 69.70833396911621, "epoch": 0.03274853801169591, "grad_norm": 2.9487477608573878, "kl": 0.001972198486328125, "learning_rate": 3.2558139534883724e-07, "loss": 0.1824, "reward": 0.04240095615386963, "reward_std": 0.013564284890890121, "rewards/accuracy_reward": 0.04240095615386963, "step": 42 }, { "clip_ratio": 0.0, "completion_length": 80.83333587646484, "epoch": 0.03352826510721248, "grad_norm": 2.2158750430327725, "kl": 0.001964569091796875, "learning_rate": 3.333333333333333e-07, "loss": 0.4318, "reward": 0.03499251697212458, "reward_std": 0.009471837896853685, "rewards/accuracy_reward": 0.03499251697212458, "step": 43 }, { "clip_ratio": 0.0, "completion_length": 63.354169845581055, "epoch": 0.034307992202729044, "grad_norm": 2.3484723186586662, "kl": 0.0038604736328125, "learning_rate": 3.4108527131782946e-07, "loss": 0.1758, "reward": 0.04066397808492184, "reward_std": 0.012765896040946245, "rewards/accuracy_reward": 0.04066397808492184, "step": 44 }, { "clip_ratio": 0.0, "completion_length": 54.06250190734863, "epoch": 0.03508771929824561, "grad_norm": 3.1431566341022905, "kl": 0.006072998046875, "learning_rate": 3.4883720930232557e-07, "loss": 0.3093, "reward": 0.04505058936774731, "reward_std": 0.009412168525159359, "rewards/accuracy_reward": 0.04505058936774731, "step": 45 }, { "clip_ratio": 0.0, "completion_length": 68.25000190734863, "epoch": 0.03586744639376218, "grad_norm": 2.2081442575684522, "kl": 0.0042266845703125, "learning_rate": 3.5658914728682174e-07, "loss": 0.0523, "reward": 0.0339159881696105, "reward_std": 0.010420492850244045, "rewards/accuracy_reward": 0.0339159881696105, "step": 46 }, { "clip_ratio": 0.0, "completion_length": 54.25000190734863, "epoch": 0.036647173489278755, "grad_norm": 2.8522356482704643, "kl": 0.00418853759765625, "learning_rate": 3.643410852713178e-07, "loss": 0.1387, "reward": 0.04024900496006012, "reward_std": 0.0033548662904649973, "rewards/accuracy_reward": 0.04024900496006012, "step": 47 }, { "clip_ratio": 0.0, "completion_length": 50.70833396911621, "epoch": 0.03742690058479532, "grad_norm": 3.1034968757450483, "kl": 0.00927734375, "learning_rate": 3.7209302325581396e-07, "loss": 0.2677, "reward": 0.04862676560878754, "reward_std": 0.010755512397736311, "rewards/accuracy_reward": 0.04862676560878754, "step": 48 }, { "clip_ratio": 0.0, "completion_length": 67.29166984558105, "epoch": 0.03820662768031189, "grad_norm": 2.3737010947513615, "kl": 0.0064544677734375, "learning_rate": 3.7984496124031006e-07, "loss": 0.2127, "reward": 0.03502965345978737, "reward_std": 0.006760421558283269, "rewards/accuracy_reward": 0.03502965345978737, "step": 49 }, { "clip_ratio": 0.0, "completion_length": 58.625003814697266, "epoch": 0.03898635477582846, "grad_norm": 3.303783495475012, "kl": 0.00738525390625, "learning_rate": 3.8759689922480623e-07, "loss": 0.1732, "reward": 0.04058924503624439, "reward_std": 0.005226747365668416, "rewards/accuracy_reward": 0.04058924503624439, "step": 50 }, { "clip_ratio": 0.0, "completion_length": 43.43750190734863, "epoch": 0.03976608187134503, "grad_norm": 3.113620942783793, "kl": 0.0135498046875, "learning_rate": 3.953488372093023e-07, "loss": 0.0656, "reward": 0.05482747219502926, "reward_std": 0.01050096983090043, "rewards/accuracy_reward": 0.05482747219502926, "step": 51 }, { "clip_ratio": 0.0, "completion_length": 45.62500190734863, "epoch": 0.0405458089668616, "grad_norm": 2.8610826676549244, "kl": 0.009246826171875, "learning_rate": 4.0310077519379845e-07, "loss": 0.0302, "reward": 0.04160502180457115, "reward_std": 0.006433177040889859, "rewards/accuracy_reward": 0.04160502180457115, "step": 52 }, { "clip_ratio": 0.0, "completion_length": 50.00000190734863, "epoch": 0.04132553606237817, "grad_norm": 2.7999246179813975, "kl": 0.01318359375, "learning_rate": 4.1085271317829456e-07, "loss": 0.233, "reward": 0.043028466403484344, "reward_std": 0.003487872891128063, "rewards/accuracy_reward": 0.043028466403484344, "step": 53 }, { "clip_ratio": 0.0, "completion_length": 52.10416793823242, "epoch": 0.042105263157894736, "grad_norm": 2.568960242942063, "kl": 0.01348876953125, "learning_rate": 4.186046511627907e-07, "loss": 0.1753, "reward": 0.038340842351317406, "reward_std": 0.007927641505375504, "rewards/accuracy_reward": 0.038340842351317406, "step": 54 }, { "clip_ratio": 0.0, "completion_length": 57.16666793823242, "epoch": 0.042884990253411304, "grad_norm": 2.4342441327480553, "kl": 0.0233154296875, "learning_rate": 4.263565891472868e-07, "loss": 0.0992, "reward": 0.047976596280932426, "reward_std": 0.013746438082307577, "rewards/accuracy_reward": 0.047976596280932426, "step": 55 }, { "clip_ratio": 0.0, "completion_length": 60.520835876464844, "epoch": 0.04366471734892788, "grad_norm": 2.770172478808812, "kl": 0.023834228515625, "learning_rate": 4.3410852713178294e-07, "loss": 0.2833, "reward": 0.03680317848920822, "reward_std": 0.005510715185664594, "rewards/accuracy_reward": 0.03680317848920822, "step": 56 }, { "clip_ratio": 0.0, "completion_length": 39.35416793823242, "epoch": 0.044444444444444446, "grad_norm": 3.68010060383285, "kl": 0.03204345703125, "learning_rate": 4.4186046511627905e-07, "loss": 0.1525, "reward": 0.04910232499241829, "reward_std": 0.007614273577928543, "rewards/accuracy_reward": 0.04910232499241829, "step": 57 }, { "clip_ratio": 0.0, "completion_length": 53.250003814697266, "epoch": 0.045224171539961014, "grad_norm": 2.70642849083914, "kl": 0.0291748046875, "learning_rate": 4.496124031007752e-07, "loss": 0.0844, "reward": 0.04976135119795799, "reward_std": 0.011796995531767607, "rewards/accuracy_reward": 0.04976135119795799, "step": 58 }, { "clip_ratio": 0.0, "completion_length": 50.895835876464844, "epoch": 0.04600389863547758, "grad_norm": 2.812435002261949, "kl": 0.03460693359375, "learning_rate": 4.5736434108527127e-07, "loss": 0.0578, "reward": 0.047287384048104286, "reward_std": 0.011063814163208008, "rewards/accuracy_reward": 0.047287384048104286, "step": 59 }, { "clip_ratio": 0.0, "completion_length": 46.60416793823242, "epoch": 0.04678362573099415, "grad_norm": 3.1631616288164066, "kl": 0.050537109375, "learning_rate": 4.6511627906976743e-07, "loss": 0.1621, "reward": 0.04765086621046066, "reward_std": 0.007884890539571643, "rewards/accuracy_reward": 0.04765086621046066, "step": 60 }, { "clip_ratio": 0.0, "completion_length": 52.125, "epoch": 0.047563352826510724, "grad_norm": 3.4169815314016287, "kl": 0.040283203125, "learning_rate": 4.7286821705426354e-07, "loss": 0.1953, "reward": 0.037693820893764496, "reward_std": 0.0043515488505363464, "rewards/accuracy_reward": 0.037693820893764496, "step": 61 }, { "clip_ratio": 0.0, "completion_length": 44.89583396911621, "epoch": 0.04834307992202729, "grad_norm": 3.7525171815893428, "kl": 0.0565185546875, "learning_rate": 4.806201550387597e-07, "loss": 0.2159, "reward": 0.0389871671795845, "reward_std": 0.004751588683575392, "rewards/accuracy_reward": 0.0389871671795845, "step": 62 }, { "clip_ratio": 0.0, "completion_length": 44.06250190734863, "epoch": 0.04912280701754386, "grad_norm": 3.1742788723870086, "kl": 0.070068359375, "learning_rate": 4.883720930232558e-07, "loss": 0.0506, "reward": 0.05800568871200085, "reward_std": 0.008456971496343613, "rewards/accuracy_reward": 0.05800568871200085, "step": 63 }, { "clip_ratio": 0.0, "completion_length": 39.72916793823242, "epoch": 0.04990253411306043, "grad_norm": 3.3869515054521253, "kl": 0.085205078125, "learning_rate": 4.961240310077519e-07, "loss": -0.0272, "reward": 0.07096873968839645, "reward_std": 0.01401633769273758, "rewards/accuracy_reward": 0.07096873968839645, "step": 64 }, { "clip_ratio": 0.0, "completion_length": 51.208335876464844, "epoch": 0.050682261208576995, "grad_norm": 2.2197188203978144, "kl": 0.070068359375, "learning_rate": 5.038759689922481e-07, "loss": 0.1532, "reward": 0.04562339745461941, "reward_std": 0.007543514017015696, "rewards/accuracy_reward": 0.04562339745461941, "step": 65 }, { "clip_ratio": 0.0, "completion_length": 34.43750190734863, "epoch": 0.05146198830409357, "grad_norm": 3.6225373710567994, "kl": 0.07861328125, "learning_rate": 5.116279069767442e-07, "loss": 0.1782, "reward": 0.04998355731368065, "reward_std": 0.004972400842234492, "rewards/accuracy_reward": 0.04998355731368065, "step": 66 }, { "clip_ratio": 0.0, "completion_length": 48.3125, "epoch": 0.05224171539961014, "grad_norm": 4.479964724095867, "kl": 0.093017578125, "learning_rate": 5.193798449612403e-07, "loss": 0.1327, "reward": 0.05973537266254425, "reward_std": 0.01126884389668703, "rewards/accuracy_reward": 0.05973537266254425, "step": 67 }, { "clip_ratio": 0.0, "completion_length": 42.08333396911621, "epoch": 0.053021442495126705, "grad_norm": 3.47463936829341, "kl": 0.111328125, "learning_rate": 5.271317829457364e-07, "loss": 0.0991, "reward": 0.05231078714132309, "reward_std": 0.004367304500192404, "rewards/accuracy_reward": 0.05231078714132309, "step": 68 }, { "clip_ratio": 0.0, "completion_length": 45.85416793823242, "epoch": 0.05380116959064327, "grad_norm": 2.984127507818342, "kl": 0.0819091796875, "learning_rate": 5.348837209302325e-07, "loss": 0.0718, "reward": 0.04398638755083084, "reward_std": 0.0028009897796437144, "rewards/accuracy_reward": 0.04398638755083084, "step": 69 }, { "clip_ratio": 0.0, "completion_length": 53.29166793823242, "epoch": 0.05458089668615984, "grad_norm": 2.5160706221746945, "kl": 0.130126953125, "learning_rate": 5.426356589147286e-07, "loss": 0.1763, "reward": 0.054356709122657776, "reward_std": 0.009095613844692707, "rewards/accuracy_reward": 0.054356709122657776, "step": 70 }, { "clip_ratio": 0.0, "completion_length": 31.187501907348633, "epoch": 0.055360623781676416, "grad_norm": 3.5612309454881075, "kl": 0.18359375, "learning_rate": 5.503875968992247e-07, "loss": 0.064, "reward": 0.06594585627317429, "reward_std": 0.008575513027608395, "rewards/accuracy_reward": 0.06594585627317429, "step": 71 }, { "clip_ratio": 0.0, "completion_length": 36.5, "epoch": 0.056140350877192984, "grad_norm": 2.8448103966973943, "kl": 0.176025390625, "learning_rate": 5.581395348837209e-07, "loss": 0.0491, "reward": 0.06807346642017365, "reward_std": 0.011498718522489071, "rewards/accuracy_reward": 0.06807346642017365, "step": 72 }, { "clip_ratio": 0.0, "completion_length": 34.95833396911621, "epoch": 0.05692007797270955, "grad_norm": 4.3184359855094945, "kl": 0.2158203125, "learning_rate": 5.658914728682171e-07, "loss": 0.1619, "reward": 0.04520327411592007, "reward_std": 0.0040213235188275576, "rewards/accuracy_reward": 0.04520327411592007, "step": 73 }, { "clip_ratio": 0.0, "completion_length": 43.50000190734863, "epoch": 0.05769980506822612, "grad_norm": 3.1250028835345613, "kl": 0.20654296875, "learning_rate": 5.736434108527132e-07, "loss": 0.0279, "reward": 0.06468020007014275, "reward_std": 0.014381224755197763, "rewards/accuracy_reward": 0.06468020007014275, "step": 74 }, { "clip_ratio": 0.0, "completion_length": 34.687500953674316, "epoch": 0.05847953216374269, "grad_norm": 3.779421591825818, "kl": 0.208984375, "learning_rate": 5.813953488372093e-07, "loss": 0.2085, "reward": 0.04720890522003174, "reward_std": 0.005046627949923277, "rewards/accuracy_reward": 0.04720890522003174, "step": 75 }, { "clip_ratio": 0.0, "completion_length": 32.45833396911621, "epoch": 0.05925925925925926, "grad_norm": 3.967730635271566, "kl": 0.21044921875, "learning_rate": 5.891472868217054e-07, "loss": 0.0697, "reward": 0.07102777436375618, "reward_std": 0.010164339561015368, "rewards/accuracy_reward": 0.07102777436375618, "step": 76 }, { "clip_ratio": 0.0, "completion_length": 47.458335876464844, "epoch": 0.06003898635477583, "grad_norm": 3.80599765313446, "kl": 0.20361328125, "learning_rate": 5.968992248062015e-07, "loss": 0.2024, "reward": 0.04373868927359581, "reward_std": 0.0035682034213095903, "rewards/accuracy_reward": 0.04373868927359581, "step": 77 }, { "clip_ratio": 0.0, "completion_length": 36.95833396911621, "epoch": 0.0608187134502924, "grad_norm": 4.269746663131536, "kl": 0.18994140625, "learning_rate": 6.046511627906976e-07, "loss": 0.1813, "reward": 0.04619111493229866, "reward_std": 0.004070800496265292, "rewards/accuracy_reward": 0.04619111493229866, "step": 78 }, { "clip_ratio": 0.0, "completion_length": 27.33333396911621, "epoch": 0.061598440545808965, "grad_norm": 4.4134349427597765, "kl": 0.218994140625, "learning_rate": 6.124031007751937e-07, "loss": 0.1671, "reward": 0.04927775636315346, "reward_std": 0.0030711302533745766, "rewards/accuracy_reward": 0.04927775636315346, "step": 79 }, { "clip_ratio": 0.0, "completion_length": 42.35416793823242, "epoch": 0.06237816764132553, "grad_norm": 4.268446019202903, "kl": 0.1572265625, "learning_rate": 6.201550387596898e-07, "loss": 0.2639, "reward": 0.043139971792697906, "reward_std": 0.005017469171434641, "rewards/accuracy_reward": 0.043139971792697906, "step": 80 }, { "clip_ratio": 0.0, "completion_length": 49.14583396911621, "epoch": 0.06315789473684211, "grad_norm": 2.4508441348623284, "kl": 0.1826171875, "learning_rate": 6.27906976744186e-07, "loss": 0.0904, "reward": 0.04530907794833183, "reward_std": 0.0029870180878788233, "rewards/accuracy_reward": 0.04530907794833183, "step": 81 }, { "clip_ratio": 0.0, "completion_length": 39.875, "epoch": 0.06393762183235867, "grad_norm": 4.338489355975129, "kl": 0.30126953125, "learning_rate": 6.356589147286822e-07, "loss": 0.0967, "reward": 0.06821105815470219, "reward_std": 0.008517026668414474, "rewards/accuracy_reward": 0.06821105815470219, "step": 82 }, { "clip_ratio": 0.0, "completion_length": 43.04166793823242, "epoch": 0.06471734892787524, "grad_norm": 2.908327544492862, "kl": 0.1826171875, "learning_rate": 6.434108527131783e-07, "loss": 0.047, "reward": 0.04304710030555725, "reward_std": 0.00637028506025672, "rewards/accuracy_reward": 0.04304710030555725, "step": 83 }, { "clip_ratio": 0.0, "completion_length": 37.3125, "epoch": 0.06549707602339182, "grad_norm": 2.3979524790932225, "kl": 0.106201171875, "learning_rate": 6.511627906976745e-07, "loss": 0.0158, "reward": 0.054630450904369354, "reward_std": 0.0019857838633470237, "rewards/accuracy_reward": 0.054630450904369354, "step": 84 }, { "clip_ratio": 0.0, "completion_length": 35.8125, "epoch": 0.06627680311890838, "grad_norm": 2.7520842907479537, "kl": 0.18408203125, "learning_rate": 6.589147286821705e-07, "loss": 0.1055, "reward": 0.06916381046175957, "reward_std": 0.005404126946814358, "rewards/accuracy_reward": 0.06916381046175957, "step": 85 }, { "clip_ratio": 0.0, "completion_length": 46.47916793823242, "epoch": 0.06705653021442495, "grad_norm": 3.0163826480352762, "kl": 0.2412109375, "learning_rate": 6.666666666666666e-07, "loss": 0.0081, "reward": 0.04147185757756233, "reward_std": 0.0031410371884703636, "rewards/accuracy_reward": 0.04147185757756233, "step": 86 }, { "clip_ratio": 0.0, "completion_length": 33.89583396911621, "epoch": 0.06783625730994151, "grad_norm": 3.040312568921289, "kl": 0.1787109375, "learning_rate": 6.744186046511627e-07, "loss": 0.1352, "reward": 0.04949432238936424, "reward_std": 0.004746574675664306, "rewards/accuracy_reward": 0.04949432238936424, "step": 87 }, { "clip_ratio": 0.0, "completion_length": 32.47916793823242, "epoch": 0.06861598440545809, "grad_norm": 4.513511687058106, "kl": 0.296875, "learning_rate": 6.821705426356589e-07, "loss": 0.0967, "reward": 0.047069137915968895, "reward_std": 0.00340858765412122, "rewards/accuracy_reward": 0.047069137915968895, "step": 88 }, { "clip_ratio": 0.0, "completion_length": 36.70833396911621, "epoch": 0.06939571150097466, "grad_norm": 3.2046936061466686, "kl": 0.185546875, "learning_rate": 6.89922480620155e-07, "loss": 0.1379, "reward": 0.045855697244405746, "reward_std": 0.002462368458509445, "rewards/accuracy_reward": 0.045855697244405746, "step": 89 }, { "clip_ratio": 0.0, "completion_length": 42.83333396911621, "epoch": 0.07017543859649122, "grad_norm": 3.4631744406118687, "kl": 0.28125, "learning_rate": 6.976744186046511e-07, "loss": 0.1755, "reward": 0.07655094563961029, "reward_std": 0.01264206226915121, "rewards/accuracy_reward": 0.07655094563961029, "step": 90 }, { "clip_ratio": 0.0, "completion_length": 35.10416793823242, "epoch": 0.0709551656920078, "grad_norm": 2.8068188607333937, "kl": 0.30419921875, "learning_rate": 7.054263565891473e-07, "loss": 0.0932, "reward": 0.05006732605397701, "reward_std": 0.005144392838701606, "rewards/accuracy_reward": 0.05006732605397701, "step": 91 }, { "clip_ratio": 0.0, "completion_length": 31.416667938232422, "epoch": 0.07173489278752436, "grad_norm": 3.6618368049219336, "kl": 0.43359375, "learning_rate": 7.131782945736435e-07, "loss": 0.0811, "reward": 0.047380127012729645, "reward_std": 0.005735602229833603, "rewards/accuracy_reward": 0.047380127012729645, "step": 92 }, { "clip_ratio": 0.0, "completion_length": 36.72916793823242, "epoch": 0.07251461988304093, "grad_norm": 2.808379222783262, "kl": 0.165771484375, "learning_rate": 7.209302325581395e-07, "loss": 0.0626, "reward": 0.040977967903018, "reward_std": 0.0020597196416929364, "rewards/accuracy_reward": 0.040977967903018, "step": 93 }, { "clip_ratio": 0.0, "completion_length": 35.20833396911621, "epoch": 0.07329434697855751, "grad_norm": 2.641084827728231, "kl": 0.2216796875, "learning_rate": 7.286821705426356e-07, "loss": -0.0525, "reward": 0.06772063486278057, "reward_std": 0.00525827007368207, "rewards/accuracy_reward": 0.06772063486278057, "step": 94 }, { "clip_ratio": 0.0, "completion_length": 34.479166984558105, "epoch": 0.07407407407407407, "grad_norm": 2.3420543141817687, "kl": 0.3154296875, "learning_rate": 7.364341085271317e-07, "loss": 0.033, "reward": 0.047532547265291214, "reward_std": 0.0015460846479982138, "rewards/accuracy_reward": 0.047532547265291214, "step": 95 }, { "clip_ratio": 0.0, "completion_length": 37.25, "epoch": 0.07485380116959064, "grad_norm": 3.8272598250695147, "kl": 0.3349609375, "learning_rate": 7.441860465116279e-07, "loss": 0.0745, "reward": 0.0494490060955286, "reward_std": 0.006539649562910199, "rewards/accuracy_reward": 0.0494490060955286, "step": 96 }, { "clip_ratio": 0.0, "completion_length": 37.66666793823242, "epoch": 0.07563352826510722, "grad_norm": 2.8283134363372766, "kl": 0.2939453125, "learning_rate": 7.51937984496124e-07, "loss": 0.0619, "reward": 0.06136970967054367, "reward_std": 0.005943382391706109, "rewards/accuracy_reward": 0.06136970967054367, "step": 97 }, { "clip_ratio": 0.0, "completion_length": 38.95833396911621, "epoch": 0.07641325536062378, "grad_norm": 4.0926550610955825, "kl": 0.26318359375, "learning_rate": 7.596899224806201e-07, "loss": 0.0133, "reward": 0.05305527709424496, "reward_std": 0.010987138841301203, "rewards/accuracy_reward": 0.05305527709424496, "step": 98 }, { "clip_ratio": 0.0, "completion_length": 27.08333396911621, "epoch": 0.07719298245614035, "grad_norm": 2.227907651484132, "kl": 0.20556640625, "learning_rate": 7.674418604651162e-07, "loss": -0.0106, "reward": 0.06309524551033974, "reward_std": 0.0008820741786621511, "rewards/accuracy_reward": 0.06309524551033974, "step": 99 }, { "clip_ratio": 0.0, "completion_length": 37.35416793823242, "epoch": 0.07797270955165692, "grad_norm": 3.4502302886910363, "kl": 0.26708984375, "learning_rate": 7.751937984496125e-07, "loss": 0.0379, "reward": 0.04482625797390938, "reward_std": 0.0022178655490279198, "rewards/accuracy_reward": 0.04482625797390938, "step": 100 }, { "clip_ratio": 0.0, "completion_length": 42.4375, "epoch": 0.07875243664717349, "grad_norm": 3.0138541752807932, "kl": 0.25830078125, "learning_rate": 7.829457364341085e-07, "loss": 0.0461, "reward": 0.04530594311654568, "reward_std": 0.0014333150465972722, "rewards/accuracy_reward": 0.04530594311654568, "step": 101 }, { "clip_ratio": 0.0, "completion_length": 43.08333396911621, "epoch": 0.07953216374269007, "grad_norm": 2.9524828938696444, "kl": 0.244140625, "learning_rate": 7.906976744186046e-07, "loss": 0.07, "reward": 0.0649220421910286, "reward_std": 0.01101213088259101, "rewards/accuracy_reward": 0.0649220421910286, "step": 102 }, { "clip_ratio": 0.0, "completion_length": 47.50000190734863, "epoch": 0.08031189083820663, "grad_norm": 2.6451002225368594, "kl": 0.19970703125, "learning_rate": 7.984496124031007e-07, "loss": 0.1396, "reward": 0.05022517405450344, "reward_std": 0.002265424351207912, "rewards/accuracy_reward": 0.05022517405450344, "step": 103 }, { "clip_ratio": 0.0, "completion_length": 46.10416793823242, "epoch": 0.0810916179337232, "grad_norm": 2.1928991316645, "kl": 0.26171875, "learning_rate": 8.062015503875969e-07, "loss": 0.0561, "reward": 0.045890724286437035, "reward_std": 0.006344977300614119, "rewards/accuracy_reward": 0.045890724286437035, "step": 104 }, { "clip_ratio": 0.0, "completion_length": 59.72916793823242, "epoch": 0.08187134502923976, "grad_norm": 2.0426574583498014, "kl": 0.17578125, "learning_rate": 8.13953488372093e-07, "loss": 0.0054, "reward": 0.06249103508889675, "reward_std": 0.0027366310823708773, "rewards/accuracy_reward": 0.06249103508889675, "step": 105 }, { "clip_ratio": 0.0, "completion_length": 37.37500190734863, "epoch": 0.08265107212475634, "grad_norm": 3.2710869824871627, "kl": 0.419921875, "learning_rate": 8.217054263565891e-07, "loss": 0.0478, "reward": 0.09790483117103577, "reward_std": 0.00470519601367414, "rewards/accuracy_reward": 0.09790483117103577, "step": 106 }, { "clip_ratio": 0.0, "completion_length": 45.10416793823242, "epoch": 0.08343079922027291, "grad_norm": 1.9761863371392694, "kl": 0.2353515625, "learning_rate": 8.294573643410852e-07, "loss": 0.0536, "reward": 0.04955454729497433, "reward_std": 0.0018941238522529602, "rewards/accuracy_reward": 0.04955454729497433, "step": 107 }, { "clip_ratio": 0.0, "completion_length": 48.54166793823242, "epoch": 0.08421052631578947, "grad_norm": 2.2771383882088854, "kl": 0.3203125, "learning_rate": 8.372093023255814e-07, "loss": 0.0618, "reward": 0.06308815255761147, "reward_std": 0.007483313325792551, "rewards/accuracy_reward": 0.06308815255761147, "step": 108 }, { "clip_ratio": 0.0, "completion_length": 38.72916793823242, "epoch": 0.08499025341130605, "grad_norm": 2.2159968094278586, "kl": 0.23046875, "learning_rate": 8.449612403100774e-07, "loss": 0.0829, "reward": 0.05840716511011124, "reward_std": 0.004084525164216757, "rewards/accuracy_reward": 0.05840716511011124, "step": 109 }, { "clip_ratio": 0.0, "completion_length": 38.895835876464844, "epoch": 0.08576998050682261, "grad_norm": 3.590058268549366, "kl": 0.4697265625, "learning_rate": 8.527131782945736e-07, "loss": 0.0697, "reward": 0.062450679019093513, "reward_std": 0.007699631154537201, "rewards/accuracy_reward": 0.062450679019093513, "step": 110 }, { "clip_ratio": 0.0, "completion_length": 37.60416793823242, "epoch": 0.08654970760233918, "grad_norm": 2.970550745807065, "kl": 0.3603515625, "learning_rate": 8.604651162790697e-07, "loss": 0.1128, "reward": 0.07873068004846573, "reward_std": 0.0034102650824934244, "rewards/accuracy_reward": 0.07873068004846573, "step": 111 }, { "clip_ratio": 0.0, "completion_length": 37.33333396911621, "epoch": 0.08732943469785576, "grad_norm": 2.159898735456206, "kl": 0.279296875, "learning_rate": 8.682170542635659e-07, "loss": 0.0268, "reward": 0.08472547307610512, "reward_std": 0.004890290321782231, "rewards/accuracy_reward": 0.08472547307610512, "step": 112 }, { "clip_ratio": 0.0, "completion_length": 41.93750190734863, "epoch": 0.08810916179337232, "grad_norm": 2.4048395200356865, "kl": 0.373046875, "learning_rate": 8.75968992248062e-07, "loss": 0.0822, "reward": 0.044123949483036995, "reward_std": 0.002274714468512684, "rewards/accuracy_reward": 0.044123949483036995, "step": 113 }, { "clip_ratio": 0.0, "completion_length": 44.79166793823242, "epoch": 0.08888888888888889, "grad_norm": 2.110076357861851, "kl": 0.349609375, "learning_rate": 8.837209302325581e-07, "loss": 0.0353, "reward": 0.05389041267335415, "reward_std": 0.004451837099622935, "rewards/accuracy_reward": 0.05389041267335415, "step": 114 }, { "clip_ratio": 0.0, "completion_length": 44.375000953674316, "epoch": 0.08966861598440545, "grad_norm": 2.388794050449426, "kl": 0.32470703125, "learning_rate": 8.914728682170542e-07, "loss": 0.0636, "reward": 0.05829278379678726, "reward_std": 0.0021095441188663244, "rewards/accuracy_reward": 0.05829278379678726, "step": 115 }, { "clip_ratio": 0.0, "completion_length": 32.08333396911621, "epoch": 0.09044834307992203, "grad_norm": 2.2972470263546634, "kl": 0.23583984375, "learning_rate": 8.992248062015504e-07, "loss": 0.0318, "reward": 0.05514957569539547, "reward_std": 0.0009780407708603889, "rewards/accuracy_reward": 0.05514957569539547, "step": 116 }, { "clip_ratio": 0.0, "completion_length": 42.43750190734863, "epoch": 0.0912280701754386, "grad_norm": 2.9001639996696222, "kl": 0.2783203125, "learning_rate": 9.069767441860464e-07, "loss": 0.2305, "reward": 0.06270255520939827, "reward_std": 0.004146690247580409, "rewards/accuracy_reward": 0.06270255520939827, "step": 117 }, { "clip_ratio": 0.0, "completion_length": 39.14583396911621, "epoch": 0.09200779727095516, "grad_norm": 2.0766943646013654, "kl": 0.3134765625, "learning_rate": 9.147286821705425e-07, "loss": 0.0964, "reward": 0.07618878036737442, "reward_std": 0.002100339625030756, "rewards/accuracy_reward": 0.07618878036737442, "step": 118 }, { "clip_ratio": 0.0, "completion_length": 39.29166793823242, "epoch": 0.09278752436647174, "grad_norm": 2.4811588444022363, "kl": 0.443359375, "learning_rate": 9.224806201550386e-07, "loss": 0.0313, "reward": 0.09027307853102684, "reward_std": 0.006509718834422529, "rewards/accuracy_reward": 0.09027307853102684, "step": 119 }, { "clip_ratio": 0.0, "completion_length": 36.9375, "epoch": 0.0935672514619883, "grad_norm": 1.594512917972913, "kl": 0.251953125, "learning_rate": 9.302325581395349e-07, "loss": 0.0489, "reward": 0.07478075847029686, "reward_std": 0.0012831513013225049, "rewards/accuracy_reward": 0.07478075847029686, "step": 120 }, { "clip_ratio": 0.0, "completion_length": 46.93750190734863, "epoch": 0.09434697855750487, "grad_norm": 2.360172429007107, "kl": 0.4814453125, "learning_rate": 9.37984496124031e-07, "loss": 0.0438, "reward": 0.052850114181637764, "reward_std": 0.005657479981891811, "rewards/accuracy_reward": 0.052850114181637764, "step": 121 }, { "clip_ratio": 0.0, "completion_length": 44.66666793823242, "epoch": 0.09512670565302145, "grad_norm": 1.7782689608680646, "kl": 0.29638671875, "learning_rate": 9.457364341085271e-07, "loss": 0.0316, "reward": 0.04962804540991783, "reward_std": 0.0010844895150512457, "rewards/accuracy_reward": 0.04962804540991783, "step": 122 }, { "clip_ratio": 0.0, "completion_length": 41.81250190734863, "epoch": 0.09590643274853801, "grad_norm": 2.064930953634142, "kl": 0.279296875, "learning_rate": 9.534883720930232e-07, "loss": 0.0474, "reward": 0.06100150756537914, "reward_std": 0.00565731490496546, "rewards/accuracy_reward": 0.06100150756537914, "step": 123 }, { "clip_ratio": 0.0, "completion_length": 44.60416793823242, "epoch": 0.09668615984405458, "grad_norm": 2.7146972146100303, "kl": 0.3837890625, "learning_rate": 9.612403100775193e-07, "loss": 0.2327, "reward": 0.04944268800318241, "reward_std": 0.004167388891801238, "rewards/accuracy_reward": 0.04944268800318241, "step": 124 }, { "clip_ratio": 0.0, "completion_length": 45.43750190734863, "epoch": 0.09746588693957114, "grad_norm": 1.315801786583085, "kl": 0.2353515625, "learning_rate": 9.689922480620153e-07, "loss": -0.0062, "reward": 0.06027457118034363, "reward_std": 0.0011762951035052538, "rewards/accuracy_reward": 0.06027457118034363, "step": 125 }, { "clip_ratio": 0.0, "completion_length": 46.31250190734863, "epoch": 0.09824561403508772, "grad_norm": 1.7782273137093043, "kl": 0.2802734375, "learning_rate": 9.767441860465115e-07, "loss": 0.0041, "reward": 0.05181322619318962, "reward_std": 0.004530643695034087, "rewards/accuracy_reward": 0.05181322619318962, "step": 126 }, { "clip_ratio": 0.0, "completion_length": 38.64583396911621, "epoch": 0.0990253411306043, "grad_norm": 1.7871033542884303, "kl": 0.25048828125, "learning_rate": 9.844961240310077e-07, "loss": 0.0708, "reward": 0.05114087648689747, "reward_std": 0.0008208061335608363, "rewards/accuracy_reward": 0.05114087648689747, "step": 127 }, { "clip_ratio": 0.0, "completion_length": 37.333335876464844, "epoch": 0.09980506822612085, "grad_norm": 2.0417982330649433, "kl": 0.314453125, "learning_rate": 9.922480620155037e-07, "loss": 0.0212, "reward": 0.06282653287053108, "reward_std": 0.0034878100268542767, "rewards/accuracy_reward": 0.06282653287053108, "step": 128 }, { "clip_ratio": 0.0, "completion_length": 39.27083492279053, "epoch": 0.10058479532163743, "grad_norm": 1.58961800509684, "kl": 0.3310546875, "learning_rate": 1e-06, "loss": 0.0221, "reward": 0.06363019719719887, "reward_std": 0.003304417827166617, "rewards/accuracy_reward": 0.06363019719719887, "step": 129 }, { "clip_ratio": 0.0, "completion_length": 34.58333396911621, "epoch": 0.10136452241715399, "grad_norm": 1.0712022227545728, "kl": 0.28125, "learning_rate": 9.9999832958885e-07, "loss": 0.0152, "reward": 0.0661378987133503, "reward_std": 0.0005932368221692741, "rewards/accuracy_reward": 0.0661378987133503, "step": 130 }, { "clip_ratio": 0.0, "completion_length": 50.77083396911621, "epoch": 0.10214424951267057, "grad_norm": 3.841618839193188, "kl": 0.490234375, "learning_rate": 9.999933183678015e-07, "loss": 0.0975, "reward": 0.09110459685325623, "reward_std": 0.004217098816297948, "rewards/accuracy_reward": 0.09110459685325623, "step": 131 }, { "clip_ratio": 0.0, "completion_length": 44.08333396911621, "epoch": 0.10292397660818714, "grad_norm": 3.4945102917866286, "kl": 0.4404296875, "learning_rate": 9.99984966374058e-07, "loss": 0.0711, "reward": 0.05980168469250202, "reward_std": 0.004065339686349034, "rewards/accuracy_reward": 0.05980168469250202, "step": 132 }, { "clip_ratio": 0.0, "completion_length": 39.750000953674316, "epoch": 0.1037037037037037, "grad_norm": 1.0296024150123595, "kl": 0.3134765625, "learning_rate": 9.99973273669625e-07, "loss": 0.0378, "reward": 0.0822172649204731, "reward_std": 0.0013668336905539036, "rewards/accuracy_reward": 0.0822172649204731, "step": 133 }, { "clip_ratio": 0.0, "completion_length": 42.60416793823242, "epoch": 0.10448343079922028, "grad_norm": 1.5599395926534922, "kl": 0.306640625, "learning_rate": 9.9995824034131e-07, "loss": 0.0433, "reward": 0.07356821000576019, "reward_std": 0.005668101832270622, "rewards/accuracy_reward": 0.07356821000576019, "step": 134 }, { "clip_ratio": 0.0, "completion_length": 37.6875, "epoch": 0.10526315789473684, "grad_norm": 1.4729301419390965, "kl": 0.3740234375, "learning_rate": 9.999398665007205e-07, "loss": 0.0572, "reward": 0.04963262751698494, "reward_std": 0.0009213387966156006, "rewards/accuracy_reward": 0.04963262751698494, "step": 135 }, { "clip_ratio": 0.0, "completion_length": 45.43750190734863, "epoch": 0.10604288499025341, "grad_norm": 2.154918468516356, "kl": 0.4912109375, "learning_rate": 9.999181522842657e-07, "loss": 0.0624, "reward": 0.06199992448091507, "reward_std": 0.004981459584087133, "rewards/accuracy_reward": 0.06199992448091507, "step": 136 }, { "clip_ratio": 0.0, "completion_length": 44.6875, "epoch": 0.10682261208576999, "grad_norm": 1.6178102316186667, "kl": 0.2998046875, "learning_rate": 9.998930978531523e-07, "loss": -0.0059, "reward": 0.07799461856484413, "reward_std": 0.0033870727056637406, "rewards/accuracy_reward": 0.07799461856484413, "step": 137 }, { "clip_ratio": 0.0, "completion_length": 47.95833396911621, "epoch": 0.10760233918128655, "grad_norm": 2.011618952941971, "kl": 0.22314453125, "learning_rate": 9.998647033933862e-07, "loss": 0.1619, "reward": 0.07276952639222145, "reward_std": 0.0057456461363472044, "rewards/accuracy_reward": 0.07276952639222145, "step": 138 }, { "clip_ratio": 0.0, "completion_length": 40.93750190734863, "epoch": 0.10838206627680312, "grad_norm": 1.1953395051316285, "kl": 0.3056640625, "learning_rate": 9.998329691157686e-07, "loss": 0.0179, "reward": 0.0824020504951477, "reward_std": 0.0007267132750712335, "rewards/accuracy_reward": 0.0824020504951477, "step": 139 }, { "clip_ratio": 0.0, "completion_length": 45.77083396911621, "epoch": 0.10916179337231968, "grad_norm": 1.4863258665541164, "kl": 0.26513671875, "learning_rate": 9.997978952558972e-07, "loss": 0.0707, "reward": 0.06538177467882633, "reward_std": 0.0016729276394471526, "rewards/accuracy_reward": 0.06538177467882633, "step": 140 }, { "clip_ratio": 0.0, "completion_length": 43.93750190734863, "epoch": 0.10994152046783626, "grad_norm": 1.2974485192868461, "kl": 0.31298828125, "learning_rate": 9.997594820741615e-07, "loss": 0.045, "reward": 0.0717059038579464, "reward_std": 0.005708017386496067, "rewards/accuracy_reward": 0.0717059038579464, "step": 141 }, { "clip_ratio": 0.0, "completion_length": 39.47916793823242, "epoch": 0.11072124756335283, "grad_norm": 1.738706218029015, "kl": 0.34765625, "learning_rate": 9.997177298557428e-07, "loss": 0.0916, "reward": 0.06964311562478542, "reward_std": 0.0017773142317309976, "rewards/accuracy_reward": 0.06964311562478542, "step": 142 }, { "clip_ratio": 0.0, "completion_length": 47.70833396911621, "epoch": 0.11150097465886939, "grad_norm": 2.617996250225219, "kl": 0.4169921875, "learning_rate": 9.996726389106119e-07, "loss": 0.0562, "reward": 0.05835900455713272, "reward_std": 0.0016873643035069108, "rewards/accuracy_reward": 0.05835900455713272, "step": 143 }, { "clip_ratio": 0.0, "completion_length": 47.33333396911621, "epoch": 0.11228070175438597, "grad_norm": 2.099286006908965, "kl": 0.431640625, "learning_rate": 9.99624209573526e-07, "loss": 0.0435, "reward": 0.056647004559636116, "reward_std": 0.005114392610266805, "rewards/accuracy_reward": 0.056647004559636116, "step": 144 }, { "clip_ratio": 0.0, "completion_length": 43.8125, "epoch": 0.11306042884990253, "grad_norm": 1.2752446361191032, "kl": 0.3056640625, "learning_rate": 9.995724422040273e-07, "loss": 0.0446, "reward": 0.04826555773615837, "reward_std": 0.0007271187496371567, "rewards/accuracy_reward": 0.04826555773615837, "step": 145 }, { "clip_ratio": 0.0, "completion_length": 34.91666793823242, "epoch": 0.1138401559454191, "grad_norm": 0.8139602608611531, "kl": 0.38671875, "learning_rate": 9.99517337186439e-07, "loss": 0.0212, "reward": 0.06071879714727402, "reward_std": 0.0009029456123244017, "rewards/accuracy_reward": 0.06071879714727402, "step": 146 }, { "clip_ratio": 0.0, "completion_length": 39.25000190734863, "epoch": 0.11461988304093568, "grad_norm": 2.3088936352215654, "kl": 0.5126953125, "learning_rate": 9.994588949298635e-07, "loss": -0.0505, "reward": 0.05469323694705963, "reward_std": 0.00674125796649605, "rewards/accuracy_reward": 0.05469323694705963, "step": 147 }, { "clip_ratio": 0.0, "completion_length": 42.20833396911621, "epoch": 0.11539961013645224, "grad_norm": 2.152011093405869, "kl": 0.3154296875, "learning_rate": 9.993971158681788e-07, "loss": 0.0024, "reward": 0.0823461264371872, "reward_std": 0.0011642208555713296, "rewards/accuracy_reward": 0.0823461264371872, "step": 148 }, { "clip_ratio": 0.0, "completion_length": 41.50000190734863, "epoch": 0.11617933723196881, "grad_norm": 1.5947713011820153, "kl": 0.396484375, "learning_rate": 9.993320004600362e-07, "loss": 0.0808, "reward": 0.06809847429394722, "reward_std": 0.0017332308925688267, "rewards/accuracy_reward": 0.06809847429394722, "step": 149 }, { "clip_ratio": 0.0, "completion_length": 32.22916793823242, "epoch": 0.11695906432748537, "grad_norm": 1.432565738285189, "kl": 0.255859375, "learning_rate": 9.992635491888554e-07, "loss": 0.0147, "reward": 0.05747877433896065, "reward_std": 0.00021101986931171268, "rewards/accuracy_reward": 0.05747877433896065, "step": 150 }, { "clip_ratio": 0.0, "completion_length": 42.29166793823242, "epoch": 0.11773879142300195, "grad_norm": 1.5598261656388719, "kl": 0.3251953125, "learning_rate": 9.991917625628223e-07, "loss": 0.0533, "reward": 0.05868055857717991, "reward_std": 0.007278923876583576, "rewards/accuracy_reward": 0.05868055857717991, "step": 151 }, { "clip_ratio": 0.0, "completion_length": 31.625000953674316, "epoch": 0.11851851851851852, "grad_norm": 3.0936394994220238, "kl": 0.27294921875, "learning_rate": 9.99116641114884e-07, "loss": 0.0408, "reward": 0.07421398535370827, "reward_std": 0.01793223712593317, "rewards/accuracy_reward": 0.07421398535370827, "step": 152 }, { "clip_ratio": 0.0, "completion_length": 36.875, "epoch": 0.11929824561403508, "grad_norm": 3.6524430060918296, "kl": 0.458984375, "learning_rate": 9.99038185402746e-07, "loss": 0.0116, "reward": 0.08958383277058601, "reward_std": 0.01494149150676094, "rewards/accuracy_reward": 0.08958383277058601, "step": 153 }, { "clip_ratio": 0.0, "completion_length": 40.27083396911621, "epoch": 0.12007797270955166, "grad_norm": 2.2393578921718267, "kl": 0.38671875, "learning_rate": 9.989563960088675e-07, "loss": -0.0241, "reward": 0.08549312502145767, "reward_std": 0.009909003507345915, "rewards/accuracy_reward": 0.08549312502145767, "step": 154 }, { "clip_ratio": 0.0, "completion_length": 41.14583396911621, "epoch": 0.12085769980506822, "grad_norm": 2.2804733788183547, "kl": 0.3916015625, "learning_rate": 9.98871273540457e-07, "loss": 0.1656, "reward": 0.04912008345127106, "reward_std": 0.0015631647547706962, "rewards/accuracy_reward": 0.04912008345127106, "step": 155 }, { "clip_ratio": 0.0, "completion_length": 43.33333396911621, "epoch": 0.1216374269005848, "grad_norm": 2.014573223378803, "kl": 0.333984375, "learning_rate": 9.987828186294677e-07, "loss": -0.0351, "reward": 0.10474389418959618, "reward_std": 0.0107078772271052, "rewards/accuracy_reward": 0.10474389418959618, "step": 156 }, { "clip_ratio": 0.0, "completion_length": 36.916666984558105, "epoch": 0.12241715399610137, "grad_norm": 1.327646280560192, "kl": 0.3857421875, "learning_rate": 9.986910319325935e-07, "loss": 0.0309, "reward": 0.07857292518019676, "reward_std": 0.0010515710746403784, "rewards/accuracy_reward": 0.07857292518019676, "step": 157 }, { "clip_ratio": 0.0, "completion_length": 38.22916793823242, "epoch": 0.12319688109161793, "grad_norm": 1.9698099567550267, "kl": 0.416015625, "learning_rate": 9.985959141312633e-07, "loss": 0.0639, "reward": 0.05107174627482891, "reward_std": 0.0014609357458539307, "rewards/accuracy_reward": 0.05107174627482891, "step": 158 }, { "clip_ratio": 0.0, "completion_length": 37.58333396911621, "epoch": 0.1239766081871345, "grad_norm": 2.55280935272325, "kl": 0.3740234375, "learning_rate": 9.984974659316361e-07, "loss": 0.0851, "reward": 0.08227085322141647, "reward_std": 0.0012928638607263565, "rewards/accuracy_reward": 0.08227085322141647, "step": 159 }, { "clip_ratio": 0.0, "completion_length": 32.479166984558105, "epoch": 0.12475633528265107, "grad_norm": 2.4000975504020046, "kl": 0.4560546875, "learning_rate": 9.983956880645967e-07, "loss": 0.0671, "reward": 0.07537728920578957, "reward_std": 0.0013081615325063467, "rewards/accuracy_reward": 0.07537728920578957, "step": 160 }, { "clip_ratio": 0.0, "completion_length": 35.10416793823242, "epoch": 0.12553606237816764, "grad_norm": 1.7086066571358172, "kl": 0.341796875, "learning_rate": 9.982905812857488e-07, "loss": 0.0484, "reward": 0.05994926765561104, "reward_std": 0.0005158578278496861, "rewards/accuracy_reward": 0.05994926765561104, "step": 161 }, { "clip_ratio": 0.0, "completion_length": 33.0625, "epoch": 0.12631578947368421, "grad_norm": 2.7607994144458665, "kl": 0.474609375, "learning_rate": 9.981821463754101e-07, "loss": 0.1264, "reward": 0.07282404229044914, "reward_std": 0.002143740246538073, "rewards/accuracy_reward": 0.07282404229044914, "step": 162 }, { "clip_ratio": 0.0, "completion_length": 37.83333396911621, "epoch": 0.1270955165692008, "grad_norm": 2.503567849299546, "kl": 0.3544921875, "learning_rate": 9.980703841386074e-07, "loss": 0.005, "reward": 0.0794006958603859, "reward_std": 0.0015508798533119261, "rewards/accuracy_reward": 0.0794006958603859, "step": 163 }, { "clip_ratio": 0.0, "completion_length": 42.91666793823242, "epoch": 0.12787524366471734, "grad_norm": 2.90340702021995, "kl": 0.4052734375, "learning_rate": 9.979552954050685e-07, "loss": 0.0641, "reward": 0.0734306201338768, "reward_std": 0.004304809612222016, "rewards/accuracy_reward": 0.0734306201338768, "step": 164 }, { "clip_ratio": 0.0, "completion_length": 38.83333396911621, "epoch": 0.1286549707602339, "grad_norm": 0.7965501666473153, "kl": 0.4326171875, "learning_rate": 9.978368810292178e-07, "loss": 0.0183, "reward": 0.06607143208384514, "reward_std": 0.0036375211784616113, "rewards/accuracy_reward": 0.06607143208384514, "step": 165 }, { "clip_ratio": 0.0, "completion_length": 37.70833396911621, "epoch": 0.12943469785575049, "grad_norm": 1.4183492878836457, "kl": 0.55078125, "learning_rate": 9.9771514189017e-07, "loss": 0.027, "reward": 0.08144724369049072, "reward_std": 0.0009976768342312425, "rewards/accuracy_reward": 0.08144724369049072, "step": 166 }, { "clip_ratio": 0.0, "completion_length": 40.79166793823242, "epoch": 0.13021442495126706, "grad_norm": 2.6856419623277525, "kl": 0.48046875, "learning_rate": 9.97590078891722e-07, "loss": 0.0681, "reward": 0.08064528182148933, "reward_std": 0.002155478810891509, "rewards/accuracy_reward": 0.08064528182148933, "step": 167 }, { "clip_ratio": 0.0, "completion_length": 35.812500953674316, "epoch": 0.13099415204678364, "grad_norm": 2.191222031223032, "kl": 0.4013671875, "learning_rate": 9.974616929623482e-07, "loss": 0.075, "reward": 0.07386208698153496, "reward_std": 0.0022978404304012656, "rewards/accuracy_reward": 0.07386208698153496, "step": 168 }, { "clip_ratio": 0.0, "completion_length": 43.72916793823242, "epoch": 0.13177387914230018, "grad_norm": 1.9036553054266525, "kl": 0.501953125, "learning_rate": 9.973299850551916e-07, "loss": 0.102, "reward": 0.08643149957060814, "reward_std": 0.003348127822391689, "rewards/accuracy_reward": 0.08643149957060814, "step": 169 }, { "clip_ratio": 0.0, "completion_length": 37.083335876464844, "epoch": 0.13255360623781676, "grad_norm": 4.344854173590328, "kl": 0.80078125, "learning_rate": 9.971949561480585e-07, "loss": 0.0647, "reward": 0.10409487038850784, "reward_std": 0.0019580154912546277, "rewards/accuracy_reward": 0.10409487038850784, "step": 170 }, { "clip_ratio": 0.0, "completion_length": 38.64583396911621, "epoch": 0.13333333333333333, "grad_norm": 2.376886804733297, "kl": 0.4404296875, "learning_rate": 9.9705660724341e-07, "loss": 0.0679, "reward": 0.08254116401076317, "reward_std": 0.001101005997043103, "rewards/accuracy_reward": 0.08254116401076317, "step": 171 }, { "clip_ratio": 0.0, "completion_length": 37.77083492279053, "epoch": 0.1341130604288499, "grad_norm": 1.2300007972044853, "kl": 0.412109375, "learning_rate": 9.969149393683554e-07, "loss": 0.0539, "reward": 0.06586156785488129, "reward_std": 0.0014088151510804892, "rewards/accuracy_reward": 0.06586156785488129, "step": 172 }, { "clip_ratio": 0.0, "completion_length": 31.354166984558105, "epoch": 0.13489278752436648, "grad_norm": 2.8013904638274596, "kl": 0.4775390625, "learning_rate": 9.967699535746442e-07, "loss": 0.1545, "reward": 0.08266054093837738, "reward_std": 0.0015777365770190954, "rewards/accuracy_reward": 0.08266054093837738, "step": 173 }, { "clip_ratio": 0.0, "completion_length": 25.77083396911621, "epoch": 0.13567251461988303, "grad_norm": 3.8423465417405556, "kl": 0.703125, "learning_rate": 9.966216509386579e-07, "loss": 0.1449, "reward": 0.07501306384801865, "reward_std": 0.002388908062130213, "rewards/accuracy_reward": 0.07501306384801865, "step": 174 }, { "clip_ratio": 0.0, "completion_length": 32.22916793823242, "epoch": 0.1364522417153996, "grad_norm": 3.0980122015647042, "kl": 0.4794921875, "learning_rate": 9.964700325614025e-07, "loss": 0.0415, "reward": 0.07346947304904461, "reward_std": 0.0008450358000118285, "rewards/accuracy_reward": 0.07346947304904461, "step": 175 }, { "clip_ratio": 0.0, "completion_length": 25.541667938232422, "epoch": 0.13723196881091618, "grad_norm": 1.4105538008175367, "kl": 0.46484375, "learning_rate": 9.963150995685005e-07, "loss": 0.023, "reward": 0.06290727108716965, "reward_std": 0.00048778130440041423, "rewards/accuracy_reward": 0.06290727108716965, "step": 176 }, { "clip_ratio": 0.0, "completion_length": 31.375001907348633, "epoch": 0.13801169590643275, "grad_norm": 3.31868796728071, "kl": 0.4736328125, "learning_rate": 9.96156853110182e-07, "loss": 0.0028, "reward": 0.06304830685257912, "reward_std": 0.0020843480597250164, "rewards/accuracy_reward": 0.06304830685257912, "step": 177 }, { "clip_ratio": 0.0, "completion_length": 32.35416793823242, "epoch": 0.13879142300194933, "grad_norm": 2.5423838619415133, "kl": 0.69921875, "learning_rate": 9.959952943612769e-07, "loss": 0.1203, "reward": 0.07849757373332977, "reward_std": 0.005623013945296407, "rewards/accuracy_reward": 0.07849757373332977, "step": 178 }, { "clip_ratio": 0.0, "completion_length": 28.27083396911621, "epoch": 0.13957115009746587, "grad_norm": 5.737666881075233, "kl": 1.26171875, "learning_rate": 9.95830424521205e-07, "loss": 0.1103, "reward": 0.09047527983784676, "reward_std": 0.005242464481852949, "rewards/accuracy_reward": 0.09047527983784676, "step": 179 }, { "clip_ratio": 0.0, "completion_length": 26.354166984558105, "epoch": 0.14035087719298245, "grad_norm": 4.495514537490841, "kl": 0.685546875, "learning_rate": 9.956622448139683e-07, "loss": 0.0692, "reward": 0.07989903539419174, "reward_std": 0.004324757028371096, "rewards/accuracy_reward": 0.07989903539419174, "step": 180 }, { "clip_ratio": 0.0, "completion_length": 24.89583396911621, "epoch": 0.14113060428849902, "grad_norm": 3.27186911727588, "kl": 0.86328125, "learning_rate": 9.95490756488141e-07, "loss": 0.1334, "reward": 0.06422952748835087, "reward_std": 0.0031616410706192255, "rewards/accuracy_reward": 0.06422952748835087, "step": 181 }, { "clip_ratio": 0.0, "completion_length": 27.5, "epoch": 0.1419103313840156, "grad_norm": 3.4239945776742995, "kl": 0.806640625, "learning_rate": 9.953159608168613e-07, "loss": 0.154, "reward": 0.05587464012205601, "reward_std": 0.0017114627989940345, "rewards/accuracy_reward": 0.05587464012205601, "step": 182 }, { "clip_ratio": 0.0, "completion_length": 22.39583396911621, "epoch": 0.14269005847953217, "grad_norm": 2.2353911526023045, "kl": 1.109375, "learning_rate": 9.95137859097821e-07, "loss": 0.0881, "reward": 0.05541001074016094, "reward_std": 0.0014768486726097763, "rewards/accuracy_reward": 0.05541001074016094, "step": 183 }, { "clip_ratio": 0.0, "completion_length": 27.458334922790527, "epoch": 0.14346978557504872, "grad_norm": 2.562843886095173, "kl": 1.09765625, "learning_rate": 9.949564526532557e-07, "loss": 0.0957, "reward": 0.09237155690789223, "reward_std": 0.0017134369118139148, "rewards/accuracy_reward": 0.09237155690789223, "step": 184 }, { "clip_ratio": 0.0, "completion_length": 22.479167938232422, "epoch": 0.1442495126705653, "grad_norm": 4.646655813493582, "kl": 1.5703125, "learning_rate": 9.94771742829936e-07, "loss": 0.1236, "reward": 0.06373582407832146, "reward_std": 0.003131686127744615, "rewards/accuracy_reward": 0.06373582407832146, "step": 185 }, { "clip_ratio": 0.0, "completion_length": 24.70833396911621, "epoch": 0.14502923976608187, "grad_norm": 3.4445156317270915, "kl": 1.2734375, "learning_rate": 9.94583730999157e-07, "loss": 0.1488, "reward": 0.09162703156471252, "reward_std": 0.0037818995770066977, "rewards/accuracy_reward": 0.09162703156471252, "step": 186 }, { "clip_ratio": 0.0, "completion_length": 20.604166984558105, "epoch": 0.14580896686159844, "grad_norm": 2.7243563627467364, "kl": 1.40234375, "learning_rate": 9.943924185567274e-07, "loss": 0.149, "reward": 0.11167534068226814, "reward_std": 0.004113486967980862, "rewards/accuracy_reward": 0.11167534068226814, "step": 187 }, { "clip_ratio": 0.0, "completion_length": 27.52083396911621, "epoch": 0.14658869395711502, "grad_norm": 11.256552780383403, "kl": 2.4375, "learning_rate": 9.941978069229606e-07, "loss": 0.308, "reward": 0.11367808282375336, "reward_std": 0.010818830225616693, "rewards/accuracy_reward": 0.11367808282375336, "step": 188 }, { "clip_ratio": 0.0, "completion_length": 21.1875, "epoch": 0.14736842105263157, "grad_norm": 3.388916708085196, "kl": 1.17578125, "learning_rate": 9.93999897542663e-07, "loss": 0.1004, "reward": 0.08362429961562157, "reward_std": 0.0025397372664883733, "rewards/accuracy_reward": 0.08362429961562157, "step": 189 }, { "clip_ratio": 0.0, "completion_length": 25.687500953674316, "epoch": 0.14814814814814814, "grad_norm": 2.177095776211899, "kl": 0.826171875, "learning_rate": 9.937986918851235e-07, "loss": 0.1237, "reward": 0.05470520816743374, "reward_std": 0.0025200056843459606, "rewards/accuracy_reward": 0.05470520816743374, "step": 190 }, { "clip_ratio": 0.0, "completion_length": 24.729166984558105, "epoch": 0.14892787524366471, "grad_norm": 3.8120407278570934, "kl": 1.29296875, "learning_rate": 9.935941914441029e-07, "loss": 0.1284, "reward": 0.09313761256635189, "reward_std": 0.006328779039904475, "rewards/accuracy_reward": 0.09313761256635189, "step": 191 }, { "clip_ratio": 0.0, "completion_length": 24.14583396911621, "epoch": 0.1497076023391813, "grad_norm": 1.7178873909449761, "kl": 0.921875, "learning_rate": 9.933863977378226e-07, "loss": 0.0558, "reward": 0.08159026131033897, "reward_std": 0.000797520304331556, "rewards/accuracy_reward": 0.08159026131033897, "step": 192 }, { "clip_ratio": 0.0, "completion_length": 23.77083396911621, "epoch": 0.15048732943469786, "grad_norm": 0.6976605560382856, "kl": 0.94921875, "learning_rate": 9.931753123089537e-07, "loss": 0.0371, "reward": 0.06940789334475994, "reward_std": 0.0, "rewards/accuracy_reward": 0.06940789334475994, "step": 193 }, { "clip_ratio": 0.0, "completion_length": 21.83333396911621, "epoch": 0.15126705653021444, "grad_norm": 2.1454273958779524, "kl": 1.201171875, "learning_rate": 9.929609367246043e-07, "loss": 0.0425, "reward": 0.08664475753903389, "reward_std": 0.000501172267831862, "rewards/accuracy_reward": 0.08664475753903389, "step": 194 }, { "clip_ratio": 0.0, "completion_length": 20.979167938232422, "epoch": 0.15204678362573099, "grad_norm": 4.148054190150091, "kl": 1.5, "learning_rate": 9.927432725763099e-07, "loss": 0.1693, "reward": 0.09565166756510735, "reward_std": 0.003208951326087117, "rewards/accuracy_reward": 0.09565166756510735, "step": 195 }, { "clip_ratio": 0.0, "completion_length": 22.14583396911621, "epoch": 0.15282651072124756, "grad_norm": 3.3162153701918204, "kl": 1.337890625, "learning_rate": 9.925223214800195e-07, "loss": 0.095, "reward": 0.08522066101431847, "reward_std": 0.005328871659003198, "rewards/accuracy_reward": 0.08522066101431847, "step": 196 }, { "clip_ratio": 0.0, "completion_length": 23.229166984558105, "epoch": 0.15360623781676414, "grad_norm": 1.4675893353585912, "kl": 0.958984375, "learning_rate": 9.922980850760853e-07, "loss": 0.0315, "reward": 0.05325814709067345, "reward_std": 0.000361748127033934, "rewards/accuracy_reward": 0.05325814709067345, "step": 197 }, { "clip_ratio": 0.0, "completion_length": 19.4375, "epoch": 0.1543859649122807, "grad_norm": 0.1993853126111708, "kl": 1.35546875, "learning_rate": 9.920705650292492e-07, "loss": 0.0539, "reward": 0.12671999633312225, "reward_std": 0.0, "rewards/accuracy_reward": 0.12671999633312225, "step": 198 }, { "clip_ratio": 0.0, "completion_length": 19.416666984558105, "epoch": 0.15516569200779728, "grad_norm": 3.0105132998155137, "kl": 1.75, "learning_rate": 9.918397630286315e-07, "loss": 0.0923, "reward": 0.10997090488672256, "reward_std": 0.0017170071951113641, "rewards/accuracy_reward": 0.10997090488672256, "step": 199 }, { "clip_ratio": 0.0, "completion_length": 24.666666984558105, "epoch": 0.15594541910331383, "grad_norm": 3.2302112602786344, "kl": 1.390625, "learning_rate": 9.916056807877178e-07, "loss": 0.1054, "reward": 0.06637427024543285, "reward_std": 0.0010128951398655772, "rewards/accuracy_reward": 0.06637427024543285, "step": 200 }, { "clip_ratio": 0.0, "completion_length": 24.062500953674316, "epoch": 0.1567251461988304, "grad_norm": 1.6619564613991185, "kl": 0.8984375, "learning_rate": 9.91368320044346e-07, "loss": 0.08, "reward": 0.05893051438033581, "reward_std": 0.0006389423506334424, "rewards/accuracy_reward": 0.05893051438033581, "step": 201 }, { "clip_ratio": 0.0, "completion_length": 21.312500953674316, "epoch": 0.15750487329434698, "grad_norm": 0.23056134045996093, "kl": 1.7890625, "learning_rate": 9.91127682560694e-07, "loss": 0.0707, "reward": 0.12712419778108597, "reward_std": 0.0, "rewards/accuracy_reward": 0.12712419778108597, "step": 202 }, { "clip_ratio": 0.0, "completion_length": 22.937500953674316, "epoch": 0.15828460038986356, "grad_norm": 26.567204103242744, "kl": 1.984375, "learning_rate": 9.908837701232667e-07, "loss": 0.1266, "reward": 0.07912641391158104, "reward_std": 0.0013177419314160943, "rewards/accuracy_reward": 0.07912641391158104, "step": 203 }, { "clip_ratio": 0.0, "completion_length": 24.666666984558105, "epoch": 0.15906432748538013, "grad_norm": 2.0500912368942474, "kl": 1.40234375, "learning_rate": 9.906365845428819e-07, "loss": 0.1702, "reward": 0.10993445664644241, "reward_std": 0.005610934458673, "rewards/accuracy_reward": 0.10993445664644241, "step": 204 }, { "clip_ratio": 0.0, "completion_length": 23.14583396911621, "epoch": 0.15984405458089668, "grad_norm": 2.096245744417209, "kl": 1.166015625, "learning_rate": 9.903861276546576e-07, "loss": 0.0771, "reward": 0.11442387849092484, "reward_std": 0.0017005138797685504, "rewards/accuracy_reward": 0.11442387849092484, "step": 205 }, { "clip_ratio": 0.0, "completion_length": 25.89583396911621, "epoch": 0.16062378167641325, "grad_norm": 2.9395943067284422, "kl": 1.19140625, "learning_rate": 9.901324013179982e-07, "loss": 0.0954, "reward": 0.061130259186029434, "reward_std": 0.006737629068084061, "rewards/accuracy_reward": 0.061130259186029434, "step": 206 }, { "clip_ratio": 0.0, "completion_length": 24.70833396911621, "epoch": 0.16140350877192983, "grad_norm": 1.7048208623990584, "kl": 0.86328125, "learning_rate": 9.898754074165806e-07, "loss": 0.0277, "reward": 0.09889741241931915, "reward_std": 0.00021101986931171268, "rewards/accuracy_reward": 0.09889741241931915, "step": 207 }, { "clip_ratio": 0.0, "completion_length": 23.77083396911621, "epoch": 0.1621832358674464, "grad_norm": 3.3598199170740166, "kl": 1.087890625, "learning_rate": 9.896151478583404e-07, "loss": 0.099, "reward": 0.053031887859106064, "reward_std": 0.0011279088648734614, "rewards/accuracy_reward": 0.053031887859106064, "step": 208 }, { "clip_ratio": 0.0, "completion_length": 21.20833396911621, "epoch": 0.16296296296296298, "grad_norm": 1.264916036059429, "kl": 1.625, "learning_rate": 9.89351624575457e-07, "loss": 0.0858, "reward": 0.12429441511631012, "reward_std": 0.0008846581331454217, "rewards/accuracy_reward": 0.12429441511631012, "step": 209 }, { "clip_ratio": 0.0, "completion_length": 25.000000953674316, "epoch": 0.16374269005847952, "grad_norm": 1.7177509565631972, "kl": 1.013671875, "learning_rate": 9.890848395243409e-07, "loss": 0.0565, "reward": 0.11526317521929741, "reward_std": 0.0005852422909811139, "rewards/accuracy_reward": 0.11526317521929741, "step": 210 }, { "clip_ratio": 0.0, "completion_length": 22.687500953674316, "epoch": 0.1645224171539961, "grad_norm": 2.998875915683939, "kl": 1.26953125, "learning_rate": 9.888147946856173e-07, "loss": 0.0846, "reward": 0.11437266692519188, "reward_std": 0.0021981876925565302, "rewards/accuracy_reward": 0.11437266692519188, "step": 211 }, { "clip_ratio": 0.0, "completion_length": 19.187500953674316, "epoch": 0.16530214424951267, "grad_norm": 3.9208512584811426, "kl": 1.34375, "learning_rate": 9.885414920641122e-07, "loss": 0.0698, "reward": 0.06773879006505013, "reward_std": 0.0006662883388344198, "rewards/accuracy_reward": 0.06773879006505013, "step": 212 }, { "clip_ratio": 0.0, "completion_length": 26.9375, "epoch": 0.16608187134502925, "grad_norm": 2.9525260615911257, "kl": 0.90625, "learning_rate": 9.88264933688838e-07, "loss": 0.0569, "reward": 0.09685194306075573, "reward_std": 0.0014135038363747299, "rewards/accuracy_reward": 0.09685194306075573, "step": 213 }, { "clip_ratio": 0.0, "completion_length": 22.291667938232422, "epoch": 0.16686159844054582, "grad_norm": 5.227246556111158, "kl": 0.94140625, "learning_rate": 9.87985121612978e-07, "loss": 0.1015, "reward": 0.05899123102426529, "reward_std": 0.001518539502285421, "rewards/accuracy_reward": 0.05899123102426529, "step": 214 }, { "clip_ratio": 0.0, "completion_length": 25.08333396911621, "epoch": 0.16764132553606237, "grad_norm": 2.326342301240418, "kl": 1.08984375, "learning_rate": 9.877020579138707e-07, "loss": 0.0782, "reward": 0.08564617857336998, "reward_std": 0.0011196026753168553, "rewards/accuracy_reward": 0.08564617857336998, "step": 215 }, { "clip_ratio": 0.0, "completion_length": 25.687500953674316, "epoch": 0.16842105263157894, "grad_norm": 1.2358534767102056, "kl": 1.08984375, "learning_rate": 9.874157446929951e-07, "loss": 0.0803, "reward": 0.08892780169844627, "reward_std": 0.0007336726412177086, "rewards/accuracy_reward": 0.08892780169844627, "step": 216 }, { "clip_ratio": 0.0, "completion_length": 22.52083396911621, "epoch": 0.16920077972709552, "grad_norm": 3.436935261638758, "kl": 1.33203125, "learning_rate": 9.871261840759545e-07, "loss": 0.0834, "reward": 0.08073633164167404, "reward_std": 0.00847914395853877, "rewards/accuracy_reward": 0.08073633164167404, "step": 217 }, { "clip_ratio": 0.0, "completion_length": 22.64583396911621, "epoch": 0.1699805068226121, "grad_norm": 2.1566623189134253, "kl": 1.71484375, "learning_rate": 9.868333782124617e-07, "loss": 0.1519, "reward": 0.11506958678364754, "reward_std": 0.0026018971111625433, "rewards/accuracy_reward": 0.11506958678364754, "step": 218 }, { "clip_ratio": 0.0, "completion_length": 20.95833396911621, "epoch": 0.17076023391812867, "grad_norm": 3.5488910051635676, "kl": 1.4609375, "learning_rate": 9.865373292763214e-07, "loss": 0.0457, "reward": 0.07090311497449875, "reward_std": 0.0042670833063311875, "rewards/accuracy_reward": 0.07090311497449875, "step": 219 }, { "clip_ratio": 0.0, "completion_length": 22.52083396911621, "epoch": 0.17153996101364521, "grad_norm": 2.8053650736648077, "kl": 1.3046875, "learning_rate": 9.862380394654162e-07, "loss": 0.0427, "reward": 0.13785403966903687, "reward_std": 0.007753290235996246, "rewards/accuracy_reward": 0.13785403966903687, "step": 220 }, { "clip_ratio": 0.0, "completion_length": 23.604167938232422, "epoch": 0.1723196881091618, "grad_norm": 2.2426806514275213, "kl": 1.4375, "learning_rate": 9.85935511001688e-07, "loss": 0.1283, "reward": 0.11332451924681664, "reward_std": 0.0006854766397736967, "rewards/accuracy_reward": 0.11332451924681664, "step": 221 }, { "clip_ratio": 0.0, "completion_length": 23.875000953674316, "epoch": 0.17309941520467836, "grad_norm": 2.7303830779076885, "kl": 1.59375, "learning_rate": 9.856297461311233e-07, "loss": 0.1, "reward": 0.06828703731298447, "reward_std": 0.004009376803878695, "rewards/accuracy_reward": 0.06828703731298447, "step": 222 }, { "clip_ratio": 0.0, "completion_length": 21.33333396911621, "epoch": 0.17387914230019494, "grad_norm": 1.7534862814182692, "kl": 1.8125, "learning_rate": 9.853207471237357e-07, "loss": 0.1116, "reward": 0.08874018490314484, "reward_std": 0.0019992401357740164, "rewards/accuracy_reward": 0.08874018490314484, "step": 223 }, { "clip_ratio": 0.0, "completion_length": 22.39583396911621, "epoch": 0.1746588693957115, "grad_norm": 2.920053943391909, "kl": 1.73828125, "learning_rate": 9.85008516273549e-07, "loss": 0.1007, "reward": 0.09884331189095974, "reward_std": 0.005048536462709308, "rewards/accuracy_reward": 0.09884331189095974, "step": 224 }, { "clip_ratio": 0.0, "completion_length": 23.89583396911621, "epoch": 0.17543859649122806, "grad_norm": 2.899972090799929, "kl": 1.328125, "learning_rate": 9.846930558985806e-07, "loss": 0.0758, "reward": 0.06051616370677948, "reward_std": 0.005370165046770126, "rewards/accuracy_reward": 0.06051616370677948, "step": 225 }, { "clip_ratio": 0.0, "completion_length": 23.104166984558105, "epoch": 0.17621832358674464, "grad_norm": 3.466343735070648, "kl": 1.48828125, "learning_rate": 9.84374368340824e-07, "loss": 0.0647, "reward": 0.10638459771871567, "reward_std": 0.00696826761122793, "rewards/accuracy_reward": 0.10638459771871567, "step": 226 }, { "clip_ratio": 0.0, "completion_length": 22.562500953674316, "epoch": 0.1769980506822612, "grad_norm": 0.5345048985939578, "kl": 1.40234375, "learning_rate": 9.840524559662313e-07, "loss": 0.0557, "reward": 0.0833333395421505, "reward_std": 0.0, "rewards/accuracy_reward": 0.0833333395421505, "step": 227 }, { "clip_ratio": 0.0, "completion_length": 20.291666984558105, "epoch": 0.17777777777777778, "grad_norm": 2.5430356173050876, "kl": 1.75, "learning_rate": 9.83727321164696e-07, "loss": 0.0834, "reward": 0.07123610191047192, "reward_std": 0.004868295218329877, "rewards/accuracy_reward": 0.07123610191047192, "step": 228 }, { "clip_ratio": 0.0, "completion_length": 24.5625, "epoch": 0.17855750487329436, "grad_norm": 3.2574801690348227, "kl": 1.765625, "learning_rate": 9.83398966350035e-07, "loss": 0.125, "reward": 0.10326099023222923, "reward_std": 0.004647268156986684, "rewards/accuracy_reward": 0.10326099023222923, "step": 229 }, { "clip_ratio": 0.0, "completion_length": 21.187500953674316, "epoch": 0.1793372319688109, "grad_norm": 1.266097751271849, "kl": 1.390625, "learning_rate": 9.830673939599705e-07, "loss": 0.0599, "reward": 0.08679194748401642, "reward_std": 0.0003180142375640571, "rewards/accuracy_reward": 0.08679194748401642, "step": 230 }, { "clip_ratio": 0.0, "completion_length": 23.08333396911621, "epoch": 0.18011695906432748, "grad_norm": 2.8167759580489222, "kl": 1.60546875, "learning_rate": 9.827326064561125e-07, "loss": 0.0762, "reward": 0.0730651281774044, "reward_std": 0.007135274587199092, "rewards/accuracy_reward": 0.0730651281774044, "step": 231 }, { "clip_ratio": 0.0, "completion_length": 14.500000953674316, "epoch": 0.18089668615984406, "grad_norm": 0.16113981868029764, "kl": 2.31640625, "learning_rate": 9.823946063239402e-07, "loss": 0.0915, "reward": 0.129901971668005, "reward_std": 0.0, "rewards/accuracy_reward": 0.129901971668005, "step": 232 }, { "clip_ratio": 0.0, "completion_length": 21.541667938232422, "epoch": 0.18167641325536063, "grad_norm": 1.332369107753439, "kl": 1.61328125, "learning_rate": 9.820533960727832e-07, "loss": 0.0441, "reward": 0.1135011799633503, "reward_std": 0.00021101981110405177, "rewards/accuracy_reward": 0.1135011799633503, "step": 233 }, { "clip_ratio": 0.0, "completion_length": 23.64583396911621, "epoch": 0.1824561403508772, "grad_norm": 1.5317502560363618, "kl": 1.19921875, "learning_rate": 9.817089782358032e-07, "loss": 0.0519, "reward": 0.11487788707017899, "reward_std": 0.004591436590999365, "rewards/accuracy_reward": 0.11487788707017899, "step": 234 }, { "clip_ratio": 0.0, "completion_length": 17.52083396911621, "epoch": 0.18323586744639375, "grad_norm": 0.2671234363243526, "kl": 2.04296875, "learning_rate": 9.813613553699755e-07, "loss": 0.0808, "reward": 0.09689543023705482, "reward_std": 0.0, "rewards/accuracy_reward": 0.09689543023705482, "step": 235 }, { "clip_ratio": 0.0, "completion_length": 22.52083396911621, "epoch": 0.18401559454191033, "grad_norm": 2.661407358134365, "kl": 2.02734375, "learning_rate": 9.810105300560692e-07, "loss": 0.0968, "reward": 0.08690152689814568, "reward_std": 0.0016228330787271261, "rewards/accuracy_reward": 0.08690152689814568, "step": 236 }, { "clip_ratio": 0.0, "completion_length": 23.979167938232422, "epoch": 0.1847953216374269, "grad_norm": 1.6977362227800048, "kl": 1.8203125, "learning_rate": 9.80656504898629e-07, "loss": 0.0952, "reward": 0.0872369036078453, "reward_std": 0.0006903500179760158, "rewards/accuracy_reward": 0.0872369036078453, "step": 237 }, { "clip_ratio": 0.0, "completion_length": 20.437500953674316, "epoch": 0.18557504873294348, "grad_norm": 3.6993083959009114, "kl": 1.66015625, "learning_rate": 9.802992825259553e-07, "loss": 0.0843, "reward": 0.07765689864754677, "reward_std": 0.011905404506251216, "rewards/accuracy_reward": 0.07765689864754677, "step": 238 }, { "clip_ratio": 0.0, "completion_length": 25.750000953674316, "epoch": 0.18635477582846005, "grad_norm": 0.237259194490552, "kl": 1.3359375, "learning_rate": 9.799388655900844e-07, "loss": 0.0533, "reward": 0.08496732264757156, "reward_std": 0.0, "rewards/accuracy_reward": 0.08496732264757156, "step": 239 }, { "clip_ratio": 0.0, "completion_length": 24.166667938232422, "epoch": 0.1871345029239766, "grad_norm": 0.325618890659599, "kl": 1.123046875, "learning_rate": 9.7957525676677e-07, "loss": 0.0445, "reward": 0.11388889327645302, "reward_std": 0.0, "rewards/accuracy_reward": 0.11388889327645302, "step": 240 }, { "clip_ratio": 0.0, "completion_length": 22.312500953674316, "epoch": 0.18791423001949317, "grad_norm": 3.0014091913037557, "kl": 1.37890625, "learning_rate": 9.792084587554613e-07, "loss": 0.0678, "reward": 0.06449008546769619, "reward_std": 0.0007475692837033421, "rewards/accuracy_reward": 0.06449008546769619, "step": 241 }, { "clip_ratio": 0.0, "completion_length": 18.937500953674316, "epoch": 0.18869395711500975, "grad_norm": 4.347230766020519, "kl": 2.1640625, "learning_rate": 9.788384742792855e-07, "loss": 0.1175, "reward": 0.08908803015947342, "reward_std": 0.012355273123830557, "rewards/accuracy_reward": 0.08908803015947342, "step": 242 }, { "clip_ratio": 0.0, "completion_length": 21.437500953674316, "epoch": 0.18947368421052632, "grad_norm": 0.25889614498853536, "kl": 1.50390625, "learning_rate": 9.784653060850256e-07, "loss": 0.0576, "reward": 0.11633987352252007, "reward_std": 0.0, "rewards/accuracy_reward": 0.11633987352252007, "step": 243 }, { "clip_ratio": 0.0, "completion_length": 22.33333396911621, "epoch": 0.1902534113060429, "grad_norm": 1.5983174576317682, "kl": 1.7890625, "learning_rate": 9.780889569431006e-07, "loss": 0.112, "reward": 0.09527980536222458, "reward_std": 0.005020395619794726, "rewards/accuracy_reward": 0.09527980536222458, "step": 244 }, { "clip_ratio": 0.0, "completion_length": 20.666667938232422, "epoch": 0.19103313840155944, "grad_norm": 2.6408241280763, "kl": 1.515625, "learning_rate": 9.77709429647545e-07, "loss": 0.057, "reward": 0.08550266921520233, "reward_std": 0.0074745421297848225, "rewards/accuracy_reward": 0.08550266921520233, "step": 245 }, { "clip_ratio": 0.0, "completion_length": 21.875000953674316, "epoch": 0.19181286549707602, "grad_norm": 1.9120022670006256, "kl": 1.58203125, "learning_rate": 9.77326727015989e-07, "loss": 0.0382, "reward": 0.12758715078234673, "reward_std": 0.006278668995946646, "rewards/accuracy_reward": 0.12758715078234673, "step": 246 }, { "clip_ratio": 0.0, "completion_length": 16.64583396911621, "epoch": 0.1925925925925926, "grad_norm": 3.9104437289728287, "kl": 1.9296875, "learning_rate": 9.76940851889635e-07, "loss": 0.1131, "reward": 0.09734449535608292, "reward_std": 0.01164670754224062, "rewards/accuracy_reward": 0.09734449535608292, "step": 247 }, { "clip_ratio": 0.0, "completion_length": 23.02083396911621, "epoch": 0.19337231968810917, "grad_norm": 3.846049762456662, "kl": 1.65625, "learning_rate": 9.765518071332396e-07, "loss": 0.0755, "reward": 0.0830635204911232, "reward_std": 0.015773563645780087, "rewards/accuracy_reward": 0.0830635204911232, "step": 248 }, { "clip_ratio": 0.0, "completion_length": 22.9375, "epoch": 0.19415204678362574, "grad_norm": 2.3131674568250538, "kl": 1.537109375, "learning_rate": 9.7615959563509e-07, "loss": 0.0206, "reward": 0.12442300841212273, "reward_std": 0.007980600406881422, "rewards/accuracy_reward": 0.12442300841212273, "step": 249 }, { "clip_ratio": 0.0, "completion_length": 25.479167938232422, "epoch": 0.1949317738791423, "grad_norm": 4.195497169434552, "kl": 1.62109375, "learning_rate": 9.757642203069843e-07, "loss": 0.035, "reward": 0.12204422801733017, "reward_std": 0.0224958173930645, "rewards/accuracy_reward": 0.12204422801733017, "step": 250 }, { "clip_ratio": 0.0, "completion_length": 19.604166984558105, "epoch": 0.19571150097465886, "grad_norm": 5.2103577929033555, "kl": 2.1015625, "learning_rate": 9.753656840842082e-07, "loss": 0.0305, "reward": 0.1147073544561863, "reward_std": 0.025445946492254734, "rewards/accuracy_reward": 0.1147073544561863, "step": 251 }, { "clip_ratio": 0.0, "completion_length": 25.812500953674316, "epoch": 0.19649122807017544, "grad_norm": 2.3995622137473163, "kl": 1.53515625, "learning_rate": 9.749639899255143e-07, "loss": 0.0673, "reward": 0.12678389996290207, "reward_std": 0.003454159013926983, "rewards/accuracy_reward": 0.12678389996290207, "step": 252 }, { "clip_ratio": 0.0, "completion_length": 22.104166984558105, "epoch": 0.197270955165692, "grad_norm": 1.7428432615273857, "kl": 1.77734375, "learning_rate": 9.745591408131e-07, "loss": 0.0729, "reward": 0.12410277128219604, "reward_std": 0.0005306522361934185, "rewards/accuracy_reward": 0.12410277128219604, "step": 253 }, { "clip_ratio": 0.0, "completion_length": 24.104167938232422, "epoch": 0.1980506822612086, "grad_norm": 1.8545050575670603, "kl": 1.3359375, "learning_rate": 9.741511397525852e-07, "loss": 0.0902, "reward": 0.13580767065286636, "reward_std": 0.0009267845889553428, "rewards/accuracy_reward": 0.13580767065286636, "step": 254 }, { "clip_ratio": 0.0, "completion_length": 23.604166984558105, "epoch": 0.19883040935672514, "grad_norm": 3.245255032482792, "kl": 1.8203125, "learning_rate": 9.737399897729902e-07, "loss": 0.1621, "reward": 0.13663917034864426, "reward_std": 0.0038499051588587463, "rewards/accuracy_reward": 0.13663917034864426, "step": 255 }, { "clip_ratio": 0.0, "completion_length": 21.937500953674316, "epoch": 0.1996101364522417, "grad_norm": 3.381318839529289, "kl": 1.890625, "learning_rate": 9.733256939267127e-07, "loss": 0.2123, "reward": 0.13044091314077377, "reward_std": 0.005797044548671693, "rewards/accuracy_reward": 0.13044091314077377, "step": 256 }, { "clip_ratio": 0.0, "completion_length": 20.666666984558105, "epoch": 0.20038986354775828, "grad_norm": 3.473785023099609, "kl": 1.91015625, "learning_rate": 9.72908255289505e-07, "loss": 0.1755, "reward": 0.1318226270377636, "reward_std": 0.0018371137557551265, "rewards/accuracy_reward": 0.1318226270377636, "step": 257 }, { "clip_ratio": 0.0, "completion_length": 20.83333396911621, "epoch": 0.20116959064327486, "grad_norm": 2.813642793968136, "kl": 1.8359375, "learning_rate": 9.724876769604534e-07, "loss": 0.0434, "reward": 0.13815467059612274, "reward_std": 0.0005697530577890575, "rewards/accuracy_reward": 0.13815467059612274, "step": 258 }, { "clip_ratio": 0.0, "completion_length": 22.291666984558105, "epoch": 0.20194931773879143, "grad_norm": 3.9107649012845216, "kl": 2.0390625, "learning_rate": 9.72063962061951e-07, "loss": 0.1536, "reward": 0.137812077999115, "reward_std": 0.002921649720519781, "rewards/accuracy_reward": 0.137812077999115, "step": 259 }, { "clip_ratio": 0.0, "completion_length": 25.02083396911621, "epoch": 0.20272904483430798, "grad_norm": 1.4346009750815534, "kl": 1.58203125, "learning_rate": 9.716371137396793e-07, "loss": 0.0892, "reward": 0.13681723177433014, "reward_std": 0.0009050843073055148, "rewards/accuracy_reward": 0.13681723177433014, "step": 260 }, { "clip_ratio": 0.0, "completion_length": 22.291667938232422, "epoch": 0.20350877192982456, "grad_norm": 3.4866115323473976, "kl": 1.74609375, "learning_rate": 9.712071351625806e-07, "loss": 0.0738, "reward": 0.13259827345609665, "reward_std": 0.003800537611823529, "rewards/accuracy_reward": 0.13259827345609665, "step": 261 }, { "clip_ratio": 0.0, "completion_length": 23.89583396911621, "epoch": 0.20428849902534113, "grad_norm": 1.7333177534526967, "kl": 1.57421875, "learning_rate": 9.707740295228377e-07, "loss": 0.0596, "reward": 0.14202070236206055, "reward_std": 0.00047169136814773083, "rewards/accuracy_reward": 0.14202070236206055, "step": 262 }, { "clip_ratio": 0.0, "completion_length": 22.08333396911621, "epoch": 0.2050682261208577, "grad_norm": 1.8505394432271243, "kl": 1.74609375, "learning_rate": 9.70337800035848e-07, "loss": 0.0727, "reward": 0.12503890693187714, "reward_std": 0.0011856432538479567, "rewards/accuracy_reward": 0.12503890693187714, "step": 263 }, { "clip_ratio": 0.0, "completion_length": 22.14583396911621, "epoch": 0.20584795321637428, "grad_norm": 1.970418354488007, "kl": 1.875, "learning_rate": 9.698984499402009e-07, "loss": 0.0725, "reward": 0.12785088270902634, "reward_std": 0.0012852847576141357, "rewards/accuracy_reward": 0.12785088270902634, "step": 264 }, { "clip_ratio": 0.0, "completion_length": 21.75, "epoch": 0.20662768031189083, "grad_norm": 3.8673657513171866, "kl": 1.8515625, "learning_rate": 9.694559824976534e-07, "loss": 0.1285, "reward": 0.13662027940154076, "reward_std": 0.0060649634106084704, "rewards/accuracy_reward": 0.13662027940154076, "step": 265 }, { "clip_ratio": 0.0, "completion_length": 23.45833396911621, "epoch": 0.2074074074074074, "grad_norm": 1.7690362347869697, "kl": 1.50390625, "learning_rate": 9.690104009931055e-07, "loss": 0.0542, "reward": 0.13606832176446915, "reward_std": 0.00047169136814773083, "rewards/accuracy_reward": 0.13606832176446915, "step": 266 }, { "clip_ratio": 0.0, "completion_length": 21.187500953674316, "epoch": 0.20818713450292398, "grad_norm": 1.82646931051954, "kl": 1.71484375, "learning_rate": 9.685617087345766e-07, "loss": 0.1062, "reward": 0.13011812418699265, "reward_std": 0.0031333768274635077, "rewards/accuracy_reward": 0.13011812418699265, "step": 267 }, { "clip_ratio": 0.0, "completion_length": 18.416666984558105, "epoch": 0.20896686159844055, "grad_norm": 1.480990254801878, "kl": 1.99609375, "learning_rate": 9.681099090531803e-07, "loss": 0.1256, "reward": 0.15551593899726868, "reward_std": 0.0017183038871735334, "rewards/accuracy_reward": 0.15551593899726868, "step": 268 }, { "clip_ratio": 0.0, "completion_length": 61.979166984558105, "epoch": 0.20974658869395713, "grad_norm": 4.45416174052944, "kl": 2.03515625, "learning_rate": 9.676550053030993e-07, "loss": 0.6251, "reward": 0.14152297377586365, "reward_std": 0.010142141720280051, "rewards/accuracy_reward": 0.14152297377586365, "step": 269 }, { "clip_ratio": 0.0, "completion_length": 17.916667938232422, "epoch": 0.21052631578947367, "grad_norm": 3.672160807656365, "kl": 2.2890625, "learning_rate": 9.671970008615625e-07, "loss": 0.1664, "reward": 0.15890305489301682, "reward_std": 0.008111248724162579, "rewards/accuracy_reward": 0.15890305489301682, "step": 270 }, { "clip_ratio": 0.0, "completion_length": 63.4375, "epoch": 0.21130604288499025, "grad_norm": 2.6396408305994674, "kl": 1.921875, "learning_rate": 9.667358991288167e-07, "loss": 0.542, "reward": 0.12156940624117851, "reward_std": 0.009339521639049053, "rewards/accuracy_reward": 0.12156940624117851, "step": 271 }, { "clip_ratio": 0.0, "completion_length": 19.541667938232422, "epoch": 0.21208576998050682, "grad_norm": 3.8827213064852795, "kl": 1.8125, "learning_rate": 9.662717035281044e-07, "loss": 0.1395, "reward": 0.16217350959777832, "reward_std": 0.00623874575830996, "rewards/accuracy_reward": 0.16217350959777832, "step": 272 }, { "clip_ratio": 0.0, "completion_length": 20.89583396911621, "epoch": 0.2128654970760234, "grad_norm": 4.872856767039602, "kl": 1.85546875, "learning_rate": 9.658044175056367e-07, "loss": 0.0915, "reward": 0.15208658576011658, "reward_std": 0.005818813806399703, "rewards/accuracy_reward": 0.15208658576011658, "step": 273 }, { "clip_ratio": 0.0, "completion_length": 16.416666984558105, "epoch": 0.21364522417153997, "grad_norm": 6.858118696733246, "kl": 2.328125, "learning_rate": 9.653340445305682e-07, "loss": 0.3973, "reward": 0.13756439834833145, "reward_std": 0.011764792259782553, "rewards/accuracy_reward": 0.13756439834833145, "step": 274 }, { "clip_ratio": 0.0, "completion_length": 17.979166984558105, "epoch": 0.21442495126705652, "grad_norm": 6.702304859912742, "kl": 2.04296875, "learning_rate": 9.648605880949713e-07, "loss": 0.16, "reward": 0.1567860245704651, "reward_std": 0.010588282719254494, "rewards/accuracy_reward": 0.1567860245704651, "step": 275 }, { "clip_ratio": 0.0, "completion_length": 17.166667461395264, "epoch": 0.2152046783625731, "grad_norm": 4.878099611797554, "kl": 2.0078125, "learning_rate": 9.643840517138098e-07, "loss": 0.1372, "reward": 0.12988395988941193, "reward_std": 0.00418971199542284, "rewards/accuracy_reward": 0.12988395988941193, "step": 276 }, { "clip_ratio": 0.0, "completion_length": 18.375000953674316, "epoch": 0.21598440545808967, "grad_norm": 3.755944790669856, "kl": 2.3046875, "learning_rate": 9.639044389249134e-07, "loss": 0.1197, "reward": 0.16065938770771027, "reward_std": 0.007759660831652582, "rewards/accuracy_reward": 0.16065938770771027, "step": 277 }, { "clip_ratio": 0.0, "completion_length": 13.541666984558105, "epoch": 0.21676413255360624, "grad_norm": 5.4220528733524604, "kl": 2.921875, "learning_rate": 9.634217532889515e-07, "loss": 0.2103, "reward": 0.1402498111128807, "reward_std": 0.008417865494266152, "rewards/accuracy_reward": 0.1402498111128807, "step": 278 }, { "clip_ratio": 0.0, "completion_length": 12.291666984558105, "epoch": 0.21754385964912282, "grad_norm": 7.0235756541437855, "kl": 3.2109375, "learning_rate": 9.62935998389406e-07, "loss": 0.3883, "reward": 0.1477484181523323, "reward_std": 0.011410552775487304, "rewards/accuracy_reward": 0.1477484181523323, "step": 279 }, { "clip_ratio": 0.0, "completion_length": 55.89583492279053, "epoch": 0.21832358674463936, "grad_norm": 10.110573353293253, "kl": 2.90625, "learning_rate": 9.624471778325455e-07, "loss": 0.5111, "reward": 0.16363994777202606, "reward_std": 0.011551804840564728, "rewards/accuracy_reward": 0.16363994777202606, "step": 280 }, { "clip_ratio": 0.0, "completion_length": 52.375001430511475, "epoch": 0.21910331384015594, "grad_norm": 22.235508699230277, "kl": 3.234375, "learning_rate": 9.619552952473976e-07, "loss": -0.2041, "reward": 0.14994164556264877, "reward_std": 0.002838627900928259, "rewards/accuracy_reward": 0.14994164556264877, "step": 281 }, { "clip_ratio": 0.0, "completion_length": 11.750000476837158, "epoch": 0.2198830409356725, "grad_norm": 8.846563763087351, "kl": 3.1171875, "learning_rate": 9.614603542857235e-07, "loss": -0.1139, "reward": 0.16706421971321106, "reward_std": 0.01608453644439578, "rewards/accuracy_reward": 0.16706421971321106, "step": 282 }, { "clip_ratio": 0.0, "completion_length": 16.020833492279053, "epoch": 0.2206627680311891, "grad_norm": 5.839429613378161, "kl": 2.7421875, "learning_rate": 9.609623586219892e-07, "loss": 0.0327, "reward": 0.15950050950050354, "reward_std": 0.014836331363767385, "rewards/accuracy_reward": 0.15950050950050354, "step": 283 }, { "clip_ratio": 0.0, "completion_length": 14.791666984558105, "epoch": 0.22144249512670566, "grad_norm": 5.743529221608405, "kl": 3.0078125, "learning_rate": 9.604613119533389e-07, "loss": 0.4062, "reward": 0.1408693641424179, "reward_std": 0.013236512430012226, "rewards/accuracy_reward": 0.1408693641424179, "step": 284 }, { "clip_ratio": 0.0, "completion_length": 15.3125, "epoch": 0.2222222222222222, "grad_norm": 3.2998580002726587, "kl": 2.8828125, "learning_rate": 9.599572179995684e-07, "loss": 0.299, "reward": 0.14530812948942184, "reward_std": 0.004396125441417098, "rewards/accuracy_reward": 0.14530812948942184, "step": 285 }, { "clip_ratio": 0.0, "completion_length": 12.770833969116211, "epoch": 0.22300194931773878, "grad_norm": 8.29951318737398, "kl": 3.046875, "learning_rate": 9.59450080503096e-07, "loss": 0.3234, "reward": 0.14096882194280624, "reward_std": 0.006384085863828659, "rewards/accuracy_reward": 0.14096882194280624, "step": 286 }, { "clip_ratio": 0.0, "completion_length": 12.229166984558105, "epoch": 0.22378167641325536, "grad_norm": 6.7364372054781825, "kl": 2.9921875, "learning_rate": 9.589399032289355e-07, "loss": 0.2129, "reward": 0.15641389042139053, "reward_std": 0.008956428384408355, "rewards/accuracy_reward": 0.15641389042139053, "step": 287 }, { "clip_ratio": 0.0, "completion_length": 16.125000476837158, "epoch": 0.22456140350877193, "grad_norm": 7.416919295448386, "kl": 2.31640625, "learning_rate": 9.584266899646684e-07, "loss": 0.3471, "reward": 0.15673290938138962, "reward_std": 0.010735958348959684, "rewards/accuracy_reward": 0.15673290938138962, "step": 288 }, { "clip_ratio": 0.0, "completion_length": 12.375, "epoch": 0.2253411306042885, "grad_norm": 6.4774087181717075, "kl": 2.890625, "learning_rate": 9.579104445204155e-07, "loss": 0.2685, "reward": 0.1649218201637268, "reward_std": 0.0024056266993284225, "rewards/accuracy_reward": 0.1649218201637268, "step": 289 }, { "clip_ratio": 0.0, "completion_length": 12.479166984558105, "epoch": 0.22612085769980506, "grad_norm": 6.171742085607571, "kl": 3.0625, "learning_rate": 9.573911707288082e-07, "loss": 0.2615, "reward": 0.15222876518964767, "reward_std": 0.00795425078831613, "rewards/accuracy_reward": 0.15222876518964767, "step": 290 }, { "clip_ratio": 0.0, "completion_length": 11.375, "epoch": 0.22690058479532163, "grad_norm": 2.8424365519530763, "kl": 2.9921875, "learning_rate": 9.568688724449613e-07, "loss": 0.1239, "reward": 0.13931213319301605, "reward_std": 0.004959697602316737, "rewards/accuracy_reward": 0.13931213319301605, "step": 291 }, { "clip_ratio": 0.0, "completion_length": 11.729166984558105, "epoch": 0.2276803118908382, "grad_norm": 4.783219578380445, "kl": 3.2265625, "learning_rate": 9.563435535464427e-07, "loss": 0.3647, "reward": 0.13829627633094788, "reward_std": 0.011282553663477302, "rewards/accuracy_reward": 0.13829627633094788, "step": 292 }, { "clip_ratio": 0.0, "completion_length": 15.0625, "epoch": 0.22846003898635478, "grad_norm": 1.7699679624581726, "kl": 2.515625, "learning_rate": 9.558152179332463e-07, "loss": 0.0712, "reward": 0.1633118912577629, "reward_std": 0.008225119207054377, "rewards/accuracy_reward": 0.1633118912577629, "step": 293 }, { "clip_ratio": 0.0, "completion_length": 13.458333969116211, "epoch": 0.22923976608187135, "grad_norm": 5.607499819766133, "kl": 2.640625, "learning_rate": 9.552838695277617e-07, "loss": 0.2946, "reward": 0.1461936980485916, "reward_std": 0.00803038408048451, "rewards/accuracy_reward": 0.1461936980485916, "step": 294 }, { "clip_ratio": 0.0, "completion_length": 12.562500476837158, "epoch": 0.2300194931773879, "grad_norm": 1.5714562706963744, "kl": 2.984375, "learning_rate": 9.547495122747457e-07, "loss": 0.0915, "reward": 0.1667862981557846, "reward_std": 0.005498574580997229, "rewards/accuracy_reward": 0.1667862981557846, "step": 295 }, { "clip_ratio": 0.0, "completion_length": 13.1875, "epoch": 0.23079922027290448, "grad_norm": 3.620635251174383, "kl": 3.15625, "learning_rate": 9.542121501412933e-07, "loss": 0.261, "reward": 0.15238095819950104, "reward_std": 0.005222418578341603, "rewards/accuracy_reward": 0.15238095819950104, "step": 296 }, { "clip_ratio": 0.0, "completion_length": 14.541667461395264, "epoch": 0.23157894736842105, "grad_norm": 4.4303120652811625, "kl": 2.6171875, "learning_rate": 9.536717871168076e-07, "loss": 0.1994, "reward": 0.17520426213741302, "reward_std": 0.005448035895824432, "rewards/accuracy_reward": 0.17520426213741302, "step": 297 }, { "clip_ratio": 0.0, "completion_length": 16.08333396911621, "epoch": 0.23235867446393763, "grad_norm": 4.022228405944924, "kl": 2.6484375, "learning_rate": 9.5312842721297e-07, "loss": 0.133, "reward": 0.15377682447433472, "reward_std": 0.011845491826534271, "rewards/accuracy_reward": 0.15377682447433472, "step": 298 }, { "clip_ratio": 0.0, "completion_length": 13.333333969116211, "epoch": 0.2331384015594542, "grad_norm": 11.681673402523783, "kl": 3.921875, "learning_rate": 9.525820744637119e-07, "loss": 0.4493, "reward": 0.1410665586590767, "reward_std": 0.014458210207521915, "rewards/accuracy_reward": 0.1410665586590767, "step": 299 }, { "clip_ratio": 0.0, "completion_length": 14.708333969116211, "epoch": 0.23391812865497075, "grad_norm": 1.4599098168537827, "kl": 2.6015625, "learning_rate": 9.520327329251831e-07, "loss": 0.0997, "reward": 0.17216148227453232, "reward_std": 0.0, "rewards/accuracy_reward": 0.17216148227453232, "step": 300 }, { "clip_ratio": 0.0, "completion_length": 12.645833969116211, "epoch": 0.23469785575048732, "grad_norm": 4.73839760766751, "kl": 2.8359375, "learning_rate": 9.514804066757219e-07, "loss": 0.2478, "reward": 0.14820358902215958, "reward_std": 0.007825780659914017, "rewards/accuracy_reward": 0.14820358902215958, "step": 301 }, { "clip_ratio": 0.0, "completion_length": 12.437500476837158, "epoch": 0.2354775828460039, "grad_norm": 4.983771234470083, "kl": 2.6875, "learning_rate": 9.509250998158262e-07, "loss": 0.1925, "reward": 0.16442348062992096, "reward_std": 0.005435790400952101, "rewards/accuracy_reward": 0.16442348062992096, "step": 302 }, { "clip_ratio": 0.0, "completion_length": 12.125000476837158, "epoch": 0.23625730994152047, "grad_norm": 6.2020739563672045, "kl": 3.0703125, "learning_rate": 9.503668164681219e-07, "loss": 0.271, "reward": 0.14058363065123558, "reward_std": 0.011176002211868763, "rewards/accuracy_reward": 0.14058363065123558, "step": 303 }, { "clip_ratio": 0.0, "completion_length": 11.770833730697632, "epoch": 0.23703703703703705, "grad_norm": 0.4211076751532243, "kl": 3.00390625, "learning_rate": 9.498055607773316e-07, "loss": 0.1176, "reward": 0.17432598769664764, "reward_std": 0.0, "rewards/accuracy_reward": 0.17432598769664764, "step": 304 }, { "clip_ratio": 0.0, "completion_length": 11.020833492279053, "epoch": 0.2378167641325536, "grad_norm": 3.1760270982848424, "kl": 3.140625, "learning_rate": 9.492413369102458e-07, "loss": 0.1373, "reward": 0.1416749358177185, "reward_std": 0.001002344535663724, "rewards/accuracy_reward": 0.1416749358177185, "step": 305 }, { "clip_ratio": 0.0, "completion_length": 12.062500476837158, "epoch": 0.23859649122807017, "grad_norm": 5.763208067499548, "kl": 3.125, "learning_rate": 9.486741490556902e-07, "loss": 0.1803, "reward": 0.14505772292613983, "reward_std": 0.0010257635731250048, "rewards/accuracy_reward": 0.14505772292613983, "step": 306 }, { "clip_ratio": 0.0, "completion_length": 8.25, "epoch": 0.23937621832358674, "grad_norm": 0.2239305538920244, "kl": 3.65625, "learning_rate": 9.481040014244958e-07, "loss": 0.1438, "reward": 0.15018796920776367, "reward_std": 0.0, "rewards/accuracy_reward": 0.15018796920776367, "step": 307 }, { "clip_ratio": 0.0, "completion_length": 11.1875, "epoch": 0.24015594541910332, "grad_norm": 0.39492674037455167, "kl": 3.05078125, "learning_rate": 9.475308982494667e-07, "loss": 0.1221, "reward": 0.15859133005142212, "reward_std": 0.0, "rewards/accuracy_reward": 0.15859133005142212, "step": 308 }, { "clip_ratio": 0.0, "completion_length": 13.354166984558105, "epoch": 0.2409356725146199, "grad_norm": 6.062163696431718, "kl": 2.921875, "learning_rate": 9.469548437853491e-07, "loss": 0.2749, "reward": 0.14673330634832382, "reward_std": 0.00951569527387619, "rewards/accuracy_reward": 0.14673330634832382, "step": 309 }, { "clip_ratio": 0.0, "completion_length": 12.020833492279053, "epoch": 0.24171539961013644, "grad_norm": 11.790569655461159, "kl": 4.0234375, "learning_rate": 9.463758423088002e-07, "loss": 0.2291, "reward": 0.1294768750667572, "reward_std": 0.007609860273078084, "rewards/accuracy_reward": 0.1294768750667572, "step": 310 }, { "clip_ratio": 0.0, "completion_length": 10.166666984558105, "epoch": 0.242495126705653, "grad_norm": 5.684964061783532, "kl": 3.21875, "learning_rate": 9.457938981183556e-07, "loss": 0.0548, "reward": 0.16567786782979965, "reward_std": 0.011467931792140007, "rewards/accuracy_reward": 0.16567786782979965, "step": 311 }, { "clip_ratio": 0.0, "completion_length": 9.375, "epoch": 0.2432748538011696, "grad_norm": 4.733368822179079, "kl": 3.5390625, "learning_rate": 9.452090155343978e-07, "loss": 0.1685, "reward": 0.1623220592737198, "reward_std": 0.01105591095983982, "rewards/accuracy_reward": 0.1623220592737198, "step": 312 }, { "clip_ratio": 0.0, "completion_length": 15.208333492279053, "epoch": 0.24405458089668616, "grad_norm": 10.099511461886369, "kl": 2.13671875, "learning_rate": 9.446211988991239e-07, "loss": 0.0399, "reward": 0.16994038969278336, "reward_std": 0.005950515856966376, "rewards/accuracy_reward": 0.16994038969278336, "step": 313 }, { "clip_ratio": 0.0, "completion_length": 8.145833492279053, "epoch": 0.24483430799220274, "grad_norm": 2.7921139224320517, "kl": 3.59375, "learning_rate": 9.440304525765139e-07, "loss": 0.1239, "reward": 0.1476425603032112, "reward_std": 0.0005697530577890575, "rewards/accuracy_reward": 0.1476425603032112, "step": 314 }, { "clip_ratio": 0.0, "completion_length": 11.875, "epoch": 0.24561403508771928, "grad_norm": 2.555359893692319, "kl": 2.953125, "learning_rate": 9.434367809522978e-07, "loss": 0.1581, "reward": 0.14604929089546204, "reward_std": 0.003157075960189104, "rewards/accuracy_reward": 0.14604929089546204, "step": 315 }, { "clip_ratio": 0.0, "completion_length": 13.645833492279053, "epoch": 0.24639376218323586, "grad_norm": 6.368718435772907, "kl": 2.7265625, "learning_rate": 9.428401884339231e-07, "loss": 0.0229, "reward": 0.1592159867286682, "reward_std": 0.01695099240168929, "rewards/accuracy_reward": 0.1592159867286682, "step": 316 }, { "clip_ratio": 0.0, "completion_length": 15.770833969116211, "epoch": 0.24717348927875243, "grad_norm": 4.946457453057061, "kl": 2.6640625, "learning_rate": 9.422406794505221e-07, "loss": 0.1924, "reward": 0.16760166734457016, "reward_std": 0.011069273576140404, "rewards/accuracy_reward": 0.16760166734457016, "step": 317 }, { "clip_ratio": 0.0, "completion_length": 8.520833492279053, "epoch": 0.247953216374269, "grad_norm": 1.8874857401256442, "kl": 3.671875, "learning_rate": 9.416382584528794e-07, "loss": 0.1572, "reward": 0.15528500080108643, "reward_std": 0.0030617068987339735, "rewards/accuracy_reward": 0.15528500080108643, "step": 318 }, { "clip_ratio": 0.0, "completion_length": 11.708333492279053, "epoch": 0.24873294346978558, "grad_norm": 1.8136733529034799, "kl": 2.859375, "learning_rate": 9.410329299133982e-07, "loss": 0.1069, "reward": 0.14370537549257278, "reward_std": 0.003252904862165451, "rewards/accuracy_reward": 0.14370537549257278, "step": 319 }, { "clip_ratio": 0.0, "completion_length": 13.166666984558105, "epoch": 0.24951267056530213, "grad_norm": 2.6927140379913945, "kl": 2.8515625, "learning_rate": 9.404246983260677e-07, "loss": 0.1004, "reward": 0.1607775092124939, "reward_std": 0.004481068812310696, "rewards/accuracy_reward": 0.1607775092124939, "step": 320 }, { "clip_ratio": 0.0, "completion_length": 10.083333730697632, "epoch": 0.25029239766081873, "grad_norm": 4.340242300695142, "kl": 3.140625, "learning_rate": 9.398135682064292e-07, "loss": 0.1486, "reward": 0.15591206401586533, "reward_std": 0.001132059609517455, "rewards/accuracy_reward": 0.15591206401586533, "step": 321 }, { "clip_ratio": 0.0, "completion_length": 14.250000476837158, "epoch": 0.2510721247563353, "grad_norm": 3.9537646142295215, "kl": 3.2578125, "learning_rate": 9.391995440915431e-07, "loss": 0.104, "reward": 0.14638651907444, "reward_std": 0.009758115746080875, "rewards/accuracy_reward": 0.14638651907444, "step": 322 }, { "clip_ratio": 0.0, "completion_length": 13.104167461395264, "epoch": 0.2518518518518518, "grad_norm": 2.4833154466196956, "kl": 2.8359375, "learning_rate": 9.385826305399549e-07, "loss": 0.1415, "reward": 0.16239789128303528, "reward_std": 0.0011320605408400297, "rewards/accuracy_reward": 0.16239789128303528, "step": 323 }, { "clip_ratio": 0.0, "completion_length": 7.125, "epoch": 0.25263157894736843, "grad_norm": 6.95972183622307, "kl": 4.4765625, "learning_rate": 9.379628321316612e-07, "loss": 0.3666, "reward": 0.14668265730142593, "reward_std": 0.01086803199723363, "rewards/accuracy_reward": 0.14668265730142593, "step": 324 }, { "clip_ratio": 0.0, "completion_length": 11.541666984558105, "epoch": 0.253411306042885, "grad_norm": 0.4617833333827291, "kl": 2.84375, "learning_rate": 9.373401534680764e-07, "loss": 0.1075, "reward": 0.15859133750200272, "reward_std": 0.0, "rewards/accuracy_reward": 0.15859133750200272, "step": 325 }, { "clip_ratio": 0.0, "completion_length": 17.20833396911621, "epoch": 0.2541910331384016, "grad_norm": 1.8132498916325728, "kl": 2.20703125, "learning_rate": 9.367145991719973e-07, "loss": 0.0625, "reward": 0.17597854137420654, "reward_std": 0.00710753258317709, "rewards/accuracy_reward": 0.17597854137420654, "step": 326 }, { "clip_ratio": 0.0, "completion_length": 14.395833969116211, "epoch": 0.2549707602339181, "grad_norm": 2.977715983331237, "kl": 2.8359375, "learning_rate": 9.360861738875704e-07, "loss": 0.1902, "reward": 0.1683361977338791, "reward_std": 0.007850188296288252, "rewards/accuracy_reward": 0.1683361977338791, "step": 327 }, { "clip_ratio": 0.0, "completion_length": 6.125000238418579, "epoch": 0.2557504873294347, "grad_norm": 3.452349275804109, "kl": 3.75, "learning_rate": 9.354548822802559e-07, "loss": 0.1766, "reward": 0.13416200131177902, "reward_std": 0.005377282854169607, "rewards/accuracy_reward": 0.13416200131177902, "step": 328 }, { "clip_ratio": 0.0, "completion_length": 9.250000476837158, "epoch": 0.2565302144249513, "grad_norm": 4.67086414848647, "kl": 3.9453125, "learning_rate": 9.348207290367941e-07, "loss": 0.3177, "reward": 0.1363159865140915, "reward_std": 0.0035982707049697638, "rewards/accuracy_reward": 0.1363159865140915, "step": 329 }, { "clip_ratio": 0.0, "completion_length": 11.625, "epoch": 0.2573099415204678, "grad_norm": 4.299597889128723, "kl": 3.4140625, "learning_rate": 9.341837188651703e-07, "loss": 0.2106, "reward": 0.15561115741729736, "reward_std": 0.002914828830398619, "rewards/accuracy_reward": 0.15561115741729736, "step": 330 }, { "clip_ratio": 0.0, "completion_length": 10.729166984558105, "epoch": 0.2580896686159844, "grad_norm": 2.140848393044737, "kl": 2.9765625, "learning_rate": 9.335438564945791e-07, "loss": 0.0593, "reward": 0.16010206937789917, "reward_std": 0.0031131631694734097, "rewards/accuracy_reward": 0.16010206937789917, "step": 331 }, { "clip_ratio": 0.0, "completion_length": 7.583333730697632, "epoch": 0.25886939571150097, "grad_norm": 1.78145936593844, "kl": 3.4765625, "learning_rate": 9.329011466753908e-07, "loss": 0.1375, "reward": 0.14783550053834915, "reward_std": 0.0, "rewards/accuracy_reward": 0.14783550053834915, "step": 332 }, { "clip_ratio": 0.0, "completion_length": 8.833333492279053, "epoch": 0.2596491228070175, "grad_norm": 9.249352111457235, "kl": 4.421875, "learning_rate": 9.322555941791149e-07, "loss": 0.2628, "reward": 0.14634396135807037, "reward_std": 0.0032874359749257565, "rewards/accuracy_reward": 0.14634396135807037, "step": 333 }, { "clip_ratio": 0.0, "completion_length": 11.020833730697632, "epoch": 0.2604288499025341, "grad_norm": 6.269717979673763, "kl": 3.1171875, "learning_rate": 9.316072037983649e-07, "loss": 0.2294, "reward": 0.16324052959680557, "reward_std": 0.008229773491621017, "rewards/accuracy_reward": 0.16324052959680557, "step": 334 }, { "clip_ratio": 0.0, "completion_length": 6.187500238418579, "epoch": 0.26120857699805067, "grad_norm": 1.75105088193478, "kl": 3.9375, "learning_rate": 9.309559803468235e-07, "loss": 0.1671, "reward": 0.14475446939468384, "reward_std": 0.005799277685582638, "rewards/accuracy_reward": 0.14475446939468384, "step": 335 }, { "clip_ratio": 0.0, "completion_length": 10.500000476837158, "epoch": 0.26198830409356727, "grad_norm": 4.172643295553986, "kl": 3.2578125, "learning_rate": 9.303019286592057e-07, "loss": 0.1823, "reward": 0.15000522881746292, "reward_std": 0.005518701858818531, "rewards/accuracy_reward": 0.15000522881746292, "step": 336 }, { "clip_ratio": 0.0, "completion_length": 7.9791669845581055, "epoch": 0.2627680311890838, "grad_norm": 0.22902570185654725, "kl": 3.3828125, "learning_rate": 9.296450535912236e-07, "loss": 0.1334, "reward": 0.14925595372915268, "reward_std": 0.0, "rewards/accuracy_reward": 0.14925595372915268, "step": 337 }, { "clip_ratio": 0.0, "completion_length": 8.354166984558105, "epoch": 0.26354775828460036, "grad_norm": 0.49800815087424305, "kl": 3.796875, "learning_rate": 9.289853600195501e-07, "loss": 0.1461, "reward": 0.1659226268529892, "reward_std": 0.0, "rewards/accuracy_reward": 0.1659226268529892, "step": 338 }, { "clip_ratio": 0.0, "completion_length": 6.000000238418579, "epoch": 0.26432748538011697, "grad_norm": 0.20112105370298947, "kl": 3.46875, "learning_rate": 9.283228528417832e-07, "loss": 0.1385, "reward": 0.13571429997682571, "reward_std": 0.0, "rewards/accuracy_reward": 0.13571429997682571, "step": 339 }, { "clip_ratio": 0.0, "completion_length": 9.041666984558105, "epoch": 0.2651072124756335, "grad_norm": 0.5183928331293166, "kl": 3.15625, "learning_rate": 9.276575369764091e-07, "loss": 0.122, "reward": 0.14783550798892975, "reward_std": 0.0, "rewards/accuracy_reward": 0.14783550798892975, "step": 340 }, { "clip_ratio": 0.0, "completion_length": 5.958333492279053, "epoch": 0.2658869395711501, "grad_norm": 0.12535914977379958, "kl": 3.6484375, "learning_rate": 9.269894173627656e-07, "loss": 0.1461, "reward": 0.13571429997682571, "reward_std": 0.0, "rewards/accuracy_reward": 0.13571429997682571, "step": 341 }, { "clip_ratio": 0.0, "completion_length": 5.0, "epoch": 0.26666666666666666, "grad_norm": 0.11410982381062941, "kl": 4.2578125, "learning_rate": 9.263184989610058e-07, "loss": 0.17, "reward": 0.15000001341104507, "reward_std": 0.0, "rewards/accuracy_reward": 0.15000001341104507, "step": 342 }, { "clip_ratio": 0.0, "completion_length": 9.8125, "epoch": 0.2674463937621832, "grad_norm": 2.8133131091506596, "kl": 3.5859375, "learning_rate": 9.256447867520612e-07, "loss": 0.2071, "reward": 0.14612681418657303, "reward_std": 0.001217866549268365, "rewards/accuracy_reward": 0.14612681418657303, "step": 343 }, { "clip_ratio": 0.0, "completion_length": 10.812500476837158, "epoch": 0.2682261208576998, "grad_norm": 0.48566940184021756, "kl": 3.2265625, "learning_rate": 9.249682857376044e-07, "loss": 0.1237, "reward": 0.16228070855140686, "reward_std": 0.0, "rewards/accuracy_reward": 0.16228070855140686, "step": 344 }, { "clip_ratio": 0.0, "completion_length": 6.458333492279053, "epoch": 0.26900584795321636, "grad_norm": 4.965130657921159, "kl": 4.1796875, "learning_rate": 9.24289000940013e-07, "loss": 0.2625, "reward": 0.13326352089643478, "reward_std": 0.0013975434703752398, "rewards/accuracy_reward": 0.13326352089643478, "step": 345 }, { "clip_ratio": 0.0, "completion_length": 10.104166984558105, "epoch": 0.26978557504873296, "grad_norm": 5.914165711549631, "kl": 3.796875, "learning_rate": 9.236069374023304e-07, "loss": 0.2648, "reward": 0.16691932827234268, "reward_std": 0.003851958317682147, "rewards/accuracy_reward": 0.16691932827234268, "step": 346 }, { "clip_ratio": 0.0, "completion_length": 6.250000238418579, "epoch": 0.2705653021442495, "grad_norm": 6.377356066175931, "kl": 4.84375, "learning_rate": 9.229221001882307e-07, "loss": 0.3321, "reward": 0.1592262014746666, "reward_std": 0.0021980637684464455, "rewards/accuracy_reward": 0.1592262014746666, "step": 347 }, { "clip_ratio": 0.0, "completion_length": 12.791666984558105, "epoch": 0.27134502923976606, "grad_norm": 3.883496879866113, "kl": 3.3515625, "learning_rate": 9.222344943819788e-07, "loss": 0.1282, "reward": 0.1351918950676918, "reward_std": 0.006383164552971721, "rewards/accuracy_reward": 0.1351918950676918, "step": 348 }, { "clip_ratio": 0.0, "completion_length": 8.5, "epoch": 0.27212475633528266, "grad_norm": 0.19387684774557348, "kl": 3.5, "learning_rate": 9.215441250883944e-07, "loss": 0.138, "reward": 0.1418546363711357, "reward_std": 0.0, "rewards/accuracy_reward": 0.1418546363711357, "step": 349 }, { "clip_ratio": 0.0, "completion_length": 12.791666984558105, "epoch": 0.2729044834307992, "grad_norm": 2.6934537197535136, "kl": 3.1328125, "learning_rate": 9.208509974328136e-07, "loss": 0.1473, "reward": 0.1584201380610466, "reward_std": 0.011034275405108929, "rewards/accuracy_reward": 0.1584201380610466, "step": 350 }, { "clip_ratio": 0.0, "completion_length": 8.770833730697632, "epoch": 0.2736842105263158, "grad_norm": 14.909744743109016, "kl": 4.359375, "learning_rate": 9.201551165610502e-07, "loss": 0.3776, "reward": 0.15026964992284775, "reward_std": 0.010787321021780372, "rewards/accuracy_reward": 0.15026964992284775, "step": 351 }, { "clip_ratio": 0.0, "completion_length": 11.125000476837158, "epoch": 0.27446393762183235, "grad_norm": 4.346522515113751, "kl": 3.5546875, "learning_rate": 9.194564876393583e-07, "loss": 0.1823, "reward": 0.16232681274414062, "reward_std": 0.009733288548886776, "rewards/accuracy_reward": 0.16232681274414062, "step": 352 }, { "clip_ratio": 0.0, "completion_length": 12.541666984558105, "epoch": 0.2752436647173489, "grad_norm": 3.9139850815442623, "kl": 3.2890625, "learning_rate": 9.187551158543938e-07, "loss": 0.2982, "reward": 0.15423484146595, "reward_std": 0.004482646938413382, "rewards/accuracy_reward": 0.15423484146595, "step": 353 }, { "clip_ratio": 0.0, "completion_length": 7.6041669845581055, "epoch": 0.2760233918128655, "grad_norm": 0.4098093213063153, "kl": 3.6484375, "learning_rate": 9.180510064131755e-07, "loss": 0.1423, "reward": 0.1478070318698883, "reward_std": 0.0, "rewards/accuracy_reward": 0.1478070318698883, "step": 354 }, { "clip_ratio": 0.0, "completion_length": 14.020833969116211, "epoch": 0.27680311890838205, "grad_norm": 0.33872616889624524, "kl": 3.015625, "learning_rate": 9.173441645430467e-07, "loss": 0.1126, "reward": 0.17127691954374313, "reward_std": 0.0, "rewards/accuracy_reward": 0.17127691954374313, "step": 355 }, { "clip_ratio": 0.0, "completion_length": 10.500000476837158, "epoch": 0.27758284600389865, "grad_norm": 13.25726618671934, "kl": 4.1875, "learning_rate": 9.166345954916366e-07, "loss": 0.1409, "reward": 0.14254386723041534, "reward_std": 0.003798356279730797, "rewards/accuracy_reward": 0.14254386723041534, "step": 356 }, { "clip_ratio": 0.0, "completion_length": 7.895833492279053, "epoch": 0.2783625730994152, "grad_norm": 4.181253217697556, "kl": 3.9453125, "learning_rate": 9.159223045268208e-07, "loss": 0.2647, "reward": 0.16294778883457184, "reward_std": 0.002057300880551338, "rewards/accuracy_reward": 0.16294778883457184, "step": 357 }, { "clip_ratio": 0.0, "completion_length": 8.791666984558105, "epoch": 0.27914230019493175, "grad_norm": 4.5724541162695544, "kl": 3.6328125, "learning_rate": 9.152072969366831e-07, "loss": 0.2806, "reward": 0.15755286812782288, "reward_std": 0.0021742512471973896, "rewards/accuracy_reward": 0.15755286812782288, "step": 358 }, { "clip_ratio": 0.0, "completion_length": 10.979166984558105, "epoch": 0.27992202729044835, "grad_norm": 3.7584675453246024, "kl": 3.4375, "learning_rate": 9.144895780294748e-07, "loss": 0.1788, "reward": 0.14665570110082626, "reward_std": 0.0017377210315316916, "rewards/accuracy_reward": 0.14665570110082626, "step": 359 }, { "clip_ratio": 0.0, "completion_length": 10.416666984558105, "epoch": 0.2807017543859649, "grad_norm": 5.963708036467045, "kl": 3.8359375, "learning_rate": 9.13769153133577e-07, "loss": 0.2013, "reward": 0.14726510643959045, "reward_std": 0.0023518577218055725, "rewards/accuracy_reward": 0.14726510643959045, "step": 360 }, { "clip_ratio": 0.0, "completion_length": 6.395833492279053, "epoch": 0.2814814814814815, "grad_norm": 10.09384850614911, "kl": 4.40625, "learning_rate": 9.130460275974596e-07, "loss": 0.244, "reward": 0.14828869700431824, "reward_std": 0.011598555836826563, "rewards/accuracy_reward": 0.14828869700431824, "step": 361 }, { "clip_ratio": 0.0, "completion_length": 6.729166746139526, "epoch": 0.28226120857699805, "grad_norm": 2.9379058535221914, "kl": 3.875, "learning_rate": 9.123202067896427e-07, "loss": 0.2484, "reward": 0.1378328800201416, "reward_std": 0.0030768176075071096, "rewards/accuracy_reward": 0.1378328800201416, "step": 362 }, { "clip_ratio": 0.0, "completion_length": 10.166666984558105, "epoch": 0.2830409356725146, "grad_norm": 0.3950653711421381, "kl": 3.5859375, "learning_rate": 9.115916960986556e-07, "loss": 0.1284, "reward": 0.16692467033863068, "reward_std": 0.0, "rewards/accuracy_reward": 0.16692467033863068, "step": 363 }, { "clip_ratio": 0.0, "completion_length": 13.145833492279053, "epoch": 0.2838206627680312, "grad_norm": 3.6595124366360494, "kl": 2.859375, "learning_rate": 9.10860500932998e-07, "loss": 0.1911, "reward": 0.1389584057033062, "reward_std": 0.0046866354532539845, "rewards/accuracy_reward": 0.1389584057033062, "step": 364 }, { "clip_ratio": 0.0, "completion_length": 8.729166984558105, "epoch": 0.28460038986354774, "grad_norm": 2.884781169318846, "kl": 3.7421875, "learning_rate": 9.10126626721099e-07, "loss": 0.2106, "reward": 0.14237084984779358, "reward_std": 0.00047169136814773083, "rewards/accuracy_reward": 0.14237084984779358, "step": 365 }, { "clip_ratio": 0.0, "completion_length": 14.395833492279053, "epoch": 0.28538011695906434, "grad_norm": 2.7804319305305816, "kl": 2.5859375, "learning_rate": 9.09390078911277e-07, "loss": 0.2129, "reward": 0.17890625447034836, "reward_std": 0.00270632840692997, "rewards/accuracy_reward": 0.17890625447034836, "step": 366 }, { "clip_ratio": 0.0, "completion_length": 7.375, "epoch": 0.2861598440545809, "grad_norm": 0.38207941828337255, "kl": 3.6796875, "learning_rate": 9.086508629716997e-07, "loss": 0.143, "reward": 0.15378788113594055, "reward_std": 0.0, "rewards/accuracy_reward": 0.15378788113594055, "step": 367 }, { "clip_ratio": 0.0, "completion_length": 6.083333492279053, "epoch": 0.28693957115009744, "grad_norm": 3.723354909225953, "kl": 3.875, "learning_rate": 9.079089843903425e-07, "loss": 0.2504, "reward": 0.13127674162387848, "reward_std": 0.00120510661508888, "rewards/accuracy_reward": 0.13127674162387848, "step": 368 }, { "clip_ratio": 0.0, "completion_length": 11.291666984558105, "epoch": 0.28771929824561404, "grad_norm": 1.3526253622053168, "kl": 2.984375, "learning_rate": 9.071644486749493e-07, "loss": 0.1132, "reward": 0.1708245947957039, "reward_std": 0.0, "rewards/accuracy_reward": 0.1708245947957039, "step": 369 }, { "clip_ratio": 0.0, "completion_length": 9.041666984558105, "epoch": 0.2884990253411306, "grad_norm": 5.737068939101509, "kl": 3.90625, "learning_rate": 9.064172613529895e-07, "loss": 0.1531, "reward": 0.16008080542087555, "reward_std": 0.007817985489964485, "rewards/accuracy_reward": 0.16008080542087555, "step": 370 }, { "clip_ratio": 0.0, "completion_length": 8.125, "epoch": 0.2892787524366472, "grad_norm": 2.813923046261043, "kl": 3.421875, "learning_rate": 9.056674279716192e-07, "loss": 0.1596, "reward": 0.14751984924077988, "reward_std": 0.001093466067686677, "rewards/accuracy_reward": 0.14751984924077988, "step": 371 }, { "clip_ratio": 0.0, "completion_length": 8.833333492279053, "epoch": 0.29005847953216374, "grad_norm": 8.069589843107199, "kl": 3.734375, "learning_rate": 9.049149540976384e-07, "loss": 0.1032, "reward": 0.15217722207307816, "reward_std": 0.005698587745428085, "rewards/accuracy_reward": 0.15217722207307816, "step": 372 }, { "clip_ratio": 0.0, "completion_length": 7.250000238418579, "epoch": 0.2908382066276803, "grad_norm": 15.197853361150488, "kl": 4.84375, "learning_rate": 9.041598453174506e-07, "loss": 0.261, "reward": 0.13517910242080688, "reward_std": 0.002728496619965881, "rewards/accuracy_reward": 0.13517910242080688, "step": 373 }, { "clip_ratio": 0.0, "completion_length": 5.812500238418579, "epoch": 0.2916179337231969, "grad_norm": 0.31898871624716346, "kl": 3.890625, "learning_rate": 9.034021072370207e-07, "loss": 0.1557, "reward": 0.14404762536287308, "reward_std": 0.0, "rewards/accuracy_reward": 0.14404762536287308, "step": 374 }, { "clip_ratio": 0.0, "completion_length": 5.75, "epoch": 0.29239766081871343, "grad_norm": 0.2822671515922015, "kl": 3.859375, "learning_rate": 9.026417454818338e-07, "loss": 0.1543, "reward": 0.1488095372915268, "reward_std": 0.0, "rewards/accuracy_reward": 0.1488095372915268, "step": 375 }, { "clip_ratio": 0.0, "completion_length": 11.645833730697632, "epoch": 0.29317738791423004, "grad_norm": 14.389676610167912, "kl": 3.3046875, "learning_rate": 9.018787656968533e-07, "loss": 0.034, "reward": 0.16039849817752838, "reward_std": 0.009118756745010614, "rewards/accuracy_reward": 0.16039849817752838, "step": 376 }, { "clip_ratio": 0.0, "completion_length": 5.812500238418579, "epoch": 0.2939571150097466, "grad_norm": 0.22846529022108444, "kl": 3.8203125, "learning_rate": 9.011131735464787e-07, "loss": 0.1519, "reward": 0.14166668057441711, "reward_std": 0.0, "rewards/accuracy_reward": 0.14166668057441711, "step": 377 }, { "clip_ratio": 0.0, "completion_length": 8.208333730697632, "epoch": 0.29473684210526313, "grad_norm": 2.11397301902399, "kl": 3.703125, "learning_rate": 9.003449747145043e-07, "loss": 0.1282, "reward": 0.15699300915002823, "reward_std": 0.0020187075715512037, "rewards/accuracy_reward": 0.15699300915002823, "step": 378 }, { "clip_ratio": 0.0, "completion_length": 9.166666746139526, "epoch": 0.29551656920077973, "grad_norm": 2.9055223019600143, "kl": 3.2578125, "learning_rate": 8.995741749040764e-07, "loss": 0.1319, "reward": 0.15838926285505295, "reward_std": 0.0066304802894592285, "rewards/accuracy_reward": 0.15838926285505295, "step": 379 }, { "clip_ratio": 0.0, "completion_length": 8.770833492279053, "epoch": 0.2962962962962963, "grad_norm": 5.40273553068801, "kl": 3.9296875, "learning_rate": 8.988007798376504e-07, "loss": 0.1897, "reward": 0.147501602768898, "reward_std": 0.0076894816011190414, "rewards/accuracy_reward": 0.147501602768898, "step": 380 }, { "clip_ratio": 0.0, "completion_length": 6.541666746139526, "epoch": 0.2970760233918129, "grad_norm": 4.315360565948567, "kl": 3.8125, "learning_rate": 8.980247952569504e-07, "loss": 0.1129, "reward": 0.1444043293595314, "reward_std": 0.003221820807084441, "rewards/accuracy_reward": 0.1444043293595314, "step": 381 }, { "clip_ratio": 0.0, "completion_length": 12.916666984558105, "epoch": 0.29785575048732943, "grad_norm": 1.9824329610853104, "kl": 2.5234375, "learning_rate": 8.97246226922924e-07, "loss": 0.0495, "reward": 0.16020382195711136, "reward_std": 0.0029046256095170975, "rewards/accuracy_reward": 0.16020382195711136, "step": 382 }, { "clip_ratio": 0.0, "completion_length": 9.916666984558105, "epoch": 0.298635477582846, "grad_norm": 3.682796593140469, "kl": 3.8515625, "learning_rate": 8.964650806157008e-07, "loss": 0.1929, "reward": 0.17954282462596893, "reward_std": 0.009522270411252975, "rewards/accuracy_reward": 0.17954282462596893, "step": 383 }, { "clip_ratio": 0.0, "completion_length": 5.625, "epoch": 0.2994152046783626, "grad_norm": 8.80835120764374, "kl": 4.375, "learning_rate": 8.956813621345501e-07, "loss": 0.3315, "reward": 0.15191799402236938, "reward_std": 0.001603750861249864, "rewards/accuracy_reward": 0.15191799402236938, "step": 384 }, { "clip_ratio": 0.0, "completion_length": 6.0, "epoch": 0.3001949317738791, "grad_norm": 10.604700425523651, "kl": 4.546875, "learning_rate": 8.948950772978367e-07, "loss": 0.292, "reward": 0.14568452537059784, "reward_std": 0.002577458042651415, "rewards/accuracy_reward": 0.14568452537059784, "step": 385 }, { "clip_ratio": 0.0, "completion_length": 17.083333492279053, "epoch": 0.30097465886939573, "grad_norm": 3.636862344464563, "kl": 2.8671875, "learning_rate": 8.941062319429781e-07, "loss": 0.1311, "reward": 0.15528112649917603, "reward_std": 0.01720402017235756, "rewards/accuracy_reward": 0.15528112649917603, "step": 386 }, { "clip_ratio": 0.0, "completion_length": 5.270833492279053, "epoch": 0.3017543859649123, "grad_norm": 0.28736744287416843, "kl": 4.1640625, "learning_rate": 8.933148319264013e-07, "loss": 0.1665, "reward": 0.15000001341104507, "reward_std": 0.0, "rewards/accuracy_reward": 0.15000001341104507, "step": 387 }, { "clip_ratio": 0.0, "completion_length": 9.166666984558105, "epoch": 0.3025341130604289, "grad_norm": 4.808257587397571, "kl": 3.4609375, "learning_rate": 8.925208831234993e-07, "loss": 0.1747, "reward": 0.14375001192092896, "reward_std": 0.005904720164835453, "rewards/accuracy_reward": 0.14375001192092896, "step": 388 }, { "clip_ratio": 0.0, "completion_length": 5.8125, "epoch": 0.3033138401559454, "grad_norm": 0.2676486555342594, "kl": 3.8828125, "learning_rate": 8.917243914285873e-07, "loss": 0.1549, "reward": 0.14642857760190964, "reward_std": 0.0, "rewards/accuracy_reward": 0.14642857760190964, "step": 389 }, { "clip_ratio": 0.0, "completion_length": 8.4375, "epoch": 0.30409356725146197, "grad_norm": 1.7504264799915286, "kl": 3.3671875, "learning_rate": 8.909253627548591e-07, "loss": 0.1268, "reward": 0.15033482760190964, "reward_std": 0.004510548897087574, "rewards/accuracy_reward": 0.15033482760190964, "step": 390 }, { "clip_ratio": 0.0, "completion_length": 8.854166984558105, "epoch": 0.3048732943469786, "grad_norm": 5.492992611635677, "kl": 3.3984375, "learning_rate": 8.901238030343432e-07, "loss": 0.268, "reward": 0.15573524683713913, "reward_std": 0.0051187395583838224, "rewards/accuracy_reward": 0.15573524683713913, "step": 391 }, { "clip_ratio": 0.0, "completion_length": 8.958333492279053, "epoch": 0.3056530214424951, "grad_norm": 3.33193497315929, "kl": 3.2734375, "learning_rate": 8.893197182178587e-07, "loss": 0.1966, "reward": 0.14758404344320297, "reward_std": 0.003648071549832821, "rewards/accuracy_reward": 0.14758404344320297, "step": 392 }, { "clip_ratio": 0.0, "completion_length": 8.020833492279053, "epoch": 0.3064327485380117, "grad_norm": 0.25274717008869885, "kl": 3.6953125, "learning_rate": 8.885131142749712e-07, "loss": 0.1452, "reward": 0.16316527873277664, "reward_std": 0.0, "rewards/accuracy_reward": 0.16316527873277664, "step": 393 }, { "clip_ratio": 0.0, "completion_length": 7.9166669845581055, "epoch": 0.30721247563352827, "grad_norm": 6.4668385222053075, "kl": 4.234375, "learning_rate": 8.877039971939481e-07, "loss": 0.1728, "reward": 0.1547979861497879, "reward_std": 0.009622504934668541, "rewards/accuracy_reward": 0.1547979861497879, "step": 394 }, { "clip_ratio": 0.0, "completion_length": 13.041666984558105, "epoch": 0.3079922027290448, "grad_norm": 0.2118585348560572, "kl": 2.6640625, "learning_rate": 8.868923729817149e-07, "loss": 0.1048, "reward": 0.17120098322629929, "reward_std": 0.0, "rewards/accuracy_reward": 0.17120098322629929, "step": 395 }, { "clip_ratio": 0.0, "completion_length": 7.291666746139526, "epoch": 0.3087719298245614, "grad_norm": 0.2801637016272516, "kl": 3.515625, "learning_rate": 8.860782476638101e-07, "loss": 0.1398, "reward": 0.1454545557498932, "reward_std": 0.0, "rewards/accuracy_reward": 0.1454545557498932, "step": 396 }, { "clip_ratio": 0.0, "completion_length": 9.145833492279053, "epoch": 0.30955165692007797, "grad_norm": 0.3351542044371962, "kl": 3.65625, "learning_rate": 8.852616272843402e-07, "loss": 0.1405, "reward": 0.16457219421863556, "reward_std": 0.0, "rewards/accuracy_reward": 0.16457219421863556, "step": 397 }, { "clip_ratio": 0.0, "completion_length": 8.604166746139526, "epoch": 0.31033138401559457, "grad_norm": 2.323477297979642, "kl": 3.390625, "learning_rate": 8.844425179059353e-07, "loss": 0.1841, "reward": 0.15334384888410568, "reward_std": 0.0021980637684464455, "rewards/accuracy_reward": 0.15334384888410568, "step": 398 }, { "clip_ratio": 0.0, "completion_length": 10.958333969116211, "epoch": 0.3111111111111111, "grad_norm": 3.3294560113411316, "kl": 3.390625, "learning_rate": 8.83620925609704e-07, "loss": 0.2522, "reward": 0.13990402966737747, "reward_std": 0.0066372910514473915, "rewards/accuracy_reward": 0.13990402966737747, "step": 399 }, { "clip_ratio": 0.0, "completion_length": 14.937500476837158, "epoch": 0.31189083820662766, "grad_norm": 0.23825880365650365, "kl": 2.484375, "learning_rate": 8.827968564951886e-07, "loss": 0.0983, "reward": 0.16851955652236938, "reward_std": 0.0, "rewards/accuracy_reward": 0.16851955652236938, "step": 400 }, { "clip_ratio": 0.0, "completion_length": 8.145833730697632, "epoch": 0.31267056530214427, "grad_norm": 2.280556383897255, "kl": 3.671875, "learning_rate": 8.819703166803186e-07, "loss": 0.1782, "reward": 0.15582707524299622, "reward_std": 0.0010852443519979715, "rewards/accuracy_reward": 0.15582707524299622, "step": 401 }, { "clip_ratio": 0.0, "completion_length": 7.458333492279053, "epoch": 0.3134502923976608, "grad_norm": 5.134190258006971, "kl": 4.28125, "learning_rate": 8.811413123013668e-07, "loss": 0.2236, "reward": 0.1684027910232544, "reward_std": 0.012028131633996964, "rewards/accuracy_reward": 0.1684027910232544, "step": 402 }, { "clip_ratio": 0.0, "completion_length": 11.437500476837158, "epoch": 0.3142300194931774, "grad_norm": 2.9231531072288197, "kl": 3.3125, "learning_rate": 8.803098495129028e-07, "loss": 0.1271, "reward": 0.1722072809934616, "reward_std": 0.0018650222336873412, "rewards/accuracy_reward": 0.1722072809934616, "step": 403 }, { "clip_ratio": 0.0, "completion_length": 9.645833969116211, "epoch": 0.31500974658869396, "grad_norm": 2.8193977710210634, "kl": 3.609375, "learning_rate": 8.794759344877474e-07, "loss": 0.1158, "reward": 0.15898126363754272, "reward_std": 0.0017131322529166937, "rewards/accuracy_reward": 0.15898126363754272, "step": 404 }, { "clip_ratio": 0.0, "completion_length": 5.729166746139526, "epoch": 0.3157894736842105, "grad_norm": 0.33673892958420754, "kl": 4.0078125, "learning_rate": 8.78639573416927e-07, "loss": 0.1579, "reward": 0.15000000596046448, "reward_std": 0.0, "rewards/accuracy_reward": 0.15000000596046448, "step": 405 }, { "clip_ratio": 0.0, "completion_length": 10.979166984558105, "epoch": 0.3165692007797271, "grad_norm": 0.61116109473038, "kl": 2.9375, "learning_rate": 8.778007725096278e-07, "loss": 0.1147, "reward": 0.16097228229045868, "reward_std": 0.0, "rewards/accuracy_reward": 0.16097228229045868, "step": 406 }, { "clip_ratio": 0.0, "completion_length": 8.0625, "epoch": 0.31734892787524366, "grad_norm": 0.44873850562449036, "kl": 3.5703125, "learning_rate": 8.769595379931495e-07, "loss": 0.1391, "reward": 0.15758929401636124, "reward_std": 0.0, "rewards/accuracy_reward": 0.15758929401636124, "step": 407 }, { "clip_ratio": 0.0, "completion_length": 10.125000476837158, "epoch": 0.31812865497076026, "grad_norm": 5.617503977168476, "kl": 3.59375, "learning_rate": 8.761158761128584e-07, "loss": 0.1939, "reward": 0.14398247003555298, "reward_std": 0.006505153025500476, "rewards/accuracy_reward": 0.14398247003555298, "step": 408 }, { "clip_ratio": 0.0, "completion_length": 11.666666984558105, "epoch": 0.3189083820662768, "grad_norm": 148.9446002531759, "kl": 10.8125, "learning_rate": 8.752697931321425e-07, "loss": 0.5094, "reward": 0.13543393835425377, "reward_std": 0.01484832726418972, "rewards/accuracy_reward": 0.13543393835425377, "step": 409 }, { "clip_ratio": 0.0, "completion_length": 5.833333492279053, "epoch": 0.31968810916179335, "grad_norm": 13.060620810867475, "kl": 4.578125, "learning_rate": 8.744212953323634e-07, "loss": 0.3419, "reward": 0.14298941940069199, "reward_std": 0.011913577443920076, "rewards/accuracy_reward": 0.14298941940069199, "step": 410 }, { "clip_ratio": 0.0, "completion_length": 10.520833492279053, "epoch": 0.32046783625730996, "grad_norm": 0.37751306006917723, "kl": 3.1328125, "learning_rate": 8.735703890128112e-07, "loss": 0.1244, "reward": 0.16097228229045868, "reward_std": 0.0, "rewards/accuracy_reward": 0.16097228229045868, "step": 411 }, { "clip_ratio": 0.0, "completion_length": 9.666666984558105, "epoch": 0.3212475633528265, "grad_norm": 7.635514043910877, "kl": 3.703125, "learning_rate": 8.727170804906562e-07, "loss": 0.1901, "reward": 0.15975462645292282, "reward_std": 0.0006014077225700021, "rewards/accuracy_reward": 0.15975462645292282, "step": 412 }, { "clip_ratio": 0.0, "completion_length": 11.270833730697632, "epoch": 0.3220272904483431, "grad_norm": 8.49376242180647, "kl": 3.0859375, "learning_rate": 8.718613761009035e-07, "loss": 0.2474, "reward": 0.14480237662792206, "reward_std": 0.02359803323633969, "rewards/accuracy_reward": 0.14480237662792206, "step": 413 }, { "clip_ratio": 0.0, "completion_length": 13.583333492279053, "epoch": 0.32280701754385965, "grad_norm": 3.3498673152295737, "kl": 2.36328125, "learning_rate": 8.710032821963446e-07, "loss": 0.1346, "reward": 0.16509104520082474, "reward_std": 0.010309826582670212, "rewards/accuracy_reward": 0.16509104520082474, "step": 414 }, { "clip_ratio": 0.0, "completion_length": 10.770833492279053, "epoch": 0.3235867446393762, "grad_norm": 0.5470724132997554, "kl": 2.765625, "learning_rate": 8.701428051475114e-07, "loss": 0.11, "reward": 0.15037594735622406, "reward_std": 0.0, "rewards/accuracy_reward": 0.15037594735622406, "step": 415 }, { "clip_ratio": 0.0, "completion_length": 7.708333492279053, "epoch": 0.3243664717348928, "grad_norm": 4.524361798588892, "kl": 3.4375, "learning_rate": 8.692799513426282e-07, "loss": 0.1458, "reward": 0.1558714210987091, "reward_std": 0.005950893857516348, "rewards/accuracy_reward": 0.1558714210987091, "step": 416 }, { "clip_ratio": 0.0, "completion_length": 8.437500238418579, "epoch": 0.32514619883040935, "grad_norm": 0.4281626061488469, "kl": 3.5234375, "learning_rate": 8.68414727187564e-07, "loss": 0.1386, "reward": 0.1478070169687271, "reward_std": 0.0, "rewards/accuracy_reward": 0.1478070169687271, "step": 417 }, { "clip_ratio": 0.0, "completion_length": 9.708333730697632, "epoch": 0.32592592592592595, "grad_norm": 0.3815308151814316, "kl": 3.3046875, "learning_rate": 8.675471391057862e-07, "loss": 0.1301, "reward": 0.16971050202846527, "reward_std": 0.0, "rewards/accuracy_reward": 0.16971050202846527, "step": 418 }, { "clip_ratio": 0.0, "completion_length": 13.604166746139526, "epoch": 0.3267056530214425, "grad_norm": 2.4913359083785487, "kl": 3.26953125, "learning_rate": 8.666771935383116e-07, "loss": 0.3131, "reward": 0.14368873089551926, "reward_std": 0.008988404646515846, "rewards/accuracy_reward": 0.14368873089551926, "step": 419 }, { "clip_ratio": 0.0, "completion_length": 8.291666746139526, "epoch": 0.32748538011695905, "grad_norm": 5.547399468303284, "kl": 3.40625, "learning_rate": 8.658048969436592e-07, "loss": 0.162, "reward": 0.14590337872505188, "reward_std": 0.010309826582670212, "rewards/accuracy_reward": 0.14590337872505188, "step": 420 }, { "clip_ratio": 0.0, "completion_length": 10.625000476837158, "epoch": 0.32826510721247565, "grad_norm": 0.3215622915459458, "kl": 3.2578125, "learning_rate": 8.649302557978022e-07, "loss": 0.121, "reward": 0.16692467033863068, "reward_std": 0.0, "rewards/accuracy_reward": 0.16692467033863068, "step": 421 }, { "clip_ratio": 0.0, "completion_length": 12.395833492279053, "epoch": 0.3290448343079922, "grad_norm": 7.027335178237231, "kl": 3.0, "learning_rate": 8.640532765941195e-07, "loss": 0.1462, "reward": 0.16403965651988983, "reward_std": 0.010524614714086056, "rewards/accuracy_reward": 0.16403965651988983, "step": 422 }, { "clip_ratio": 0.0, "completion_length": 12.354166984558105, "epoch": 0.3298245614035088, "grad_norm": 4.179235033753008, "kl": 3.296875, "learning_rate": 8.631739658433485e-07, "loss": 0.1947, "reward": 0.181765578687191, "reward_std": 0.005102843977510929, "rewards/accuracy_reward": 0.181765578687191, "step": 423 }, { "clip_ratio": 0.0, "completion_length": 6.7291669845581055, "epoch": 0.33060428849902534, "grad_norm": 2.285625994483756, "kl": 4.40625, "learning_rate": 8.622923300735358e-07, "loss": 0.1663, "reward": 0.14741533994674683, "reward_std": 0.008490445092320442, "rewards/accuracy_reward": 0.14741533994674683, "step": 424 }, { "clip_ratio": 0.0, "completion_length": 14.208333969116211, "epoch": 0.3313840155945419, "grad_norm": 3.689726399174867, "kl": 2.5703125, "learning_rate": 8.614083758299891e-07, "loss": 0.0867, "reward": 0.17276040464639664, "reward_std": 0.008812322281301022, "rewards/accuracy_reward": 0.17276040464639664, "step": 425 }, { "clip_ratio": 0.0, "completion_length": 8.25, "epoch": 0.3321637426900585, "grad_norm": 0.43087054871204195, "kl": 3.5703125, "learning_rate": 8.605221096752281e-07, "loss": 0.1397, "reward": 0.15018796920776367, "reward_std": 0.0, "rewards/accuracy_reward": 0.15018796920776367, "step": 426 }, { "clip_ratio": 0.0, "completion_length": 11.666666984558105, "epoch": 0.33294346978557504, "grad_norm": 3.0318116030178044, "kl": 3.109375, "learning_rate": 8.596335381889369e-07, "loss": 0.1307, "reward": 0.17109375447034836, "reward_std": 0.015056601725518703, "rewards/accuracy_reward": 0.17109375447034836, "step": 427 }, { "clip_ratio": 0.0, "completion_length": 8.041666984558105, "epoch": 0.33372319688109164, "grad_norm": 5.6166536824761915, "kl": 3.6640625, "learning_rate": 8.587426679679144e-07, "loss": 0.228, "reward": 0.13671675696969032, "reward_std": 0.013925573555752635, "rewards/accuracy_reward": 0.13671675696969032, "step": 428 }, { "clip_ratio": 0.0, "completion_length": 7.625000238418579, "epoch": 0.3345029239766082, "grad_norm": 4.669338580758022, "kl": 3.609375, "learning_rate": 8.57849505626025e-07, "loss": 0.1877, "reward": 0.1486913487315178, "reward_std": 0.011565987952053547, "rewards/accuracy_reward": 0.1486913487315178, "step": 429 }, { "clip_ratio": 0.0, "completion_length": 11.270833492279053, "epoch": 0.33528265107212474, "grad_norm": 4.710731228467101, "kl": 3.28125, "learning_rate": 8.569540577941504e-07, "loss": 0.1441, "reward": 0.14722733944654465, "reward_std": 0.009623454534448683, "rewards/accuracy_reward": 0.14722733944654465, "step": 430 }, { "clip_ratio": 0.0, "completion_length": 9.604166746139526, "epoch": 0.33606237816764134, "grad_norm": 2.922842872301241, "kl": 3.328125, "learning_rate": 8.560563311201394e-07, "loss": 0.1658, "reward": 0.15690506249666214, "reward_std": 0.000991815933957696, "rewards/accuracy_reward": 0.15690506249666214, "step": 431 }, { "clip_ratio": 0.0, "completion_length": 10.479166984558105, "epoch": 0.3368421052631579, "grad_norm": 5.40669886700069, "kl": 3.3828125, "learning_rate": 8.551563322687596e-07, "loss": 0.5143, "reward": 0.1448487639427185, "reward_std": 0.009350221138447523, "rewards/accuracy_reward": 0.1448487639427185, "step": 432 }, { "clip_ratio": 0.0, "completion_length": 8.333333969116211, "epoch": 0.3376218323586745, "grad_norm": 6.955284672632261, "kl": 3.890625, "learning_rate": 8.542540679216471e-07, "loss": 0.2239, "reward": 0.17085595428943634, "reward_std": 0.004075850360095501, "rewards/accuracy_reward": 0.17085595428943634, "step": 433 }, { "clip_ratio": 0.0, "completion_length": 7.937500238418579, "epoch": 0.33840155945419104, "grad_norm": 0.4577562610711916, "kl": 3.8359375, "learning_rate": 8.533495447772567e-07, "loss": 0.1481, "reward": 0.16450217366218567, "reward_std": 0.0, "rewards/accuracy_reward": 0.16450217366218567, "step": 434 }, { "clip_ratio": 0.0, "completion_length": 9.750000476837158, "epoch": 0.3391812865497076, "grad_norm": 135.6382012209044, "kl": 7.328125, "learning_rate": 8.524427695508136e-07, "loss": 0.4871, "reward": 0.15703052282333374, "reward_std": 0.003380400245077908, "rewards/accuracy_reward": 0.15703052282333374, "step": 435 }, { "clip_ratio": 0.0, "completion_length": 9.270833730697632, "epoch": 0.3399610136452242, "grad_norm": 3.2506215294238103, "kl": 3.90625, "learning_rate": 8.515337489742618e-07, "loss": 0.2412, "reward": 0.17852745950222015, "reward_std": 0.0009021099540404975, "rewards/accuracy_reward": 0.17852745950222015, "step": 436 }, { "clip_ratio": 0.0, "completion_length": 7.395833492279053, "epoch": 0.34074074074074073, "grad_norm": 4.37038860934232, "kl": 4.421875, "learning_rate": 8.50622489796215e-07, "loss": 0.1736, "reward": 0.1708763688802719, "reward_std": 0.011707212310284376, "rewards/accuracy_reward": 0.1708763688802719, "step": 437 }, { "clip_ratio": 0.0, "completion_length": 6.645833492279053, "epoch": 0.34152046783625734, "grad_norm": 15.208494503746422, "kl": 4.015625, "learning_rate": 8.497089987819064e-07, "loss": 0.1959, "reward": 0.1393013894557953, "reward_std": 0.011867815104778856, "rewards/accuracy_reward": 0.1393013894557953, "step": 438 }, { "clip_ratio": 0.0, "completion_length": 6.958333492279053, "epoch": 0.3423001949317739, "grad_norm": 2.500487241111051, "kl": 4.359375, "learning_rate": 8.487932827131389e-07, "loss": 0.1615, "reward": 0.17685356736183167, "reward_std": 0.0007959796348586679, "rewards/accuracy_reward": 0.17685356736183167, "step": 439 }, { "clip_ratio": 0.0, "completion_length": 7.520833492279053, "epoch": 0.34307992202729043, "grad_norm": 10.947441423527012, "kl": 4.328125, "learning_rate": 8.478753483882337e-07, "loss": 0.1412, "reward": 0.15975943952798843, "reward_std": 0.019432506524026394, "rewards/accuracy_reward": 0.15975943952798843, "step": 440 }, { "clip_ratio": 0.0, "completion_length": 6.833333492279053, "epoch": 0.34385964912280703, "grad_norm": 5.608765808188827, "kl": 3.859375, "learning_rate": 8.46955202621981e-07, "loss": 0.2917, "reward": 0.1515037640929222, "reward_std": 0.0030386867001652718, "rewards/accuracy_reward": 0.1515037640929222, "step": 441 }, { "clip_ratio": 0.0, "completion_length": 8.104166984558105, "epoch": 0.3446393762183236, "grad_norm": 5.376429802411146, "kl": 3.8671875, "learning_rate": 8.460328522455882e-07, "loss": 0.1342, "reward": 0.1472402662038803, "reward_std": 0.010309826582670212, "rewards/accuracy_reward": 0.1472402662038803, "step": 442 }, { "clip_ratio": 0.0, "completion_length": 7.75, "epoch": 0.3454191033138402, "grad_norm": 0.7375507125862572, "kl": 3.8203125, "learning_rate": 8.451083041066302e-07, "loss": 0.1534, "reward": 0.15899620950222015, "reward_std": 0.0, "rewards/accuracy_reward": 0.15899620950222015, "step": 443 }, { "clip_ratio": 0.0, "completion_length": 8.291666746139526, "epoch": 0.34619883040935673, "grad_norm": 2.58514884483921, "kl": 3.9609375, "learning_rate": 8.441815650689987e-07, "loss": 0.1441, "reward": 0.16193846613168716, "reward_std": 0.0006330604082904756, "rewards/accuracy_reward": 0.16193846613168716, "step": 444 }, { "clip_ratio": 0.0, "completion_length": 8.333333492279053, "epoch": 0.3469785575048733, "grad_norm": 3.865142309488586, "kl": 4.171875, "learning_rate": 8.432526420128497e-07, "loss": 0.1664, "reward": 0.16354405879974365, "reward_std": 0.017936360090970993, "rewards/accuracy_reward": 0.16354405879974365, "step": 445 }, { "clip_ratio": 0.0, "completion_length": 5.562500238418579, "epoch": 0.3477582846003899, "grad_norm": 0.5520638259162657, "kl": 3.875, "learning_rate": 8.423215418345544e-07, "loss": 0.1542, "reward": 0.14404762536287308, "reward_std": 0.0, "rewards/accuracy_reward": 0.14404762536287308, "step": 446 }, { "clip_ratio": 0.0, "completion_length": 6.291666746139526, "epoch": 0.3485380116959064, "grad_norm": 6.287980237166874, "kl": 4.046875, "learning_rate": 8.413882714466463e-07, "loss": 0.2107, "reward": 0.1572420671582222, "reward_std": 0.012028131633996964, "rewards/accuracy_reward": 0.1572420671582222, "step": 447 }, { "clip_ratio": 0.0, "completion_length": 8.458333730697632, "epoch": 0.349317738791423, "grad_norm": 4.546388481488884, "kl": 4.703125, "learning_rate": 8.404528377777714e-07, "loss": 0.2971, "reward": 0.18159721791744232, "reward_std": 0.016218727454543114, "rewards/accuracy_reward": 0.18159721791744232, "step": 448 }, { "clip_ratio": 0.0, "completion_length": 8.354166984558105, "epoch": 0.3500974658869396, "grad_norm": 3.3143977506986015, "kl": 3.4375, "learning_rate": 8.395152477726354e-07, "loss": 0.1129, "reward": 0.15585318207740784, "reward_std": 0.001093466067686677, "rewards/accuracy_reward": 0.15585318207740784, "step": 449 }, { "clip_ratio": 0.0, "completion_length": 11.3125, "epoch": 0.3508771929824561, "grad_norm": 0.3608819023159808, "kl": 3.28125, "learning_rate": 8.38575508391953e-07, "loss": 0.1247, "reward": 0.16003789007663727, "reward_std": 0.0, "rewards/accuracy_reward": 0.16003789007663727, "step": 450 }, { "clip_ratio": 0.0, "completion_length": 6.125000238418579, "epoch": 0.3516569200779727, "grad_norm": 0.6240970965730207, "kl": 3.546875, "learning_rate": 8.376336266123955e-07, "loss": 0.1415, "reward": 0.13809525966644287, "reward_std": 0.0, "rewards/accuracy_reward": 0.13809525966644287, "step": 451 }, { "clip_ratio": 0.0, "completion_length": 9.020833492279053, "epoch": 0.35243664717348927, "grad_norm": 3.8262236713205184, "kl": 3.53125, "learning_rate": 8.366896094265407e-07, "loss": 0.189, "reward": 0.1437252163887024, "reward_std": 0.0021478808484971523, "rewards/accuracy_reward": 0.1437252163887024, "step": 452 }, { "clip_ratio": 0.0, "completion_length": 6.604166746139526, "epoch": 0.3532163742690059, "grad_norm": 4.693590266385996, "kl": 4.2109375, "learning_rate": 8.357434638428183e-07, "loss": 0.312, "reward": 0.15640433132648468, "reward_std": 0.006682294886559248, "rewards/accuracy_reward": 0.15640433132648468, "step": 453 }, { "clip_ratio": 0.0, "completion_length": 6.1666669845581055, "epoch": 0.3539961013645224, "grad_norm": 16.412568739508906, "kl": 5.234375, "learning_rate": 8.347951968854605e-07, "loss": 0.293, "reward": 0.15729168057441711, "reward_std": 0.021650638431310654, "rewards/accuracy_reward": 0.15729168057441711, "step": 454 }, { "clip_ratio": 0.0, "completion_length": 9.270833492279053, "epoch": 0.35477582846003897, "grad_norm": 2.852199857180022, "kl": 3.890625, "learning_rate": 8.338448155944479e-07, "loss": 0.1697, "reward": 0.17543309926986694, "reward_std": 0.0007959796348586679, "rewards/accuracy_reward": 0.17543309926986694, "step": 455 }, { "clip_ratio": 0.0, "completion_length": 9.208333730697632, "epoch": 0.35555555555555557, "grad_norm": 2.5125320869917895, "kl": 3.796875, "learning_rate": 8.328923270254584e-07, "loss": 0.1117, "reward": 0.15251225978136063, "reward_std": 0.0009050846565514803, "rewards/accuracy_reward": 0.15251225978136063, "step": 456 }, { "clip_ratio": 0.0, "completion_length": 8.541666746139526, "epoch": 0.3563352826510721, "grad_norm": 5.639644968710531, "kl": 3.9921875, "learning_rate": 8.319377382498143e-07, "loss": 0.4313, "reward": 0.16532737761735916, "reward_std": 0.010309826582670212, "rewards/accuracy_reward": 0.16532737761735916, "step": 457 }, { "clip_ratio": 0.0, "completion_length": 6.645833492279053, "epoch": 0.3571150097465887, "grad_norm": 9.579854633126313, "kl": 4.140625, "learning_rate": 8.309810563544297e-07, "loss": 0.1893, "reward": 0.14931797236204147, "reward_std": 0.03522123023867607, "rewards/accuracy_reward": 0.14931797236204147, "step": 458 }, { "clip_ratio": 0.0, "completion_length": 7.125000238418579, "epoch": 0.35789473684210527, "grad_norm": 3.684533841377317, "kl": 3.7421875, "learning_rate": 8.300222884417591e-07, "loss": 0.278, "reward": 0.15312793850898743, "reward_std": 0.005660296883434057, "rewards/accuracy_reward": 0.15312793850898743, "step": 459 }, { "clip_ratio": 0.0, "completion_length": 7.6875, "epoch": 0.3586744639376218, "grad_norm": 6.299174771605969, "kl": 4.078125, "learning_rate": 8.290614416297422e-07, "loss": 0.2222, "reward": 0.14708946645259857, "reward_std": 0.0018490657676011324, "rewards/accuracy_reward": 0.14708946645259857, "step": 460 }, { "clip_ratio": 0.0, "completion_length": 7.583333492279053, "epoch": 0.3594541910331384, "grad_norm": 6.59923965919033, "kl": 3.59375, "learning_rate": 8.280985230517537e-07, "loss": 0.2232, "reward": 0.14995040744543076, "reward_std": 0.021541289053857327, "rewards/accuracy_reward": 0.14995040744543076, "step": 461 }, { "clip_ratio": 0.0, "completion_length": 6.3541669845581055, "epoch": 0.36023391812865496, "grad_norm": 0.5795825418946184, "kl": 4.1328125, "learning_rate": 8.271335398565493e-07, "loss": 0.1657, "reward": 0.16904763132333755, "reward_std": 0.0, "rewards/accuracy_reward": 0.16904763132333755, "step": 462 }, { "clip_ratio": 0.0, "completion_length": 7.104166746139526, "epoch": 0.36101364522417156, "grad_norm": 5.49279777276861, "kl": 4.234375, "learning_rate": 8.261664992082112e-07, "loss": 0.1771, "reward": 0.16036707162857056, "reward_std": 0.01297498308122158, "rewards/accuracy_reward": 0.16036707162857056, "step": 463 }, { "clip_ratio": 0.0, "completion_length": 10.166666746139526, "epoch": 0.3617933723196881, "grad_norm": 2.9309597239317005, "kl": 3.34375, "learning_rate": 8.251974082860976e-07, "loss": 0.141, "reward": 0.17147818207740784, "reward_std": 0.0047181760892271996, "rewards/accuracy_reward": 0.17147818207740784, "step": 464 }, { "clip_ratio": 0.0, "completion_length": 5.375000238418579, "epoch": 0.36257309941520466, "grad_norm": 8.422336053830872, "kl": 3.8046875, "learning_rate": 8.242262742847874e-07, "loss": 0.1985, "reward": 0.1394001916050911, "reward_std": 0.016099190339446068, "rewards/accuracy_reward": 0.1394001916050911, "step": 465 }, { "clip_ratio": 0.0, "completion_length": 6.958333492279053, "epoch": 0.36335282651072126, "grad_norm": 1.5131045081758705, "kl": 4.2265625, "learning_rate": 8.232531044140276e-07, "loss": 0.1676, "reward": 0.17425595223903656, "reward_std": 0.0, "rewards/accuracy_reward": 0.17425595223903656, "step": 466 }, { "clip_ratio": 0.0, "completion_length": 6.416666746139526, "epoch": 0.3641325536062378, "grad_norm": 10.85177258251265, "kl": 4.8203125, "learning_rate": 8.222779058986797e-07, "loss": 0.2036, "reward": 0.1517857238650322, "reward_std": 0.010309826582670212, "rewards/accuracy_reward": 0.1517857238650322, "step": 467 }, { "clip_ratio": 0.0, "completion_length": 5.854166746139526, "epoch": 0.3649122807017544, "grad_norm": 7.651454012467134, "kl": 3.5078125, "learning_rate": 8.213006859786654e-07, "loss": 0.2309, "reward": 0.14295635372400284, "reward_std": 0.012028131633996964, "rewards/accuracy_reward": 0.14295635372400284, "step": 468 }, { "clip_ratio": 0.0, "completion_length": 7.083333492279053, "epoch": 0.36569200779727096, "grad_norm": 3.810188977813238, "kl": 4.640625, "learning_rate": 8.203214519089142e-07, "loss": 0.3917, "reward": 0.16564542800188065, "reward_std": 0.003537686076015234, "rewards/accuracy_reward": 0.16564542800188065, "step": 469 }, { "clip_ratio": 0.0, "completion_length": 7.5, "epoch": 0.3664717348927875, "grad_norm": 0.7025200767559716, "kl": 3.8828125, "learning_rate": 8.193402109593081e-07, "loss": 0.1516, "reward": 0.14791666716337204, "reward_std": 0.0, "rewards/accuracy_reward": 0.14791666716337204, "step": 470 }, { "clip_ratio": 0.0, "completion_length": 6.250000238418579, "epoch": 0.3672514619883041, "grad_norm": 7.1769899639721775, "kl": 4.0, "learning_rate": 8.183569704146289e-07, "loss": 0.3655, "reward": 0.14375001192092896, "reward_std": 0.021650636568665504, "rewards/accuracy_reward": 0.14375001192092896, "step": 471 }, { "clip_ratio": 0.0, "completion_length": 6.520833492279053, "epoch": 0.36803118908382065, "grad_norm": 0.5211054823348614, "kl": 3.9609375, "learning_rate": 8.173717375745026e-07, "loss": 0.1568, "reward": 0.14925595372915268, "reward_std": 0.0, "rewards/accuracy_reward": 0.14925595372915268, "step": 472 }, { "clip_ratio": 0.0, "completion_length": 6.291666746139526, "epoch": 0.36881091617933726, "grad_norm": 3.6466347866363082, "kl": 4.21875, "learning_rate": 8.163845197533471e-07, "loss": 0.1828, "reward": 0.16442213207483292, "reward_std": 0.0051978714764118195, "rewards/accuracy_reward": 0.16442213207483292, "step": 473 }, { "clip_ratio": 0.0, "completion_length": 8.208333492279053, "epoch": 0.3695906432748538, "grad_norm": 5.619622481899318, "kl": 3.9453125, "learning_rate": 8.15395324280316e-07, "loss": 0.1728, "reward": 0.14863887429237366, "reward_std": 0.012499823234975338, "rewards/accuracy_reward": 0.14863887429237366, "step": 474 }, { "clip_ratio": 0.0, "completion_length": 5.791666746139526, "epoch": 0.37037037037037035, "grad_norm": 6.443185239282619, "kl": 4.5, "learning_rate": 8.144041584992458e-07, "loss": 0.1488, "reward": 0.15070685744285583, "reward_std": 0.02384147245902568, "rewards/accuracy_reward": 0.15070685744285583, "step": 475 }, { "clip_ratio": 0.0, "completion_length": 5.333333492279053, "epoch": 0.37115009746588695, "grad_norm": 0.4963476320879215, "kl": 3.765625, "learning_rate": 8.134110297686005e-07, "loss": 0.1503, "reward": 0.14404763281345367, "reward_std": 0.0, "rewards/accuracy_reward": 0.14404763281345367, "step": 476 }, { "clip_ratio": 0.0, "completion_length": 9.937500238418579, "epoch": 0.3719298245614035, "grad_norm": 3.881383212104769, "kl": 3.484375, "learning_rate": 8.124159454614167e-07, "loss": 0.1355, "reward": 0.14419206976890564, "reward_std": 0.0005306530511006713, "rewards/accuracy_reward": 0.14419206976890564, "step": 477 }, { "clip_ratio": 0.0, "completion_length": 8.437500476837158, "epoch": 0.3727095516569201, "grad_norm": 0.4735702922949389, "kl": 3.5234375, "learning_rate": 8.114189129652498e-07, "loss": 0.14, "reward": 0.16233766078948975, "reward_std": 0.0, "rewards/accuracy_reward": 0.16233766078948975, "step": 478 }, { "clip_ratio": 0.0, "completion_length": 7.312500476837158, "epoch": 0.37348927875243665, "grad_norm": 0.5317182633715427, "kl": 3.5625, "learning_rate": 8.104199396821183e-07, "loss": 0.1415, "reward": 0.14783550053834915, "reward_std": 0.0, "rewards/accuracy_reward": 0.14783550053834915, "step": 479 }, { "clip_ratio": 0.0, "completion_length": 6.895833492279053, "epoch": 0.3742690058479532, "grad_norm": 3.5235062812971973, "kl": 5.03125, "learning_rate": 8.094190330284497e-07, "loss": 0.2354, "reward": 0.17164523154497147, "reward_std": 0.0007959796348586679, "rewards/accuracy_reward": 0.17164523154497147, "step": 480 }, { "clip_ratio": 0.0, "completion_length": 5.458333492279053, "epoch": 0.3750487329434698, "grad_norm": 0.5615752188944567, "kl": 4.015625, "learning_rate": 8.084162004350239e-07, "loss": 0.1592, "reward": 0.160714291036129, "reward_std": 0.0, "rewards/accuracy_reward": 0.160714291036129, "step": 481 }, { "clip_ratio": 0.0, "completion_length": 6.416666746139526, "epoch": 0.37582846003898635, "grad_norm": 6.646761619006107, "kl": 3.8125, "learning_rate": 8.074114493469205e-07, "loss": 0.1806, "reward": 0.15319941192865372, "reward_std": 0.005412658676505089, "rewards/accuracy_reward": 0.15319941192865372, "step": 482 }, { "clip_ratio": 0.0, "completion_length": 9.104166746139526, "epoch": 0.37660818713450295, "grad_norm": 3.960634104851548, "kl": 3.84375, "learning_rate": 8.064047872234611e-07, "loss": 0.1371, "reward": 0.15019404888153076, "reward_std": 0.010153157985769212, "rewards/accuracy_reward": 0.15019404888153076, "step": 483 }, { "clip_ratio": 0.0, "completion_length": 8.145833492279053, "epoch": 0.3773879142300195, "grad_norm": 0.4723750019516396, "kl": 4.5625, "learning_rate": 8.053962215381551e-07, "loss": 0.1804, "reward": 0.18399621546268463, "reward_std": 0.0, "rewards/accuracy_reward": 0.18399621546268463, "step": 484 }, { "clip_ratio": 0.0, "completion_length": 6.583333492279053, "epoch": 0.37816764132553604, "grad_norm": 3.7740670405849204, "kl": 4.296875, "learning_rate": 8.043857597786442e-07, "loss": 0.1442, "reward": 0.17500000447034836, "reward_std": 0.013121597468852997, "rewards/accuracy_reward": 0.17500000447034836, "step": 485 }, { "clip_ratio": 0.0, "completion_length": 6.562500238418579, "epoch": 0.37894736842105264, "grad_norm": 0.29289778279495476, "kl": 3.703125, "learning_rate": 8.033734094466466e-07, "loss": 0.1486, "reward": 0.1454545557498932, "reward_std": 0.0, "rewards/accuracy_reward": 0.1454545557498932, "step": 486 }, { "clip_ratio": 0.0, "completion_length": 8.312500238418579, "epoch": 0.3797270955165692, "grad_norm": 3.492405176530902, "kl": 3.65625, "learning_rate": 8.023591780579013e-07, "loss": 0.2074, "reward": 0.1638144999742508, "reward_std": 0.0023821950890123844, "rewards/accuracy_reward": 0.1638144999742508, "step": 487 }, { "clip_ratio": 0.0, "completion_length": 6.187500238418579, "epoch": 0.3805068226120858, "grad_norm": 2.099727811298138, "kl": 4.1640625, "learning_rate": 8.013430731421125e-07, "loss": 0.1786, "reward": 0.16051588207483292, "reward_std": 0.000687321531586349, "rewards/accuracy_reward": 0.16051588207483292, "step": 488 }, { "clip_ratio": 0.0, "completion_length": 7.145833492279053, "epoch": 0.38128654970760234, "grad_norm": 0.37614889520365113, "kl": 3.515625, "learning_rate": 8.003251022428932e-07, "loss": 0.1406, "reward": 0.1454545557498932, "reward_std": 0.0, "rewards/accuracy_reward": 0.1454545557498932, "step": 489 }, { "clip_ratio": 0.0, "completion_length": 8.166666746139526, "epoch": 0.3820662768031189, "grad_norm": 3.2490831993692377, "kl": 3.859375, "learning_rate": 7.993052729177101e-07, "loss": 0.1166, "reward": 0.14361871033906937, "reward_std": 0.008988404646515846, "rewards/accuracy_reward": 0.14361871033906937, "step": 490 }, { "clip_ratio": 0.0, "completion_length": 8.083333492279053, "epoch": 0.3828460038986355, "grad_norm": 4.144005050441794, "kl": 3.84375, "learning_rate": 7.982835927378264e-07, "loss": 0.1758, "reward": 0.17722538113594055, "reward_std": 0.005412658676505089, "rewards/accuracy_reward": 0.17722538113594055, "step": 491 }, { "clip_ratio": 0.0, "completion_length": 9.354166984558105, "epoch": 0.38362573099415204, "grad_norm": 0.8134852287394632, "kl": 4.328125, "learning_rate": 7.972600692882467e-07, "loss": 0.1695, "reward": 0.16458334028720856, "reward_std": 0.0, "rewards/accuracy_reward": 0.16458334028720856, "step": 492 }, { "clip_ratio": 0.0, "completion_length": 6.479166746139526, "epoch": 0.38440545808966864, "grad_norm": 8.111287295998025, "kl": 3.734375, "learning_rate": 7.962347101676593e-07, "loss": 0.4144, "reward": 0.14890874922275543, "reward_std": 0.012028131633996964, "rewards/accuracy_reward": 0.14890874922275543, "step": 493 }, { "clip_ratio": 0.0, "completion_length": 8.770833492279053, "epoch": 0.3851851851851852, "grad_norm": 7.427994817914454, "kl": 4.1875, "learning_rate": 7.952075229883816e-07, "loss": 0.4443, "reward": 0.15552400052547455, "reward_std": 0.016583421966060996, "rewards/accuracy_reward": 0.15552400052547455, "step": 494 }, { "clip_ratio": 0.0, "completion_length": 6.625000238418579, "epoch": 0.38596491228070173, "grad_norm": 6.887978718238284, "kl": 4.15625, "learning_rate": 7.941785153763017e-07, "loss": 0.3331, "reward": 0.1592758148908615, "reward_std": 0.0024056262336671352, "rewards/accuracy_reward": 0.1592758148908615, "step": 495 }, { "clip_ratio": 0.0, "completion_length": 6.958333492279053, "epoch": 0.38674463937621834, "grad_norm": 6.2963934999274365, "kl": 4.15625, "learning_rate": 7.931476949708237e-07, "loss": 0.2142, "reward": 0.16041813790798187, "reward_std": 0.013024244457483292, "rewards/accuracy_reward": 0.16041813790798187, "step": 496 }, { "clip_ratio": 0.0, "completion_length": 8.979166984558105, "epoch": 0.3875243664717349, "grad_norm": 9.269295451472685, "kl": 4.421875, "learning_rate": 7.921150694248087e-07, "loss": 0.3698, "reward": 0.1551213338971138, "reward_std": 0.02220164705067873, "rewards/accuracy_reward": 0.1551213338971138, "step": 497 }, { "clip_ratio": 0.0, "completion_length": 6.479166746139526, "epoch": 0.3883040935672515, "grad_norm": 41.599638759436765, "kl": 5.4765625, "learning_rate": 7.910806464045205e-07, "loss": 0.3987, "reward": 0.12738095968961716, "reward_std": 0.015075568109750748, "rewards/accuracy_reward": 0.12738095968961716, "step": 498 }, { "clip_ratio": 0.0, "completion_length": 9.270833492279053, "epoch": 0.38908382066276803, "grad_norm": 0.4715560234291706, "kl": 3.046875, "learning_rate": 7.900444335895664e-07, "loss": 0.1183, "reward": 0.13601191341876984, "reward_std": 0.0, "rewards/accuracy_reward": 0.13601191341876984, "step": 499 }, { "clip_ratio": 0.0, "completion_length": 8.145833492279053, "epoch": 0.3898635477582846, "grad_norm": 0.45938131017775613, "kl": 3.296875, "learning_rate": 7.89006438672842e-07, "loss": 0.1291, "reward": 0.15854978561401367, "reward_std": 0.0, "rewards/accuracy_reward": 0.15854978561401367, "step": 500 }, { "clip_ratio": 0.0, "completion_length": 11.562500476837158, "epoch": 0.3906432748538012, "grad_norm": 1.832127531221603, "kl": 3.2109375, "learning_rate": 7.87966669360473e-07, "loss": 0.0843, "reward": 0.15031566470861435, "reward_std": 0.0048112524673342705, "rewards/accuracy_reward": 0.15031566470861435, "step": 501 }, { "clip_ratio": 0.0, "completion_length": 50.500001430511475, "epoch": 0.39142300194931773, "grad_norm": 19.32139593047235, "kl": 4.0078125, "learning_rate": 7.86925133371758e-07, "loss": 0.636, "reward": 0.13084637373685837, "reward_std": 0.025604712776839733, "rewards/accuracy_reward": 0.13084637373685837, "step": 502 }, { "clip_ratio": 0.0, "completion_length": 10.4375, "epoch": 0.39220272904483433, "grad_norm": 4.128289333676487, "kl": 3.9765625, "learning_rate": 7.858818384391121e-07, "loss": 0.3373, "reward": 0.15272794663906097, "reward_std": 0.010852573439478874, "rewards/accuracy_reward": 0.15272794663906097, "step": 503 }, { "clip_ratio": 0.0, "completion_length": 6.270833492279053, "epoch": 0.3929824561403509, "grad_norm": 6.89689577293208, "kl": 3.84375, "learning_rate": 7.848367923080083e-07, "loss": 0.3489, "reward": 0.14126984775066376, "reward_std": 0.009622504934668541, "rewards/accuracy_reward": 0.14126984775066376, "step": 504 }, { "clip_ratio": 0.0, "completion_length": 8.145833730697632, "epoch": 0.3937621832358674, "grad_norm": 10.45347074303406, "kl": 3.671875, "learning_rate": 7.837900027369213e-07, "loss": 0.5478, "reward": 0.1450827717781067, "reward_std": 0.021204533986747265, "rewards/accuracy_reward": 0.1450827717781067, "step": 505 }, { "clip_ratio": 0.0, "completion_length": 6.875, "epoch": 0.394541910331384, "grad_norm": 13.539747324408372, "kl": 4.109375, "learning_rate": 7.827414774972684e-07, "loss": 0.3884, "reward": 0.1531454399228096, "reward_std": 0.025450320914387703, "rewards/accuracy_reward": 0.1531454399228096, "step": 506 }, { "clip_ratio": 0.0, "completion_length": 6.270833492279053, "epoch": 0.3953216374269006, "grad_norm": 4.85728328238381, "kl": 4.0859375, "learning_rate": 7.816912243733529e-07, "loss": 0.2003, "reward": 0.15173611044883728, "reward_std": 0.012028131633996964, "rewards/accuracy_reward": 0.15173611044883728, "step": 507 }, { "clip_ratio": 0.0, "completion_length": 8.708333492279053, "epoch": 0.3961013645224172, "grad_norm": 10.250122131032365, "kl": 3.5390625, "learning_rate": 7.806392511623063e-07, "loss": 0.2273, "reward": 0.14918291568756104, "reward_std": 0.0038615207886323333, "rewards/accuracy_reward": 0.14918291568756104, "step": 508 }, { "clip_ratio": 0.0, "completion_length": 6.229166746139526, "epoch": 0.3968810916179337, "grad_norm": 0.5825117318442314, "kl": 3.9453125, "learning_rate": 7.795855656740297e-07, "loss": 0.1568, "reward": 0.16904762387275696, "reward_std": 0.0, "rewards/accuracy_reward": 0.16904762387275696, "step": 509 }, { "clip_ratio": 0.0, "completion_length": 7.395833492279053, "epoch": 0.39766081871345027, "grad_norm": 3.9864577401633303, "kl": 3.7578125, "learning_rate": 7.785301757311361e-07, "loss": 0.1404, "reward": 0.15848101675510406, "reward_std": 0.005111956037580967, "rewards/accuracy_reward": 0.15848101675510406, "step": 510 }, { "clip_ratio": 0.0, "completion_length": 8.291666984558105, "epoch": 0.3984405458089669, "grad_norm": 5.844626597048682, "kl": 4.53125, "learning_rate": 7.774730891688935e-07, "loss": 0.4012, "reward": 0.1799585521221161, "reward_std": 0.009113149717450142, "rewards/accuracy_reward": 0.1799585521221161, "step": 511 }, { "clip_ratio": 0.0, "completion_length": 7.020833492279053, "epoch": 0.3992202729044834, "grad_norm": 0.7290873142092051, "kl": 4.78125, "learning_rate": 7.764143138351641e-07, "loss": 0.1908, "reward": 0.171875, "reward_std": 0.0, "rewards/accuracy_reward": 0.171875, "step": 512 }, { "clip_ratio": 0.0, "completion_length": 6.020833492279053, "epoch": 0.4, "grad_norm": 0.4909406743346614, "kl": 3.578125, "learning_rate": 7.75353857590349e-07, "loss": 0.1434, "reward": 0.13333335518836975, "reward_std": 0.0, "rewards/accuracy_reward": 0.13333335518836975, "step": 513 }, { "clip_ratio": 0.0, "completion_length": 5.520833492279053, "epoch": 0.40077972709551657, "grad_norm": 0.5378890006390055, "kl": 4.578125, "learning_rate": 7.742917283073276e-07, "loss": 0.1841, "reward": 0.16354166716337204, "reward_std": 0.0, "rewards/accuracy_reward": 0.16354166716337204, "step": 514 }, { "clip_ratio": 0.0, "completion_length": 6.937500238418579, "epoch": 0.4015594541910331, "grad_norm": 0.4663833223109688, "kl": 4.375, "learning_rate": 7.732279338714004e-07, "loss": 0.1736, "reward": 0.17566289007663727, "reward_std": 0.0, "rewards/accuracy_reward": 0.17566289007663727, "step": 515 }, { "clip_ratio": 0.0, "completion_length": 6.270833492279053, "epoch": 0.4023391812865497, "grad_norm": 7.038585808964307, "kl": 4.515625, "learning_rate": 7.721624821802301e-07, "loss": 0.1654, "reward": 0.1465649977326393, "reward_std": 0.020147119648754597, "rewards/accuracy_reward": 0.1465649977326393, "step": 516 }, { "clip_ratio": 0.0, "completion_length": 6.062500238418579, "epoch": 0.40311890838206627, "grad_norm": 6.319549014406511, "kl": 3.828125, "learning_rate": 7.710953811437829e-07, "loss": 0.1979, "reward": 0.14126984775066376, "reward_std": 0.009622504934668541, "rewards/accuracy_reward": 0.14126984775066376, "step": 517 }, { "clip_ratio": 0.0, "completion_length": 7.687500238418579, "epoch": 0.40389863547758287, "grad_norm": 14.443799166033836, "kl": 6.484375, "learning_rate": 7.700266386842695e-07, "loss": 0.2401, "reward": 0.15711000561714172, "reward_std": 0.03797685354948044, "rewards/accuracy_reward": 0.15711000561714172, "step": 518 }, { "clip_ratio": 0.0, "completion_length": 8.145833492279053, "epoch": 0.4046783625730994, "grad_norm": 0.7291630360265167, "kl": 3.546875, "learning_rate": 7.689562627360871e-07, "loss": 0.1409, "reward": 0.16829004883766174, "reward_std": 0.0, "rewards/accuracy_reward": 0.16829004883766174, "step": 519 }, { "clip_ratio": 0.0, "completion_length": 6.854166746139526, "epoch": 0.40545808966861596, "grad_norm": 2.284150274568634, "kl": 4.0859375, "learning_rate": 7.678842612457598e-07, "loss": 0.1693, "reward": 0.15869100391864777, "reward_std": 0.004431416746228933, "rewards/accuracy_reward": 0.15869100391864777, "step": 520 }, { "clip_ratio": 0.0, "completion_length": 6.354166746139526, "epoch": 0.40623781676413256, "grad_norm": 22.641069404296434, "kl": 4.90625, "learning_rate": 7.668106421718798e-07, "loss": 0.337, "reward": 0.15684913843870163, "reward_std": 0.005141436355188489, "rewards/accuracy_reward": 0.15684913843870163, "step": 521 }, { "clip_ratio": 0.0, "completion_length": 6.166666746139526, "epoch": 0.4070175438596491, "grad_norm": 4.529722215791823, "kl": 3.8359375, "learning_rate": 7.657354134850481e-07, "loss": 0.1501, "reward": 0.13661955296993256, "reward_std": 0.005111956037580967, "rewards/accuracy_reward": 0.13661955296993256, "step": 522 }, { "clip_ratio": 0.0, "completion_length": 10.020833492279053, "epoch": 0.4077972709551657, "grad_norm": 3.5132587713319325, "kl": 3.078125, "learning_rate": 7.646585831678159e-07, "loss": 0.105, "reward": 0.1413690522313118, "reward_std": 0.010309826582670212, "rewards/accuracy_reward": 0.1413690522313118, "step": 523 }, { "clip_ratio": 0.0, "completion_length": 6.229166746139526, "epoch": 0.40857699805068226, "grad_norm": 6.427493426480892, "kl": 4.21875, "learning_rate": 7.635801592146249e-07, "loss": 0.1411, "reward": 0.15313854068517685, "reward_std": 0.011724278330802917, "rewards/accuracy_reward": 0.15313854068517685, "step": 524 }, { "clip_ratio": 0.0, "completion_length": 6.333333492279053, "epoch": 0.4093567251461988, "grad_norm": 5.535171825715842, "kl": 4.953125, "learning_rate": 7.625001496317478e-07, "loss": 0.1891, "reward": 0.15191420912742615, "reward_std": 0.00865181372500956, "rewards/accuracy_reward": 0.15191420912742615, "step": 525 }, { "clip_ratio": 0.0, "completion_length": 7.6666669845581055, "epoch": 0.4101364522417154, "grad_norm": 4.893961672897194, "kl": 3.71875, "learning_rate": 7.614185624372293e-07, "loss": 0.183, "reward": 0.16566459834575653, "reward_std": 0.007699065841734409, "rewards/accuracy_reward": 0.16566459834575653, "step": 526 }, { "clip_ratio": 0.0, "completion_length": 7.291666746139526, "epoch": 0.41091617933723196, "grad_norm": 0.6185057194613697, "kl": 4.171875, "learning_rate": 7.603354056608267e-07, "loss": 0.1639, "reward": 0.17425595223903656, "reward_std": 0.0, "rewards/accuracy_reward": 0.17425595223903656, "step": 527 }, { "clip_ratio": 0.0, "completion_length": 8.520833969116211, "epoch": 0.41169590643274856, "grad_norm": 682.0184211954377, "kl": 13.2734375, "learning_rate": 7.592506873439491e-07, "loss": 2.432, "reward": 0.14337120950222015, "reward_std": 0.007216879166662693, "rewards/accuracy_reward": 0.14337120950222015, "step": 528 }, { "clip_ratio": 0.0, "completion_length": 7.250000238418579, "epoch": 0.4124756335282651, "grad_norm": 3555.1028884450907, "kl": 181.03125, "learning_rate": 7.58164415539599e-07, "loss": 7.6453, "reward": 0.15654495358467102, "reward_std": 0.020854391157627106, "rewards/accuracy_reward": 0.15654495358467102, "step": 529 }, { "clip_ratio": 0.0, "completion_length": 5.645833492279053, "epoch": 0.41325536062378165, "grad_norm": 0.6055506502939557, "kl": 4.328125, "learning_rate": 7.570765983123125e-07, "loss": 0.1733, "reward": 0.1666666716337204, "reward_std": 0.0, "rewards/accuracy_reward": 0.1666666716337204, "step": 530 }, { "clip_ratio": 0.0, "completion_length": 5.666666746139526, "epoch": 0.41403508771929826, "grad_norm": 0.690057128263379, "kl": 4.359375, "learning_rate": 7.559872437380985e-07, "loss": 0.1731, "reward": 0.17500000447034836, "reward_std": 0.0, "rewards/accuracy_reward": 0.17500000447034836, "step": 531 }, { "clip_ratio": 0.0, "completion_length": 8.687500238418579, "epoch": 0.4148148148148148, "grad_norm": 1315.802458090792, "kl": 61.703125, "learning_rate": 7.548963599043792e-07, "loss": 3.0454, "reward": 0.17348606884479523, "reward_std": 0.018366033560596406, "rewards/accuracy_reward": 0.17348606884479523, "step": 532 }, { "clip_ratio": 0.0, "completion_length": 7.958333730697632, "epoch": 0.4155945419103314, "grad_norm": 7.708914928216294, "kl": 4.078125, "learning_rate": 7.538039549099302e-07, "loss": 0.2319, "reward": 0.14933712780475616, "reward_std": 0.016966513358056545, "rewards/accuracy_reward": 0.14933712780475616, "step": 533 }, { "clip_ratio": 0.0, "completion_length": 11.6875, "epoch": 0.41637426900584795, "grad_norm": 2.582191201975053, "kl": 2.7734375, "learning_rate": 7.527100368648205e-07, "loss": 0.1156, "reward": 0.1288398653268814, "reward_std": 0.001132059609517455, "rewards/accuracy_reward": 0.1288398653268814, "step": 534 }, { "clip_ratio": 0.0, "completion_length": 7.437500238418579, "epoch": 0.4171539961013645, "grad_norm": 7.689630134942464, "kl": 3.5703125, "learning_rate": 7.516146138903518e-07, "loss": 0.2095, "reward": 0.16102994978427887, "reward_std": 0.012028131633996964, "rewards/accuracy_reward": 0.16102994978427887, "step": 535 }, { "clip_ratio": 0.0, "completion_length": 5.833333492279053, "epoch": 0.4179337231968811, "grad_norm": 6.255243670074004, "kl": 4.25, "learning_rate": 7.505176941189986e-07, "loss": 0.2267, "reward": 0.1492311656475067, "reward_std": 0.010911233723163605, "rewards/accuracy_reward": 0.1492311656475067, "step": 536 }, { "clip_ratio": 0.0, "completion_length": 5.750000238418579, "epoch": 0.41871345029239765, "grad_norm": 4.185175126819574, "kl": 4.2578125, "learning_rate": 7.494192856943475e-07, "loss": 0.1593, "reward": 0.1481274887919426, "reward_std": 0.010477647185325623, "rewards/accuracy_reward": 0.1481274887919426, "step": 537 }, { "clip_ratio": 0.0, "completion_length": 7.333333492279053, "epoch": 0.41949317738791425, "grad_norm": 6.808007850322068, "kl": 3.4609375, "learning_rate": 7.483193967710372e-07, "loss": 0.3202, "reward": 0.1556977480649948, "reward_std": 0.011876894859597087, "rewards/accuracy_reward": 0.1556977480649948, "step": 538 }, { "clip_ratio": 0.0, "completion_length": 6.458333492279053, "epoch": 0.4202729044834308, "grad_norm": 86.26158691980832, "kl": 7.875, "learning_rate": 7.472180355146975e-07, "loss": 0.4188, "reward": 0.14704862236976624, "reward_std": 0.01655529160052538, "rewards/accuracy_reward": 0.14704862236976624, "step": 539 }, { "clip_ratio": 0.0, "completion_length": 6.5, "epoch": 0.42105263157894735, "grad_norm": 4.266988022777769, "kl": 3.875, "learning_rate": 7.461152101018888e-07, "loss": 0.1728, "reward": 0.14872636646032333, "reward_std": 0.00679755536839366, "rewards/accuracy_reward": 0.14872636646032333, "step": 540 }, { "clip_ratio": 0.0, "completion_length": 6.125000238418579, "epoch": 0.42183235867446395, "grad_norm": 9.790453302000945, "kl": 4.6796875, "learning_rate": 7.45010928720042e-07, "loss": 0.1953, "reward": 0.13483188301324844, "reward_std": 0.023676427081227303, "rewards/accuracy_reward": 0.13483188301324844, "step": 541 }, { "clip_ratio": 0.0, "completion_length": 5.75, "epoch": 0.4226120857699805, "grad_norm": 0.7638438910268821, "kl": 3.734375, "learning_rate": 7.439051995673965e-07, "loss": 0.1497, "reward": 0.14404762536287308, "reward_std": 0.0, "rewards/accuracy_reward": 0.14404762536287308, "step": 542 }, { "clip_ratio": 0.0, "completion_length": 7.3125, "epoch": 0.4233918128654971, "grad_norm": 6.076664081585385, "kl": 3.8359375, "learning_rate": 7.427980308529404e-07, "loss": 0.1624, "reward": 0.15981241315603256, "reward_std": 0.003203280735760927, "rewards/accuracy_reward": 0.15981241315603256, "step": 543 }, { "clip_ratio": 0.0, "completion_length": 5.8125, "epoch": 0.42417153996101364, "grad_norm": 6.954069996234832, "kl": 3.7265625, "learning_rate": 7.416894307963492e-07, "loss": 0.1379, "reward": 0.14126984775066376, "reward_std": 0.009622504934668541, "rewards/accuracy_reward": 0.14126984775066376, "step": 544 }, { "clip_ratio": 0.0, "completion_length": 7.708333492279053, "epoch": 0.4249512670565302, "grad_norm": 10.959691309789722, "kl": 3.75, "learning_rate": 7.405794076279245e-07, "loss": 0.18, "reward": 0.1448412761092186, "reward_std": 0.021244490519165993, "rewards/accuracy_reward": 0.1448412761092186, "step": 545 }, { "clip_ratio": 0.0, "completion_length": 9.208333492279053, "epoch": 0.4257309941520468, "grad_norm": 15.575307553434769, "kl": 4.359375, "learning_rate": 7.394679695885333e-07, "loss": 0.1878, "reward": 0.18217329680919647, "reward_std": 0.010592571459710598, "rewards/accuracy_reward": 0.18217329680919647, "step": 546 }, { "clip_ratio": 0.0, "completion_length": 6.562500238418579, "epoch": 0.42651072124756334, "grad_norm": 8.815329392247659, "kl": 3.8125, "learning_rate": 7.38355124929547e-07, "loss": 0.1348, "reward": 0.15213586390018463, "reward_std": 0.007432721555233002, "rewards/accuracy_reward": 0.15213586390018463, "step": 547 }, { "clip_ratio": 0.0, "completion_length": 6.479166746139526, "epoch": 0.42729044834307994, "grad_norm": 32.96098064687768, "kl": 6.4453125, "learning_rate": 7.372408819127791e-07, "loss": 0.229, "reward": 0.1510416641831398, "reward_std": 0.010309826582670212, "rewards/accuracy_reward": 0.1510416641831398, "step": 548 }, { "clip_ratio": 0.0, "completion_length": 16.333333492279053, "epoch": 0.4280701754385965, "grad_norm": 11.840648605106866, "kl": 3.015625, "learning_rate": 7.361252488104251e-07, "loss": 0.2657, "reward": 0.17088532447814941, "reward_std": 0.005972900427877903, "rewards/accuracy_reward": 0.17088532447814941, "step": 549 }, { "clip_ratio": 0.0, "completion_length": 6.229166746139526, "epoch": 0.42884990253411304, "grad_norm": 6.800456709016594, "kl": 3.9296875, "learning_rate": 7.350082339050006e-07, "loss": 0.1374, "reward": 0.1436508148908615, "reward_std": 0.009622504934668541, "rewards/accuracy_reward": 0.1436508148908615, "step": 550 }, { "clip_ratio": 0.0, "completion_length": 9.875000238418579, "epoch": 0.42962962962962964, "grad_norm": 8.290262336360817, "kl": 3.40625, "learning_rate": 7.338898454892794e-07, "loss": 0.1172, "reward": 0.1795128434896469, "reward_std": 0.003976011881604791, "rewards/accuracy_reward": 0.1795128434896469, "step": 551 }, { "clip_ratio": 0.0, "completion_length": 5.416666746139526, "epoch": 0.4304093567251462, "grad_norm": 1.8505175096133353, "kl": 3.765625, "learning_rate": 7.327700918662325e-07, "loss": 0.15, "reward": 0.1460784375667572, "reward_std": 0.0, "rewards/accuracy_reward": 0.1460784375667572, "step": 552 }, { "clip_ratio": 0.0, "completion_length": 8.75, "epoch": 0.4311890838206628, "grad_norm": 30.438136118565836, "kl": 4.796875, "learning_rate": 7.316489813489666e-07, "loss": 0.4529, "reward": 0.16495657712221146, "reward_std": 0.01821788214147091, "rewards/accuracy_reward": 0.16495657712221146, "step": 553 }, { "clip_ratio": 0.0, "completion_length": 6.354166746139526, "epoch": 0.43196881091617934, "grad_norm": 9.017816425897559, "kl": 3.6328125, "learning_rate": 7.305265222606613e-07, "loss": 0.1944, "reward": 0.15818039327859879, "reward_std": 0.006200214847922325, "rewards/accuracy_reward": 0.15818039327859879, "step": 554 }, { "clip_ratio": 0.0, "completion_length": 5.395833492279053, "epoch": 0.4327485380116959, "grad_norm": 106.8464704607081, "kl": 7.359375, "learning_rate": 7.294027229345089e-07, "loss": 0.3112, "reward": 0.15555556118488312, "reward_std": 0.009622504934668541, "rewards/accuracy_reward": 0.15555556118488312, "step": 555 }, { "clip_ratio": 0.0, "completion_length": 5.625, "epoch": 0.4335282651072125, "grad_norm": 2.5148575588241613, "kl": 3.3125, "learning_rate": 7.28277591713651e-07, "loss": 0.1323, "reward": 0.1547619178891182, "reward_std": 0.0, "rewards/accuracy_reward": 0.1547619178891182, "step": 556 }, { "clip_ratio": 0.0, "completion_length": 6.187500238418579, "epoch": 0.43430799220272903, "grad_norm": 7.805638355295732, "kl": 3.4375, "learning_rate": 7.271511369511177e-07, "loss": 0.1802, "reward": 0.16076389700174332, "reward_std": 0.009622504934668541, "rewards/accuracy_reward": 0.16076389700174332, "step": 557 }, { "clip_ratio": 0.0, "completion_length": 10.166666984558105, "epoch": 0.43508771929824563, "grad_norm": 9.47863228334864, "kl": 2.6015625, "learning_rate": 7.26023367009765e-07, "loss": 0.6941, "reward": 0.14044123888015747, "reward_std": 0.01778208464384079, "rewards/accuracy_reward": 0.14044123888015747, "step": 558 }, { "clip_ratio": 0.0, "completion_length": 7.125, "epoch": 0.4358674463937622, "grad_norm": 8.557694457533575, "kl": 4.609375, "learning_rate": 7.248942902622126e-07, "loss": 0.1994, "reward": 0.1728670671582222, "reward_std": 0.0048112524673342705, "rewards/accuracy_reward": 0.1728670671582222, "step": 559 }, { "clip_ratio": 0.0, "completion_length": 9.354166984558105, "epoch": 0.43664717348927873, "grad_norm": 71.43433571432669, "kl": 5.3984375, "learning_rate": 7.237639150907825e-07, "loss": 0.509, "reward": 0.1398187130689621, "reward_std": 0.007432510843500495, "rewards/accuracy_reward": 0.1398187130689621, "step": 560 }, { "clip_ratio": 0.0, "completion_length": 6.250000238418579, "epoch": 0.43742690058479533, "grad_norm": 2.662085684698807, "kl": 2.7109375, "learning_rate": 7.226322498874357e-07, "loss": 0.1075, "reward": 0.15758929401636124, "reward_std": 0.0, "rewards/accuracy_reward": 0.15758929401636124, "step": 561 }, { "clip_ratio": 0.0, "completion_length": 13.583333730697632, "epoch": 0.4382066276803119, "grad_norm": 10.278524666464056, "kl": 2.5078125, "learning_rate": 7.214993030537111e-07, "loss": 0.2708, "reward": 0.14937517046928406, "reward_std": 0.021752064116299152, "rewards/accuracy_reward": 0.14937517046928406, "step": 562 }, { "clip_ratio": 0.0, "completion_length": 6.75, "epoch": 0.4389863547758285, "grad_norm": 10.598618560101574, "kl": 3.015625, "learning_rate": 7.203650830006619e-07, "loss": 0.1615, "reward": 0.17000534385442734, "reward_std": 0.0064766849391162395, "rewards/accuracy_reward": 0.17000534385442734, "step": 563 }, { "clip_ratio": 0.0, "completion_length": 5.916666746139526, "epoch": 0.439766081871345, "grad_norm": 18.706992459057485, "kl": 3.6875, "learning_rate": 7.19229598148794e-07, "loss": 0.1516, "reward": 0.14305556565523148, "reward_std": 0.024056261405348778, "rewards/accuracy_reward": 0.14305556565523148, "step": 564 }, { "clip_ratio": 0.0, "completion_length": 5.854166746139526, "epoch": 0.4405458089668616, "grad_norm": 42.33722894553616, "kl": 4.21875, "learning_rate": 7.180928569280033e-07, "loss": 0.1635, "reward": 0.14345239102840424, "reward_std": 0.010309826582670212, "rewards/accuracy_reward": 0.14345239102840424, "step": 565 }, { "clip_ratio": 0.0, "completion_length": 6.187500238418579, "epoch": 0.4413255360623782, "grad_norm": 9.503215781168503, "kl": 1.68359375, "learning_rate": 7.169548677775129e-07, "loss": 0.114, "reward": 0.14387401193380356, "reward_std": 0.0006014079554006457, "rewards/accuracy_reward": 0.14387401193380356, "step": 566 }, { "clip_ratio": 0.0, "completion_length": 5.666666746139526, "epoch": 0.4421052631578947, "grad_norm": 31.061007770377056, "kl": 2.984375, "learning_rate": 7.158156391458107e-07, "loss": 0.1577, "reward": 0.14574654400348663, "reward_std": 0.014734460972249508, "rewards/accuracy_reward": 0.14574654400348663, "step": 567 }, { "clip_ratio": 0.0, "completion_length": 6.75, "epoch": 0.4428849902534113, "grad_norm": 12.502539514434291, "kl": 3.046875, "learning_rate": 7.146751794905864e-07, "loss": 0.2006, "reward": 0.17068368196487427, "reward_std": 0.004126847488805652, "rewards/accuracy_reward": 0.17068368196487427, "step": 568 }, { "clip_ratio": 0.0, "completion_length": 7.645833492279053, "epoch": 0.4436647173489279, "grad_norm": 36.17713421370529, "kl": 5.390625, "learning_rate": 7.13533497278669e-07, "loss": 0.3954, "reward": 0.17440824210643768, "reward_std": 0.005313411355018616, "rewards/accuracy_reward": 0.17440824210643768, "step": 569 }, { "clip_ratio": 0.0, "completion_length": 11.437500476837158, "epoch": 0.4444444444444444, "grad_norm": 384.8178421954848, "kl": 24.453125, "learning_rate": 7.12390600985964e-07, "loss": 1.1603, "reward": 0.16798757761716843, "reward_std": 0.027818378526717424, "rewards/accuracy_reward": 0.16798757761716843, "step": 570 }, { "clip_ratio": 0.0, "completion_length": 7.833333492279053, "epoch": 0.445224171539961, "grad_norm": 2.4800641018854104, "kl": 1.74609375, "learning_rate": 7.112464990973899e-07, "loss": 0.0703, "reward": 0.14196428656578064, "reward_std": 0.0, "rewards/accuracy_reward": 0.14196428656578064, "step": 571 }, { "clip_ratio": 0.0, "completion_length": 6.520833492279053, "epoch": 0.44600389863547757, "grad_norm": 8.462434420586678, "kl": 2.2421875, "learning_rate": 7.101012001068159e-07, "loss": 0.3008, "reward": 0.1650463044643402, "reward_std": 0.005613128654658794, "rewards/accuracy_reward": 0.1650463044643402, "step": 572 }, { "clip_ratio": 0.0, "completion_length": 6.770833492279053, "epoch": 0.44678362573099417, "grad_norm": 19.51349408685107, "kl": 4.859375, "learning_rate": 7.089547125169986e-07, "loss": 0.1691, "reward": 0.15353422611951828, "reward_std": 0.011175260122399777, "rewards/accuracy_reward": 0.15353422611951828, "step": 573 }, { "clip_ratio": 0.0, "completion_length": 7.0416669845581055, "epoch": 0.4475633528265107, "grad_norm": 17.078049285990165, "kl": 5.703125, "learning_rate": 7.078070448395183e-07, "loss": 0.1995, "reward": 0.1815476268529892, "reward_std": 0.014433758333325386, "rewards/accuracy_reward": 0.1815476268529892, "step": 574 }, { "clip_ratio": 0.0, "completion_length": 8.541666984558105, "epoch": 0.44834307992202727, "grad_norm": 46.846754836989255, "kl": 5.56640625, "learning_rate": 7.066582055947172e-07, "loss": 0.216, "reward": 0.14058562368154526, "reward_std": 0.004775875713676214, "rewards/accuracy_reward": 0.14058562368154526, "step": 575 }, { "clip_ratio": 0.0, "completion_length": 6.583333492279053, "epoch": 0.44912280701754387, "grad_norm": 997.5685150926331, "kl": 28.6015625, "learning_rate": 7.055082033116345e-07, "loss": 2.3898, "reward": 0.15674273669719696, "reward_std": 0.0022111828438937664, "rewards/accuracy_reward": 0.15674273669719696, "step": 576 }, { "clip_ratio": 0.0, "completion_length": 10.979166746139526, "epoch": 0.4499025341130604, "grad_norm": 20.967147453075277, "kl": 3.1484375, "learning_rate": 7.043570465279441e-07, "loss": 0.5456, "reward": 0.16420137137174606, "reward_std": 0.017183993943035603, "rewards/accuracy_reward": 0.16420137137174606, "step": 577 }, { "clip_ratio": 0.0, "completion_length": 6.708333492279053, "epoch": 0.450682261208577, "grad_norm": 93.87789381256485, "kl": 5.56640625, "learning_rate": 7.032047437898911e-07, "loss": 0.655, "reward": 0.14348546415567398, "reward_std": 0.010195272974669933, "rewards/accuracy_reward": 0.14348546415567398, "step": 578 }, { "clip_ratio": 0.0, "completion_length": 12.104166984558105, "epoch": 0.45146198830409356, "grad_norm": 20.936925012062346, "kl": 2.984375, "learning_rate": 7.020513036522282e-07, "loss": 0.3587, "reward": 0.154551699757576, "reward_std": 0.008684921078383923, "rewards/accuracy_reward": 0.154551699757576, "step": 579 }, { "clip_ratio": 0.0, "completion_length": 8.479166984558105, "epoch": 0.4522417153996101, "grad_norm": 32.07952155390828, "kl": 6.1171875, "learning_rate": 7.00896734678152e-07, "loss": 0.2775, "reward": 0.15596140176057816, "reward_std": 0.0150727485306561, "rewards/accuracy_reward": 0.15596140176057816, "step": 580 }, { "clip_ratio": 0.0, "completion_length": 6.520833492279053, "epoch": 0.4530214424951267, "grad_norm": 10.493960694464665, "kl": 3.1015625, "learning_rate": 6.997410454392399e-07, "loss": 0.1336, "reward": 0.15995670855045319, "reward_std": 0.001769316615536809, "rewards/accuracy_reward": 0.15995670855045319, "step": 581 }, { "clip_ratio": 0.0, "completion_length": 9.854166984558105, "epoch": 0.45380116959064326, "grad_norm": 17.20521639371433, "kl": 3.21875, "learning_rate": 6.98584244515386e-07, "loss": 0.4798, "reward": 0.1522551327943802, "reward_std": 0.03442382253706455, "rewards/accuracy_reward": 0.1522551327943802, "step": 582 }, { "clip_ratio": 0.0, "completion_length": 11.145833492279053, "epoch": 0.45458089668615986, "grad_norm": 18.88129187265365, "kl": 3.8984375, "learning_rate": 6.974263404947377e-07, "loss": 0.5307, "reward": 0.17063726484775543, "reward_std": 0.027801679214462638, "rewards/accuracy_reward": 0.17063726484775543, "step": 583 }, { "clip_ratio": 0.0, "completion_length": 7.125000238418579, "epoch": 0.4553606237816764, "grad_norm": 5.00385108032432, "kl": 3.1875, "learning_rate": 6.962673419736318e-07, "loss": 0.1553, "reward": 0.1709383875131607, "reward_std": 0.0016980890650302172, "rewards/accuracy_reward": 0.1709383875131607, "step": 584 }, { "clip_ratio": 0.0, "completion_length": 7.4166669845581055, "epoch": 0.45614035087719296, "grad_norm": 14.437853590371247, "kl": 3.015625, "learning_rate": 6.951072575565308e-07, "loss": 0.2736, "reward": 0.14713334292173386, "reward_std": 0.015151983126997948, "rewards/accuracy_reward": 0.14713334292173386, "step": 585 }, { "clip_ratio": 0.0, "completion_length": 6.729166746139526, "epoch": 0.45692007797270956, "grad_norm": 6.579632822723212, "kl": 3.2890625, "learning_rate": 6.939460958559588e-07, "loss": 0.1675, "reward": 0.15719697624444962, "reward_std": 0.0020557595416903496, "rewards/accuracy_reward": 0.15719697624444962, "step": 586 }, { "clip_ratio": 0.0, "completion_length": 6.958333492279053, "epoch": 0.4576998050682261, "grad_norm": 7.143676976157255, "kl": 3.4453125, "learning_rate": 6.92783865492438e-07, "loss": 0.2347, "reward": 0.17388392984867096, "reward_std": 0.0012887290213257074, "rewards/accuracy_reward": 0.17388392984867096, "step": 587 }, { "clip_ratio": 0.0, "completion_length": 9.666666984558105, "epoch": 0.4584795321637427, "grad_norm": 1.856929945772351, "kl": 3.40625, "learning_rate": 6.916205750944241e-07, "loss": 0.1325, "reward": 0.16383928060531616, "reward_std": 0.0, "rewards/accuracy_reward": 0.16383928060531616, "step": 588 }, { "clip_ratio": 0.0, "completion_length": 6.354166746139526, "epoch": 0.45925925925925926, "grad_norm": 2.995201588788458, "kl": 2.52734375, "learning_rate": 6.904562332982428e-07, "loss": 0.1025, "reward": 0.14925595372915268, "reward_std": 0.0, "rewards/accuracy_reward": 0.14925595372915268, "step": 589 }, { "clip_ratio": 0.0, "completion_length": 6.354166746139526, "epoch": 0.4600389863547758, "grad_norm": 6.372953796870178, "kl": 2.20703125, "learning_rate": 6.892908487480251e-07, "loss": 0.1526, "reward": 0.1655934453010559, "reward_std": 0.0026334160938858986, "rewards/accuracy_reward": 0.1655934453010559, "step": 590 }, { "clip_ratio": 0.0, "completion_length": 5.875, "epoch": 0.4608187134502924, "grad_norm": 79.3770471401924, "kl": 4.330078125, "learning_rate": 6.881244300956435e-07, "loss": 0.3731, "reward": 0.1313258744776249, "reward_std": 0.0016172275645658374, "rewards/accuracy_reward": 0.1313258744776249, "step": 591 }, { "clip_ratio": 0.0, "completion_length": 6.416666746139526, "epoch": 0.46159844054580895, "grad_norm": 20.50863726574966, "kl": 3.5703125, "learning_rate": 6.869569860006482e-07, "loss": 0.1614, "reward": 0.16135913878679276, "reward_std": 0.013928290456533432, "rewards/accuracy_reward": 0.16135913878679276, "step": 592 }, { "clip_ratio": 0.0, "completion_length": 6.541666746139526, "epoch": 0.46237816764132555, "grad_norm": 7.188915830201685, "kl": 2.8125, "learning_rate": 6.85788525130202e-07, "loss": 0.1639, "reward": 0.1533953994512558, "reward_std": 0.0006360284751281142, "rewards/accuracy_reward": 0.1533953994512558, "step": 593 }, { "clip_ratio": 0.0, "completion_length": 6.937500238418579, "epoch": 0.4631578947368421, "grad_norm": 198.12095002925503, "kl": 6.171875, "learning_rate": 6.846190561590163e-07, "loss": 0.6569, "reward": 0.14233194291591644, "reward_std": 0.001819381257519126, "rewards/accuracy_reward": 0.14233194291591644, "step": 594 }, { "clip_ratio": 0.0, "completion_length": 6.479166746139526, "epoch": 0.46393762183235865, "grad_norm": 9.851311899725188, "kl": 2.3359375, "learning_rate": 6.834485877692865e-07, "loss": 0.0951, "reward": 0.1607142984867096, "reward_std": 0.0, "rewards/accuracy_reward": 0.1607142984867096, "step": 595 }, { "clip_ratio": 0.0, "completion_length": 6.270833492279053, "epoch": 0.46471734892787525, "grad_norm": 23.489094889453526, "kl": 3.25, "learning_rate": 6.82277128650628e-07, "loss": 0.1268, "reward": 0.1563674435019493, "reward_std": 0.024852244183421135, "rewards/accuracy_reward": 0.1563674435019493, "step": 596 }, { "clip_ratio": 0.0, "completion_length": 7.666666746139526, "epoch": 0.4654970760233918, "grad_norm": 1103.0842668510945, "kl": 298.046875, "learning_rate": 6.811046875000114e-07, "loss": 4.6009, "reward": 0.1563657447695732, "reward_std": 0.013424476608633995, "rewards/accuracy_reward": 0.1563657447695732, "step": 597 }, { "clip_ratio": 0.0, "completion_length": 6.125, "epoch": 0.4662768031189084, "grad_norm": 8.04656294897895, "kl": 1.009765625, "learning_rate": 6.799312730216979e-07, "loss": 0.0409, "reward": 0.13412699848413467, "reward_std": 0.005498574580997229, "rewards/accuracy_reward": 0.13412699848413467, "step": 598 }, { "clip_ratio": 0.0, "completion_length": 7.583333492279053, "epoch": 0.46705653021442495, "grad_norm": 5.060463177505536, "kl": 3.3203125, "learning_rate": 6.787568939271746e-07, "loss": 0.1351, "reward": 0.18333334475755692, "reward_std": 0.0, "rewards/accuracy_reward": 0.18333334475755692, "step": 599 }, { "clip_ratio": 0.0, "completion_length": 7.0625, "epoch": 0.4678362573099415, "grad_norm": 60.322601083901915, "kl": 5.0, "learning_rate": 6.775815589350904e-07, "loss": 0.4171, "reward": 0.16909722983837128, "reward_std": 0.009622504934668541, "rewards/accuracy_reward": 0.16909722983837128, "step": 600 }, { "clip_ratio": 0.0, "completion_length": 8.645833730697632, "epoch": 0.4686159844054581, "grad_norm": 44.481275386553236, "kl": 4.5625, "learning_rate": 6.764052767711903e-07, "loss": 0.5164, "reward": 0.15505515038967133, "reward_std": 0.01400415413081646, "rewards/accuracy_reward": 0.15505515038967133, "step": 601 }, { "clip_ratio": 0.0, "completion_length": 6.479166746139526, "epoch": 0.46939571150097464, "grad_norm": 8.632585486112204, "kl": 1.859375, "learning_rate": 6.752280561682514e-07, "loss": 0.0929, "reward": 0.16639957576990128, "reward_std": 0.0009252413874492049, "rewards/accuracy_reward": 0.16639957576990128, "step": 602 }, { "clip_ratio": 0.0, "completion_length": 10.020833492279053, "epoch": 0.47017543859649125, "grad_norm": 12.620593909317027, "kl": 2.1484375, "learning_rate": 6.740499058660177e-07, "loss": 0.7253, "reward": 0.15492628514766693, "reward_std": 0.023291989229619503, "rewards/accuracy_reward": 0.15492628514766693, "step": 603 }, { "clip_ratio": 0.0, "completion_length": 6.916666746139526, "epoch": 0.4709551656920078, "grad_norm": 6.023948645936651, "kl": 3.5, "learning_rate": 6.728708346111358e-07, "loss": 0.1402, "reward": 0.17861690372228622, "reward_std": 0.005512893199920654, "rewards/accuracy_reward": 0.17861690372228622, "step": 604 }, { "clip_ratio": 0.0, "completion_length": 6.750000238418579, "epoch": 0.47173489278752434, "grad_norm": 5.879352808794209, "kl": 2.7421875, "learning_rate": 6.716908511570886e-07, "loss": 0.132, "reward": 0.1728006899356842, "reward_std": 0.005041202064603567, "rewards/accuracy_reward": 0.1728006899356842, "step": 605 }, { "clip_ratio": 0.0, "completion_length": 5.687500238418579, "epoch": 0.47251461988304094, "grad_norm": 7.675213068398439, "kl": 1.48046875, "learning_rate": 6.705099642641321e-07, "loss": 0.1459, "reward": 0.1426633819937706, "reward_std": 0.0031509576365351677, "rewards/accuracy_reward": 0.1426633819937706, "step": 606 }, { "clip_ratio": 0.0, "completion_length": 6.4375, "epoch": 0.4732943469785575, "grad_norm": 18.023879241016054, "kl": 2.6953125, "learning_rate": 6.693281826992291e-07, "loss": 0.1586, "reward": 0.13234128057956696, "reward_std": 0.019932333379983902, "rewards/accuracy_reward": 0.13234128057956696, "step": 607 }, { "clip_ratio": 0.0, "completion_length": 6.1041669845581055, "epoch": 0.4740740740740741, "grad_norm": 5.748733933006918, "kl": 1.7890625, "learning_rate": 6.681455152359844e-07, "loss": 0.0642, "reward": 0.16458334773778915, "reward_std": 0.005154913291335106, "rewards/accuracy_reward": 0.16458334773778915, "step": 608 }, { "clip_ratio": 0.0, "completion_length": 6.458333492279053, "epoch": 0.47485380116959064, "grad_norm": 9.9287640933695, "kl": 1.640625, "learning_rate": 6.669619706545797e-07, "loss": 0.0755, "reward": 0.15793652087450027, "reward_std": 0.009622504934668541, "rewards/accuracy_reward": 0.15793652087450027, "step": 609 }, { "clip_ratio": 0.0, "completion_length": 7.9375, "epoch": 0.4756335282651072, "grad_norm": 15.281853206542879, "kl": 2.25, "learning_rate": 6.657775577417089e-07, "loss": 0.4116, "reward": 0.1750330924987793, "reward_std": 0.008133307099342346, "rewards/accuracy_reward": 0.1750330924987793, "step": 610 }, { "clip_ratio": 0.0, "completion_length": 9.354166746139526, "epoch": 0.4764132553606238, "grad_norm": 12.471122766753131, "kl": 2.984375, "learning_rate": 6.645922852905121e-07, "loss": 0.1106, "reward": 0.1470194309949875, "reward_std": 0.0005306530511006713, "rewards/accuracy_reward": 0.1470194309949875, "step": 611 }, { "clip_ratio": 0.0, "completion_length": 8.937500238418579, "epoch": 0.47719298245614034, "grad_norm": 3.0001199756443504, "kl": 2.15234375, "learning_rate": 6.634061621005106e-07, "loss": 0.0872, "reward": 0.16417215019464493, "reward_std": 0.0014243837213143706, "rewards/accuracy_reward": 0.16417215019464493, "step": 612 }, { "clip_ratio": 0.0, "completion_length": 7.500000238418579, "epoch": 0.47797270955165694, "grad_norm": 4.179672100281533, "kl": 2.78515625, "learning_rate": 6.622191969775419e-07, "loss": 0.1269, "reward": 0.18232886493206024, "reward_std": 0.0009021099540404975, "rewards/accuracy_reward": 0.18232886493206024, "step": 613 }, { "clip_ratio": 0.0, "completion_length": 6.125000238418579, "epoch": 0.4787524366471735, "grad_norm": 8.951380579323073, "kl": 2.0546875, "learning_rate": 6.610313987336938e-07, "loss": 0.0962, "reward": 0.15555556863546371, "reward_std": 0.009622504934668541, "rewards/accuracy_reward": 0.15555556863546371, "step": 614 }, { "clip_ratio": 0.0, "completion_length": 6.770833492279053, "epoch": 0.47953216374269003, "grad_norm": 10.399509178769897, "kl": 2.0859375, "learning_rate": 6.598427761872391e-07, "loss": 0.1449, "reward": 0.15236419439315796, "reward_std": 0.005728523246943951, "rewards/accuracy_reward": 0.15236419439315796, "step": 615 }, { "clip_ratio": 0.0, "completion_length": 8.916666984558105, "epoch": 0.48031189083820663, "grad_norm": 4.422649413931237, "kl": 2.515625, "learning_rate": 6.586533381625709e-07, "loss": 0.1128, "reward": 0.17276348173618317, "reward_std": 0.0005306530511006713, "rewards/accuracy_reward": 0.17276348173618317, "step": 616 }, { "clip_ratio": 0.0, "completion_length": 6.4375, "epoch": 0.4810916179337232, "grad_norm": 129.67718452079302, "kl": 5.2734375, "learning_rate": 6.574630934901358e-07, "loss": 0.5996, "reward": 0.1466388925909996, "reward_std": 0.006306389521341771, "rewards/accuracy_reward": 0.1466388925909996, "step": 617 }, { "clip_ratio": 0.0, "completion_length": 9.479166746139526, "epoch": 0.4818713450292398, "grad_norm": 17.16827123030281, "kl": 2.6171875, "learning_rate": 6.562720510063694e-07, "loss": 0.4359, "reward": 0.1692412719130516, "reward_std": 0.012123214080929756, "rewards/accuracy_reward": 0.1692412719130516, "step": 618 }, { "clip_ratio": 0.0, "completion_length": 6.708333492279053, "epoch": 0.48265107212475633, "grad_norm": 2.363160291070905, "kl": 1.546875, "learning_rate": 6.5508021955363e-07, "loss": 0.0631, "reward": 0.15238095819950104, "reward_std": 0.0, "rewards/accuracy_reward": 0.15238095819950104, "step": 619 }, { "clip_ratio": 0.0, "completion_length": 6.500000238418579, "epoch": 0.4834307992202729, "grad_norm": 4.739304596391792, "kl": 1.53515625, "learning_rate": 6.538876079801331e-07, "loss": 0.0598, "reward": 0.17053572088479996, "reward_std": 0.005154913291335106, "rewards/accuracy_reward": 0.17053572088479996, "step": 620 }, { "clip_ratio": 0.0, "completion_length": 8.145833492279053, "epoch": 0.4842105263157895, "grad_norm": 1.7255607833059352, "kl": 1.9921875, "learning_rate": 6.526942251398866e-07, "loss": 0.0769, "reward": 0.14479167759418488, "reward_std": 0.0, "rewards/accuracy_reward": 0.14479167759418488, "step": 621 }, { "clip_ratio": 0.0, "completion_length": 6.354166746139526, "epoch": 0.484990253411306, "grad_norm": 47.40812056596257, "kl": 4.578125, "learning_rate": 6.515000798926235e-07, "loss": 0.2508, "reward": 0.159835085272789, "reward_std": 0.0025448258966207504, "rewards/accuracy_reward": 0.159835085272789, "step": 622 }, { "clip_ratio": 0.0, "completion_length": 7.625000238418579, "epoch": 0.48576998050682263, "grad_norm": 22.518783224194827, "kl": 2.390625, "learning_rate": 6.503051811037377e-07, "loss": 0.3716, "reward": 0.16134236752986908, "reward_std": 0.015866519417613745, "rewards/accuracy_reward": 0.16134236752986908, "step": 623 }, { "clip_ratio": 0.0, "completion_length": 6.7916669845581055, "epoch": 0.4865497076023392, "grad_norm": 30.88465054349459, "kl": 6.75, "learning_rate": 6.491095376442164e-07, "loss": 0.1673, "reward": 0.15853175520896912, "reward_std": 0.035625264048576355, "rewards/accuracy_reward": 0.15853175520896912, "step": 624 }, { "clip_ratio": 0.0, "completion_length": 7.375000238418579, "epoch": 0.4873294346978557, "grad_norm": 19.168217749151395, "kl": 1.021484375, "learning_rate": 6.479131583905766e-07, "loss": 0.409, "reward": 0.1417711079120636, "reward_std": 0.01613397477194667, "rewards/accuracy_reward": 0.1417711079120636, "step": 625 }, { "clip_ratio": 0.0, "completion_length": 8.333333492279053, "epoch": 0.4881091617933723, "grad_norm": 15.944055747770157, "kl": 3.5078125, "learning_rate": 6.467160522247965e-07, "loss": 0.0806, "reward": 0.14295197278261185, "reward_std": 0.016854544635862112, "rewards/accuracy_reward": 0.14295197278261185, "step": 626 }, { "clip_ratio": 0.0, "completion_length": 9.541666984558105, "epoch": 0.4888888888888889, "grad_norm": 15.846691362415955, "kl": 2.23828125, "learning_rate": 6.455182280342522e-07, "loss": 0.1388, "reward": 0.1582958921790123, "reward_std": 0.01749379187822342, "rewards/accuracy_reward": 0.1582958921790123, "step": 627 }, { "clip_ratio": 0.0, "completion_length": 7.6041669845581055, "epoch": 0.4896686159844055, "grad_norm": 2.2593549347639135, "kl": 2.046875, "learning_rate": 6.443196947116492e-07, "loss": 0.0824, "reward": 0.18854166567325592, "reward_std": 0.0, "rewards/accuracy_reward": 0.18854166567325592, "step": 628 }, { "clip_ratio": 0.0, "completion_length": 6.604166746139526, "epoch": 0.490448343079922, "grad_norm": 5.918806433786088, "kl": 0.94921875, "learning_rate": 6.431204611549589e-07, "loss": 0.0259, "reward": 0.18194445222616196, "reward_std": 0.0048112524673342705, "rewards/accuracy_reward": 0.18194445222616196, "step": 629 }, { "clip_ratio": 0.0, "completion_length": 6.500000238418579, "epoch": 0.49122807017543857, "grad_norm": 12.937843315553032, "kl": 1.33984375, "learning_rate": 6.419205362673503e-07, "loss": 0.116, "reward": 0.15198414027690887, "reward_std": 0.009622504934668541, "rewards/accuracy_reward": 0.15198414027690887, "step": 630 }, { "clip_ratio": 0.0, "completion_length": 5.666666746139526, "epoch": 0.49200779727095517, "grad_norm": 35.714845503092974, "kl": 1.595703125, "learning_rate": 6.407199289571254e-07, "loss": 0.2641, "reward": 0.13688726723194122, "reward_std": 0.011462102644145489, "rewards/accuracy_reward": 0.13688726723194122, "step": 631 }, { "clip_ratio": 0.0, "completion_length": 5.812500238418579, "epoch": 0.4927875243664717, "grad_norm": 69.86771767706655, "kl": 5.318359375, "learning_rate": 6.395186481376521e-07, "loss": 0.222, "reward": 0.13960815221071243, "reward_std": 0.02216244861483574, "rewards/accuracy_reward": 0.13960815221071243, "step": 632 }, { "clip_ratio": 0.0, "completion_length": 8.4375, "epoch": 0.4935672514619883, "grad_norm": 8.01478186582231, "kl": 2.1181640625, "learning_rate": 6.383167027272992e-07, "loss": 0.0617, "reward": 0.1440332680940628, "reward_std": 0.013452515006065369, "rewards/accuracy_reward": 0.1440332680940628, "step": 633 }, { "clip_ratio": 0.0, "completion_length": 6.604166746139526, "epoch": 0.49434697855750487, "grad_norm": 8.795807152079473, "kl": 1.96875, "learning_rate": 6.371141016493684e-07, "loss": 0.046, "reward": 0.16944444924592972, "reward_std": 0.01297498308122158, "rewards/accuracy_reward": 0.16944444924592972, "step": 634 }, { "clip_ratio": 0.0, "completion_length": 6.375000238418579, "epoch": 0.4951267056530214, "grad_norm": 9.0496512155821, "kl": 2.1796875, "learning_rate": 6.359108538320302e-07, "loss": 0.1689, "reward": 0.14785150811076164, "reward_std": 0.005993825849145651, "rewards/accuracy_reward": 0.14785150811076164, "step": 635 }, { "clip_ratio": 0.0, "completion_length": 6.395833492279053, "epoch": 0.495906432748538, "grad_norm": 7193.058519057484, "kl": 187.134765625, "learning_rate": 6.347069682082558e-07, "loss": 15.7364, "reward": 0.160714291036129, "reward_std": 0.009622504934668541, "rewards/accuracy_reward": 0.160714291036129, "step": 636 }, { "clip_ratio": 0.0, "completion_length": 7.458333492279053, "epoch": 0.49668615984405456, "grad_norm": 9.988452809840618, "kl": 1.6328125, "learning_rate": 6.335024537157516e-07, "loss": 0.0923, "reward": 0.18113402277231216, "reward_std": 0.00504120159894228, "rewards/accuracy_reward": 0.18113402277231216, "step": 637 }, { "clip_ratio": 0.0, "completion_length": 7.625000238418579, "epoch": 0.49746588693957117, "grad_norm": 10.971990521167271, "kl": 3.09765625, "learning_rate": 6.322973192968931e-07, "loss": 0.1809, "reward": 0.1791294664144516, "reward_std": 0.01146968174725771, "rewards/accuracy_reward": 0.1791294664144516, "step": 638 }, { "clip_ratio": 0.0, "completion_length": 6.041666746139526, "epoch": 0.4982456140350877, "grad_norm": 7.030819740507299, "kl": 1.2158203125, "learning_rate": 6.310915738986581e-07, "loss": 0.0645, "reward": 0.16208640486001968, "reward_std": 0.00504120159894228, "rewards/accuracy_reward": 0.16208640486001968, "step": 639 }, { "clip_ratio": 0.0, "completion_length": 6.979166746139526, "epoch": 0.49902534113060426, "grad_norm": 1.9810138733082172, "kl": 1.806640625, "learning_rate": 6.298852264725597e-07, "loss": 0.0721, "reward": 0.1666666716337204, "reward_std": 0.0, "rewards/accuracy_reward": 0.1666666716337204, "step": 640 }, { "clip_ratio": 0.0, "completion_length": 6.958333492279053, "epoch": 0.49980506822612086, "grad_norm": 17.153507425862223, "kl": 1.984375, "learning_rate": 6.286782859745814e-07, "loss": 0.0376, "reward": 0.1480654776096344, "reward_std": 0.020619653165340424, "rewards/accuracy_reward": 0.1480654776096344, "step": 641 }, { "clip_ratio": 0.0, "completion_length": 7.145833492279053, "epoch": 0.5005847953216375, "grad_norm": 6.674066550522118, "kl": 1.58984375, "learning_rate": 6.274707613651092e-07, "loss": 0.0586, "reward": 0.1725504994392395, "reward_std": 0.007756044622510672, "rewards/accuracy_reward": 0.1725504994392395, "step": 642 }, { "clip_ratio": 0.0, "completion_length": 6.1666669845581055, "epoch": 0.501364522417154, "grad_norm": 12.824860359289994, "kl": 1.0380859375, "learning_rate": 6.262626616088653e-07, "loss": 0.0922, "reward": 0.15486112236976624, "reward_std": 0.0008109381305985153, "rewards/accuracy_reward": 0.15486112236976624, "step": 643 }, { "clip_ratio": 0.0, "completion_length": 5.687500238418579, "epoch": 0.5021442495126706, "grad_norm": 5.728281211182756, "kl": 0.494140625, "learning_rate": 6.250539956748425e-07, "loss": 0.0061, "reward": 0.16006945073604584, "reward_std": 0.0060140653513371944, "rewards/accuracy_reward": 0.16006945073604584, "step": 644 }, { "clip_ratio": 0.0, "completion_length": 6.1875, "epoch": 0.5029239766081871, "grad_norm": 30.717564417513398, "kl": 4.1953125, "learning_rate": 6.238447725362365e-07, "loss": 0.2052, "reward": 0.14884259551763535, "reward_std": 0.026606866158545017, "rewards/accuracy_reward": 0.14884259551763535, "step": 645 }, { "clip_ratio": 0.0, "completion_length": 7.770833492279053, "epoch": 0.5037037037037037, "grad_norm": 11.833191675001, "kl": 2.2734375, "learning_rate": 6.226350011703799e-07, "loss": 0.1026, "reward": 0.17846541106700897, "reward_std": 0.005884554237127304, "rewards/accuracy_reward": 0.17846541106700897, "step": 646 }, { "clip_ratio": 0.0, "completion_length": 8.604166746139526, "epoch": 0.5044834307992203, "grad_norm": 14.240639321293012, "kl": 0.912109375, "learning_rate": 6.214246905586751e-07, "loss": 0.1685, "reward": 0.1389467641711235, "reward_std": 0.002205158583819866, "rewards/accuracy_reward": 0.1389467641711235, "step": 647 }, { "clip_ratio": 0.0, "completion_length": 6.854166746139526, "epoch": 0.5052631578947369, "grad_norm": 9.647720482389149, "kl": 1.154296875, "learning_rate": 6.202138496865281e-07, "loss": 0.0552, "reward": 0.16805556416511536, "reward_std": 0.00682500284165144, "rewards/accuracy_reward": 0.16805556416511536, "step": 648 }, { "clip_ratio": 0.0, "completion_length": 7.708333492279053, "epoch": 0.5060428849902534, "grad_norm": 12.229031584877808, "kl": 1.56640625, "learning_rate": 6.190024875432818e-07, "loss": 0.2373, "reward": 0.1589185893535614, "reward_std": 0.006443812511861324, "rewards/accuracy_reward": 0.1589185893535614, "step": 649 }, { "clip_ratio": 0.0, "completion_length": 7.75, "epoch": 0.50682261208577, "grad_norm": 14.465505864382934, "kl": 1.59375, "learning_rate": 6.177906131221483e-07, "loss": 0.1529, "reward": 0.16737351566553116, "reward_std": 0.02384147420525551, "rewards/accuracy_reward": 0.16737351566553116, "step": 650 }, { "clip_ratio": 0.0, "completion_length": 6.270833492279053, "epoch": 0.5076023391812865, "grad_norm": 10.700838307797138, "kl": 0.4462890625, "learning_rate": 6.165782354201435e-07, "loss": -0.0086, "reward": 0.15114088356494904, "reward_std": 0.016323892399668694, "rewards/accuracy_reward": 0.15114088356494904, "step": 651 }, { "clip_ratio": 0.0, "completion_length": 6.937500238418579, "epoch": 0.5083820662768032, "grad_norm": 18.398483095593406, "kl": 2.203125, "learning_rate": 6.153653634380198e-07, "loss": 0.1543, "reward": 0.1621279940009117, "reward_std": 0.023587769828736782, "rewards/accuracy_reward": 0.1621279940009117, "step": 652 }, { "clip_ratio": 0.0, "completion_length": 7.812500238418579, "epoch": 0.5091617933723197, "grad_norm": 283.6952956407202, "kl": 12.71875, "learning_rate": 6.141520061801988e-07, "loss": 0.9772, "reward": 0.1845824345946312, "reward_std": 0.014288676902651787, "rewards/accuracy_reward": 0.1845824345946312, "step": 653 }, { "clip_ratio": 0.0, "completion_length": 6.645833492279053, "epoch": 0.5099415204678363, "grad_norm": 8.87848252936082, "kl": 1.640625, "learning_rate": 6.129381726547049e-07, "loss": 0.065, "reward": 0.171875, "reward_std": 0.0, "rewards/accuracy_reward": 0.171875, "step": 654 }, { "clip_ratio": 0.0, "completion_length": 9.0, "epoch": 0.5107212475633528, "grad_norm": 17.34603911024246, "kl": 2.1796875, "learning_rate": 6.117238718730982e-07, "loss": 0.1457, "reward": 0.144157275557518, "reward_std": 0.010380841908045113, "rewards/accuracy_reward": 0.144157275557518, "step": 655 }, { "clip_ratio": 0.0, "completion_length": 7.458333492279053, "epoch": 0.5115009746588693, "grad_norm": 49.85826248694607, "kl": 6.953125, "learning_rate": 6.10509112850408e-07, "loss": 0.3299, "reward": 0.15697727352380753, "reward_std": 0.021236851811408997, "rewards/accuracy_reward": 0.15697727352380753, "step": 656 }, { "clip_ratio": 0.0, "completion_length": 6.270833492279053, "epoch": 0.512280701754386, "grad_norm": 5.076072964666256, "kl": 0.595703125, "learning_rate": 6.092939046050654e-07, "loss": 0.0111, "reward": 0.159722238779068, "reward_std": 0.0048112524673342705, "rewards/accuracy_reward": 0.159722238779068, "step": 657 }, { "clip_ratio": 0.0, "completion_length": 8.270833492279053, "epoch": 0.5130604288499026, "grad_norm": 26.830201776101685, "kl": 2.640625, "learning_rate": 6.080782561588366e-07, "loss": 0.0486, "reward": 0.170258529484272, "reward_std": 0.028441181406378746, "rewards/accuracy_reward": 0.170258529484272, "step": 658 }, { "clip_ratio": 0.0, "completion_length": 5.916666746139526, "epoch": 0.5138401559454191, "grad_norm": 43.680268961762586, "kl": 7.28515625, "learning_rate": 6.068621765367563e-07, "loss": 0.0624, "reward": 0.15620040148496628, "reward_std": 0.02526606386527419, "rewards/accuracy_reward": 0.15620040148496628, "step": 659 }, { "clip_ratio": 0.0, "completion_length": 6.875, "epoch": 0.5146198830409356, "grad_norm": 9.658475945557967, "kl": 1.146484375, "learning_rate": 6.056456747670594e-07, "loss": 0.0218, "reward": 0.16845238953828812, "reward_std": 0.016333107836544514, "rewards/accuracy_reward": 0.16845238953828812, "step": 660 }, { "clip_ratio": 0.0, "completion_length": 7.041666746139526, "epoch": 0.5153996101364522, "grad_norm": 17.11592550448531, "kl": 1.650390625, "learning_rate": 6.044287598811156e-07, "loss": 0.2725, "reward": 0.1574452891945839, "reward_std": 0.016178556601516902, "rewards/accuracy_reward": 0.1574452891945839, "step": 661 }, { "clip_ratio": 0.0, "completion_length": 6.645833492279053, "epoch": 0.5161793372319688, "grad_norm": 16.750914811252894, "kl": 0.970703125, "learning_rate": 6.03211440913361e-07, "loss": 0.0353, "reward": 0.1451389044523239, "reward_std": 0.021835491061210632, "rewards/accuracy_reward": 0.1451389044523239, "step": 662 }, { "clip_ratio": 0.0, "completion_length": 7.375000238418579, "epoch": 0.5169590643274854, "grad_norm": 2050.8657533772443, "kl": 61.25390625, "learning_rate": 6.019937269012325e-07, "loss": 2.4176, "reward": 0.173611119389534, "reward_std": 0.01114263292402029, "rewards/accuracy_reward": 0.173611119389534, "step": 663 }, { "clip_ratio": 0.0, "completion_length": 6.666666746139526, "epoch": 0.5177387914230019, "grad_norm": 84.41950741746031, "kl": 5.953125, "learning_rate": 6.007756268850987e-07, "loss": 0.3029, "reward": 0.17240846902132034, "reward_std": 0.016569858882576227, "rewards/accuracy_reward": 0.17240846902132034, "step": 664 }, { "clip_ratio": 0.0, "completion_length": 7.250000238418579, "epoch": 0.5185185185185185, "grad_norm": 10.632124676640212, "kl": 3.75, "learning_rate": 5.995571499081946e-07, "loss": 0.1058, "reward": 0.16848959028720856, "reward_std": 0.010446218773722649, "rewards/accuracy_reward": 0.16848959028720856, "step": 665 }, { "clip_ratio": 0.0, "completion_length": 7.875000476837158, "epoch": 0.519298245614035, "grad_norm": 156.28888702429649, "kl": 7.90625, "learning_rate": 5.983383050165539e-07, "loss": 0.3183, "reward": 0.18854168057441711, "reward_std": 0.0, "rewards/accuracy_reward": 0.18854168057441711, "step": 666 }, { "clip_ratio": 0.0, "completion_length": 6.104166746139526, "epoch": 0.5200779727095517, "grad_norm": 5087.227959639842, "kl": 154.83984375, "learning_rate": 5.971191012589415e-07, "loss": 10.0331, "reward": 0.159132719039917, "reward_std": 0.028588212095201015, "rewards/accuracy_reward": 0.159132719039917, "step": 667 }, { "clip_ratio": 0.0, "completion_length": 6.833333492279053, "epoch": 0.5208576998050682, "grad_norm": 24.811678337046942, "kl": 2.40625, "learning_rate": 5.958995476867862e-07, "loss": 0.0705, "reward": 0.17213542014360428, "reward_std": 0.02588246390223503, "rewards/accuracy_reward": 0.17213542014360428, "step": 668 }, { "clip_ratio": 0.0, "completion_length": 6.166666746139526, "epoch": 0.5216374269005848, "grad_norm": 19.89640599697181, "kl": 3.5234375, "learning_rate": 5.946796533541143e-07, "loss": 0.0842, "reward": 0.18283730000257492, "reward_std": 0.0351847056299448, "rewards/accuracy_reward": 0.18283730000257492, "step": 669 }, { "clip_ratio": 0.0, "completion_length": 7.791666746139526, "epoch": 0.5224171539961013, "grad_norm": 2.3450295174618097, "kl": 1.6640625, "learning_rate": 5.934594273174818e-07, "loss": 0.0663, "reward": 0.19166667759418488, "reward_std": 0.0, "rewards/accuracy_reward": 0.19166667759418488, "step": 670 }, { "clip_ratio": 0.0, "completion_length": 7.458333492279053, "epoch": 0.5231968810916179, "grad_norm": 5.56837939255499, "kl": 2.9765625, "learning_rate": 5.922388786359073e-07, "loss": 0.1084, "reward": 0.18318451941013336, "reward_std": 0.009325588122010231, "rewards/accuracy_reward": 0.18318451941013336, "step": 671 }, { "clip_ratio": 0.0, "completion_length": 5.833333492279053, "epoch": 0.5239766081871345, "grad_norm": 29.431000852624095, "kl": 4.828125, "learning_rate": 5.910180163708046e-07, "loss": 0.1548, "reward": 0.17681623995304108, "reward_std": 0.02648637257516384, "rewards/accuracy_reward": 0.17681623995304108, "step": 672 }, { "clip_ratio": 0.0, "completion_length": 6.750000238418579, "epoch": 0.5247563352826511, "grad_norm": 13.758226623764198, "kl": 1.984375, "learning_rate": 5.897968495859157e-07, "loss": 0.0634, "reward": 0.18368057161569595, "reward_std": 0.022755810990929604, "rewards/accuracy_reward": 0.18368057161569595, "step": 673 }, { "clip_ratio": 0.0, "completion_length": 9.916666746139526, "epoch": 0.5255360623781676, "grad_norm": 139.22857474442833, "kl": 12.96875, "learning_rate": 5.885753873472434e-07, "loss": 0.4604, "reward": 0.16230159997940063, "reward_std": 0.015001907013356686, "rewards/accuracy_reward": 0.16230159997940063, "step": 674 }, { "clip_ratio": 0.0, "completion_length": 7.437500476837158, "epoch": 0.5263157894736842, "grad_norm": 154.49189447310852, "kl": 12.5, "learning_rate": 5.87353638722984e-07, "loss": 0.6283, "reward": 0.16762153804302216, "reward_std": 0.03621844947338104, "rewards/accuracy_reward": 0.16762153804302216, "step": 675 }, { "clip_ratio": 0.0, "completion_length": 7.125000238418579, "epoch": 0.5270955165692007, "grad_norm": 28.054526732656722, "kl": 2.7890625, "learning_rate": 5.861316127834592e-07, "loss": 0.1547, "reward": 0.17039930820465088, "reward_std": 0.014133053831756115, "rewards/accuracy_reward": 0.17039930820465088, "step": 676 }, { "clip_ratio": 0.0, "completion_length": 6.750000238418579, "epoch": 0.5278752436647174, "grad_norm": 14.296662507127708, "kl": 1.556640625, "learning_rate": 5.849093186010509e-07, "loss": 0.0602, "reward": 0.17213542759418488, "reward_std": 0.02106943028047681, "rewards/accuracy_reward": 0.17213542759418488, "step": 677 }, { "clip_ratio": 0.0, "completion_length": 5.5625, "epoch": 0.5286549707602339, "grad_norm": 130.2400127702207, "kl": 6.99609375, "learning_rate": 5.836867652501313e-07, "loss": 0.3414, "reward": 0.14906556904315948, "reward_std": 0.020471498370170593, "rewards/accuracy_reward": 0.14906556904315948, "step": 678 }, { "clip_ratio": 0.0, "completion_length": 7.812500238418579, "epoch": 0.5294346978557505, "grad_norm": 11.871878217320992, "kl": 2.01171875, "learning_rate": 5.824639618069973e-07, "loss": 0.0839, "reward": 0.19124504923820496, "reward_std": 0.014874828048050404, "rewards/accuracy_reward": 0.19124504923820496, "step": 679 }, { "clip_ratio": 0.0, "completion_length": 8.854166746139526, "epoch": 0.530214424951267, "grad_norm": 17.26443076390969, "kl": 2.2734375, "learning_rate": 5.812409173498019e-07, "loss": 0.1086, "reward": 0.1638144925236702, "reward_std": 0.02170541975647211, "rewards/accuracy_reward": 0.1638144925236702, "step": 680 }, { "clip_ratio": 0.0, "completion_length": 6.75, "epoch": 0.5309941520467836, "grad_norm": 28.133039339991658, "kl": 3.16015625, "learning_rate": 5.80017640958488e-07, "loss": 0.1581, "reward": 0.1653050184249878, "reward_std": 0.006909038871526718, "rewards/accuracy_reward": 0.1653050184249878, "step": 681 }, { "clip_ratio": 0.0, "completion_length": 6.750000238418579, "epoch": 0.5317738791423002, "grad_norm": 11.186608179646798, "kl": 1.28515625, "learning_rate": 5.7879414171472e-07, "loss": 0.0213, "reward": 0.18263889104127884, "reward_std": 0.017628328874707222, "rewards/accuracy_reward": 0.18263889104127884, "step": 682 }, { "clip_ratio": 0.0, "completion_length": 6.312500238418579, "epoch": 0.5325536062378168, "grad_norm": 13.021101597241106, "kl": 2.8359375, "learning_rate": 5.775704287018171e-07, "loss": 0.1604, "reward": 0.1791973114013672, "reward_std": 0.012062542024068534, "rewards/accuracy_reward": 0.1791973114013672, "step": 683 }, { "clip_ratio": 0.0, "completion_length": 7.104166746139526, "epoch": 0.5333333333333333, "grad_norm": 53.50209530804248, "kl": 4.1484375, "learning_rate": 5.763465110046853e-07, "loss": 0.1854, "reward": 0.18683037161827087, "reward_std": 0.025573696941137314, "rewards/accuracy_reward": 0.18683037161827087, "step": 684 }, { "clip_ratio": 0.0, "completion_length": 9.604166984558105, "epoch": 0.5341130604288499, "grad_norm": 23.466449976806057, "kl": 2.828125, "learning_rate": 5.751223977097501e-07, "loss": 0.4945, "reward": 0.14113732427358627, "reward_std": 0.02884014882147312, "rewards/accuracy_reward": 0.14113732427358627, "step": 685 }, { "clip_ratio": 0.0, "completion_length": 7.000000238418579, "epoch": 0.5348927875243664, "grad_norm": 44.963489753777345, "kl": 4.515625, "learning_rate": 5.738980979048895e-07, "loss": 0.1324, "reward": 0.17809193581342697, "reward_std": 0.029883730225265026, "rewards/accuracy_reward": 0.17809193581342697, "step": 686 }, { "clip_ratio": 0.0, "completion_length": 7.166666746139526, "epoch": 0.5356725146198831, "grad_norm": 49.31301511235178, "kl": 6.2109375, "learning_rate": 5.726736206793657e-07, "loss": 0.3362, "reward": 0.1874600201845169, "reward_std": 0.01942795980721712, "rewards/accuracy_reward": 0.1874600201845169, "step": 687 }, { "clip_ratio": 0.0, "completion_length": 7.145833492279053, "epoch": 0.5364522417153996, "grad_norm": 16.179513267930382, "kl": 3.765625, "learning_rate": 5.714489751237582e-07, "loss": 0.1491, "reward": 0.1918584108352661, "reward_std": 0.021227598190307617, "rewards/accuracy_reward": 0.1918584108352661, "step": 688 }, { "clip_ratio": 0.0, "completion_length": 10.145833492279053, "epoch": 0.5372319688109162, "grad_norm": 13.499458967313133, "kl": 2.26171875, "learning_rate": 5.702241703298966e-07, "loss": 0.0808, "reward": 0.1741752177476883, "reward_std": 0.01613913895562291, "rewards/accuracy_reward": 0.1741752177476883, "step": 689 }, { "clip_ratio": 0.0, "completion_length": 8.000000238418579, "epoch": 0.5380116959064327, "grad_norm": 8.157827902881921, "kl": 3.578125, "learning_rate": 5.689992153907924e-07, "loss": 0.1595, "reward": 0.18358135968446732, "reward_std": 0.022893482819199562, "rewards/accuracy_reward": 0.18358135968446732, "step": 690 }, { "clip_ratio": 0.0, "completion_length": 7.583333492279053, "epoch": 0.5387914230019493, "grad_norm": 22.99930865181368, "kl": 2.9453125, "learning_rate": 5.677741194005715e-07, "loss": 0.1294, "reward": 0.19384921342134476, "reward_std": 0.0048112524673342705, "rewards/accuracy_reward": 0.19384921342134476, "step": 691 }, { "clip_ratio": 0.0, "completion_length": 8.854166746139526, "epoch": 0.5395711500974659, "grad_norm": 162.69802776558012, "kl": 7.90625, "learning_rate": 5.665488914544074e-07, "loss": 1.3169, "reward": 0.19633013010025024, "reward_std": 0.019426160492002964, "rewards/accuracy_reward": 0.19633013010025024, "step": 692 }, { "clip_ratio": 0.0, "completion_length": 9.145833492279053, "epoch": 0.5403508771929825, "grad_norm": 641.6373505465062, "kl": 37.140625, "learning_rate": 5.653235406484531e-07, "loss": 1.8916, "reward": 0.19438251852989197, "reward_std": 0.03529222495853901, "rewards/accuracy_reward": 0.19438251852989197, "step": 693 }, { "clip_ratio": 0.0, "completion_length": 6.9791669845581055, "epoch": 0.541130604288499, "grad_norm": 542.3112645059924, "kl": 20.328125, "learning_rate": 5.640980760797737e-07, "loss": 1.3815, "reward": 0.19339163601398468, "reward_std": 0.007063568918965757, "rewards/accuracy_reward": 0.19339163601398468, "step": 694 }, { "clip_ratio": 0.0, "completion_length": 7.854166746139526, "epoch": 0.5419103313840156, "grad_norm": 24.741152175313765, "kl": 4.953125, "learning_rate": 5.628725068462787e-07, "loss": 0.4404, "reward": 0.19107971340417862, "reward_std": 0.02465215139091015, "rewards/accuracy_reward": 0.19107971340417862, "step": 695 }, { "clip_ratio": 0.0, "completion_length": 8.145833492279053, "epoch": 0.5426900584795321, "grad_norm": 7.452326717408337, "kl": 1.71484375, "learning_rate": 5.616468420466551e-07, "loss": 0.0925, "reward": 0.19727183133363724, "reward_std": 0.007431631907820702, "rewards/accuracy_reward": 0.19727183133363724, "step": 696 }, { "clip_ratio": 0.0, "completion_length": 7.916666746139526, "epoch": 0.5434697855750488, "grad_norm": 14.993960149756191, "kl": 3.703125, "learning_rate": 5.60421090780299e-07, "loss": 0.1764, "reward": 0.1950024962425232, "reward_std": 0.022151668556034565, "rewards/accuracy_reward": 0.1950024962425232, "step": 697 }, { "clip_ratio": 0.0, "completion_length": 6.833333492279053, "epoch": 0.5442495126705653, "grad_norm": 11.883147731601257, "kl": 4.578125, "learning_rate": 5.591952621472485e-07, "loss": 0.191, "reward": 0.20087379217147827, "reward_std": 0.03229519259184599, "rewards/accuracy_reward": 0.20087379217147827, "step": 698 }, { "clip_ratio": 0.0, "completion_length": 9.0, "epoch": 0.5450292397660819, "grad_norm": 104.15676464754617, "kl": 10.6796875, "learning_rate": 5.579693652481161e-07, "loss": 0.6969, "reward": 0.19982202351093292, "reward_std": 0.02143909689038992, "rewards/accuracy_reward": 0.19982202351093292, "step": 699 }, { "clip_ratio": 0.0, "completion_length": 8.770833492279053, "epoch": 0.5458089668615984, "grad_norm": 24162.883116184807, "kl": 1673.453125, "learning_rate": 5.567434091840212e-07, "loss": 74.2185, "reward": 0.17545372247695923, "reward_std": 0.017187719931825995, "rewards/accuracy_reward": 0.17545372247695923, "step": 700 }, { "clip_ratio": 0.0, "completion_length": 9.083333492279053, "epoch": 0.546588693957115, "grad_norm": 311.0922699460503, "kl": 23.4765625, "learning_rate": 5.555174030565222e-07, "loss": 1.3108, "reward": 0.20801282674074173, "reward_std": 0.043087988160550594, "rewards/accuracy_reward": 0.20801282674074173, "step": 701 }, { "clip_ratio": 0.0, "completion_length": 9.645833730697632, "epoch": 0.5473684210526316, "grad_norm": 52.34143136163874, "kl": 9.6484375, "learning_rate": 5.542913559675497e-07, "loss": 0.3221, "reward": 0.17341270297765732, "reward_std": 0.02454973664134741, "rewards/accuracy_reward": 0.17341270297765732, "step": 702 }, { "clip_ratio": 0.0, "completion_length": 7.770833730697632, "epoch": 0.5481481481481482, "grad_norm": 63.190213433185555, "kl": 7.4453125, "learning_rate": 5.530652770193379e-07, "loss": 0.4314, "reward": 0.19103111326694489, "reward_std": 0.018122155219316483, "rewards/accuracy_reward": 0.19103111326694489, "step": 703 }, { "clip_ratio": 0.0, "completion_length": 8.770833492279053, "epoch": 0.5489278752436647, "grad_norm": 57.51952961665184, "kl": 2.78125, "learning_rate": 5.518391753143574e-07, "loss": 0.2208, "reward": 0.1961694210767746, "reward_std": 0.019015775993466377, "rewards/accuracy_reward": 0.1961694210767746, "step": 704 }, { "clip_ratio": 0.0, "completion_length": 9.145833492279053, "epoch": 0.5497076023391813, "grad_norm": 180.4042428106239, "kl": 8.7548828125, "learning_rate": 5.506130599552483e-07, "loss": 0.8192, "reward": 0.17864811420440674, "reward_std": 0.008954516611993313, "rewards/accuracy_reward": 0.17864811420440674, "step": 705 }, { "clip_ratio": 0.0, "completion_length": 14.833333969116211, "epoch": 0.5504873294346978, "grad_norm": 4418.477234155181, "kl": 157.359375, "learning_rate": 5.493869400447518e-07, "loss": 10.2428, "reward": 0.15092042833566666, "reward_std": 0.0321147795766592, "rewards/accuracy_reward": 0.15092042833566666, "step": 706 }, { "clip_ratio": 0.0, "completion_length": 7.0416669845581055, "epoch": 0.5512670565302145, "grad_norm": 17.197449789383192, "kl": 3.4765625, "learning_rate": 5.481608246856427e-07, "loss": 0.2102, "reward": 0.22228986769914627, "reward_std": 0.020675611682236195, "rewards/accuracy_reward": 0.22228986769914627, "step": 707 }, { "clip_ratio": 0.0, "completion_length": 7.395833492279053, "epoch": 0.552046783625731, "grad_norm": 24.75895248248711, "kl": 5.03125, "learning_rate": 5.469347229806622e-07, "loss": 0.2661, "reward": 0.21269650757312775, "reward_std": 0.030957388691604137, "rewards/accuracy_reward": 0.21269650757312775, "step": 708 }, { "clip_ratio": 0.0, "completion_length": 7.0625, "epoch": 0.5528265107212476, "grad_norm": 12.262279297133833, "kl": 3.6328125, "learning_rate": 5.457086440324503e-07, "loss": 0.1487, "reward": 0.2014680802822113, "reward_std": 0.012678503233473748, "rewards/accuracy_reward": 0.2014680802822113, "step": 709 }, { "clip_ratio": 0.0, "completion_length": 8.604166746139526, "epoch": 0.5536062378167641, "grad_norm": 14.267412304401052, "kl": 3.390625, "learning_rate": 5.444825969434777e-07, "loss": 0.3105, "reward": 0.20235450565814972, "reward_std": 0.023722592275589705, "rewards/accuracy_reward": 0.20235450565814972, "step": 710 }, { "clip_ratio": 0.0, "completion_length": 8.062500238418579, "epoch": 0.5543859649122806, "grad_norm": 3.098212699507371, "kl": 2.0390625, "learning_rate": 5.432565908159789e-07, "loss": 0.0804, "reward": 0.19099266082048416, "reward_std": 0.0, "rewards/accuracy_reward": 0.19099266082048416, "step": 711 }, { "clip_ratio": 0.0, "completion_length": 7.020833492279053, "epoch": 0.5551656920077973, "grad_norm": 12.61422364904933, "kl": 3.6796875, "learning_rate": 5.420306347518841e-07, "loss": 0.1088, "reward": 0.2105216085910797, "reward_std": 0.03166159801185131, "rewards/accuracy_reward": 0.2105216085910797, "step": 712 }, { "clip_ratio": 0.0, "completion_length": 7.7291669845581055, "epoch": 0.5559454191033139, "grad_norm": 12.778117750548397, "kl": 2.52734375, "learning_rate": 5.408047378527515e-07, "loss": 0.1102, "reward": 0.18544147163629532, "reward_std": 0.02681065909564495, "rewards/accuracy_reward": 0.18544147163629532, "step": 713 }, { "clip_ratio": 0.0, "completion_length": 7.666666746139526, "epoch": 0.5567251461988304, "grad_norm": 19.36387005810071, "kl": 2.9375, "learning_rate": 5.39578909219701e-07, "loss": 0.1413, "reward": 0.1749693676829338, "reward_std": 0.0251549887470901, "rewards/accuracy_reward": 0.1749693676829338, "step": 714 }, { "clip_ratio": 0.0, "completion_length": 8.020833730697632, "epoch": 0.557504873294347, "grad_norm": 15.732964286537323, "kl": 2.5, "learning_rate": 5.383531579533448e-07, "loss": 0.2041, "reward": 0.19231152534484863, "reward_std": 0.018857845105230808, "rewards/accuracy_reward": 0.19231152534484863, "step": 715 }, { "clip_ratio": 0.0, "completion_length": 9.958333492279053, "epoch": 0.5582846003898635, "grad_norm": 27.36454473112182, "kl": 4.203125, "learning_rate": 5.371274931537212e-07, "loss": 0.3749, "reward": 0.20143437385559082, "reward_std": 0.029813488014042377, "rewards/accuracy_reward": 0.20143437385559082, "step": 716 }, { "clip_ratio": 0.0, "completion_length": 8.291666746139526, "epoch": 0.5590643274853802, "grad_norm": 29.97490811347302, "kl": 5.203125, "learning_rate": 5.359019239202262e-07, "loss": 0.5586, "reward": 0.16298101097345352, "reward_std": 0.022775106132030487, "rewards/accuracy_reward": 0.16298101097345352, "step": 717 }, { "clip_ratio": 0.0, "completion_length": 7.000000238418579, "epoch": 0.5598440545808967, "grad_norm": 524.2191846566968, "kl": 26.203125, "learning_rate": 5.346764593515468e-07, "loss": 1.2854, "reward": 0.2096840739250183, "reward_std": 0.044943878427147865, "rewards/accuracy_reward": 0.2096840739250183, "step": 718 }, { "clip_ratio": 0.0, "completion_length": 9.479166984558105, "epoch": 0.5606237816764132, "grad_norm": 16.78877130061686, "kl": 1.6953125, "learning_rate": 5.334511085455925e-07, "loss": 0.2368, "reward": 0.19805872440338135, "reward_std": 0.006724819540977478, "rewards/accuracy_reward": 0.19805872440338135, "step": 719 }, { "clip_ratio": 0.0, "completion_length": 7.958333492279053, "epoch": 0.5614035087719298, "grad_norm": 18.92408129852908, "kl": 4.71875, "learning_rate": 5.322258805994284e-07, "loss": 0.2904, "reward": 0.20917277038097382, "reward_std": 0.015409362851642072, "rewards/accuracy_reward": 0.20917277038097382, "step": 720 }, { "clip_ratio": 0.0, "completion_length": 7.8125, "epoch": 0.5621832358674463, "grad_norm": 10.665002131264536, "kl": 2.453125, "learning_rate": 5.310007846092077e-07, "loss": 0.1507, "reward": 0.19428324699401855, "reward_std": 0.0033077364787459373, "rewards/accuracy_reward": 0.19428324699401855, "step": 721 }, { "clip_ratio": 0.0, "completion_length": 7.687500238418579, "epoch": 0.562962962962963, "grad_norm": 40.64523919462699, "kl": 3.6015625, "learning_rate": 5.297758296701033e-07, "loss": 0.2089, "reward": 0.1823449209332466, "reward_std": 0.022853082045912743, "rewards/accuracy_reward": 0.1823449209332466, "step": 722 }, { "clip_ratio": 0.0, "completion_length": 7.875, "epoch": 0.5637426900584795, "grad_norm": 13.365020075746704, "kl": 2.71875, "learning_rate": 5.285510248762418e-07, "loss": 0.139, "reward": 0.21067707240581512, "reward_std": 0.00542085338383913, "rewards/accuracy_reward": 0.21067707240581512, "step": 723 }, { "clip_ratio": 0.0, "completion_length": 9.708333969116211, "epoch": 0.5645224171539961, "grad_norm": 1111.0698899104896, "kl": 51.9375, "learning_rate": 5.273263793206345e-07, "loss": 3.3511, "reward": 0.20131982117891312, "reward_std": 0.028339996933937073, "rewards/accuracy_reward": 0.20131982117891312, "step": 724 }, { "clip_ratio": 0.0, "completion_length": 8.125, "epoch": 0.5653021442495126, "grad_norm": 23.95313847129953, "kl": 3.6015625, "learning_rate": 5.261019020951106e-07, "loss": 0.3006, "reward": 0.21709401160478592, "reward_std": 0.021138248965144157, "rewards/accuracy_reward": 0.21709401160478592, "step": 725 }, { "clip_ratio": 0.0, "completion_length": 6.916666746139526, "epoch": 0.5660818713450292, "grad_norm": 34.92136709623118, "kl": 5.640625, "learning_rate": 5.2487760229025e-07, "loss": 0.2838, "reward": 0.18407223373651505, "reward_std": 0.02077169483527541, "rewards/accuracy_reward": 0.18407223373651505, "step": 726 }, { "clip_ratio": 0.0, "completion_length": 10.4375, "epoch": 0.5668615984405458, "grad_norm": 21.273447907786856, "kl": 4.2734375, "learning_rate": 5.236534889953149e-07, "loss": 0.7199, "reward": 0.19003383815288544, "reward_std": 0.025045832619071007, "rewards/accuracy_reward": 0.19003383815288544, "step": 727 }, { "clip_ratio": 0.0, "completion_length": 9.9375, "epoch": 0.5676413255360624, "grad_norm": 921.0420812355022, "kl": 46.65234375, "learning_rate": 5.224295712981829e-07, "loss": 3.088, "reward": 0.17517749965190887, "reward_std": 0.027015223167836666, "rewards/accuracy_reward": 0.17517749965190887, "step": 728 }, { "clip_ratio": 0.0, "completion_length": 50.3541693687439, "epoch": 0.5684210526315789, "grad_norm": 20.23029607238146, "kl": 2.765625, "learning_rate": 5.2120585828528e-07, "loss": 0.4956, "reward": 0.18020396679639816, "reward_std": 0.0344222504645586, "rewards/accuracy_reward": 0.18020396679639816, "step": 729 }, { "clip_ratio": 0.0, "completion_length": 9.666666984558105, "epoch": 0.5692007797270955, "grad_norm": 167.7698861790345, "kl": 8.21875, "learning_rate": 5.19982359041512e-07, "loss": 0.7493, "reward": 0.16905220597982407, "reward_std": 0.013457732740789652, "rewards/accuracy_reward": 0.16905220597982407, "step": 730 }, { "clip_ratio": 0.0, "completion_length": 7.729166746139526, "epoch": 0.569980506822612, "grad_norm": 26.219479385465597, "kl": 3.8359375, "learning_rate": 5.187590826501982e-07, "loss": 0.2268, "reward": 0.1933784931898117, "reward_std": 0.03980731964111328, "rewards/accuracy_reward": 0.1933784931898117, "step": 731 }, { "clip_ratio": 0.0, "completion_length": 14.541666984558105, "epoch": 0.5707602339181287, "grad_norm": 211.92177757459132, "kl": 12.78125, "learning_rate": 5.175360381930028e-07, "loss": 1.0304, "reward": 0.15167047083377838, "reward_std": 0.040984055027365685, "rewards/accuracy_reward": 0.15167047083377838, "step": 732 }, { "clip_ratio": 0.0, "completion_length": 9.4375, "epoch": 0.5715399610136452, "grad_norm": 46.23421803023925, "kl": 5.6015625, "learning_rate": 5.163132347498686e-07, "loss": 0.5211, "reward": 0.16398446261882782, "reward_std": 0.03464341349899769, "rewards/accuracy_reward": 0.16398446261882782, "step": 733 }, { "clip_ratio": 0.0, "completion_length": 7.770833492279053, "epoch": 0.5723196881091618, "grad_norm": 22.4583145542789, "kl": 3.953125, "learning_rate": 5.15090681398949e-07, "loss": 0.3211, "reward": 0.20709607005119324, "reward_std": 0.053975099697709084, "rewards/accuracy_reward": 0.20709607005119324, "step": 734 }, { "clip_ratio": 0.0, "completion_length": 6.979166746139526, "epoch": 0.5730994152046783, "grad_norm": 14.503801968147574, "kl": 3.0703125, "learning_rate": 5.138683872165408e-07, "loss": 0.118, "reward": 0.20572249591350555, "reward_std": 0.011753120459616184, "rewards/accuracy_reward": 0.20572249591350555, "step": 735 }, { "clip_ratio": 0.0, "completion_length": 50.9166693687439, "epoch": 0.5738791423001949, "grad_norm": 62.1279647946864, "kl": 6.0, "learning_rate": 5.126463612770161e-07, "loss": 0.8819, "reward": 0.19868828356266022, "reward_std": 0.03587975725531578, "rewards/accuracy_reward": 0.19868828356266022, "step": 736 }, { "clip_ratio": 0.0, "completion_length": 8.166666746139526, "epoch": 0.5746588693957115, "grad_norm": 15.643713565093693, "kl": 4.2734375, "learning_rate": 5.114246126527565e-07, "loss": 0.3693, "reward": 0.17953062057495117, "reward_std": 0.020495770033448935, "rewards/accuracy_reward": 0.17953062057495117, "step": 737 }, { "clip_ratio": 0.0, "completion_length": 7.625000238418579, "epoch": 0.5754385964912281, "grad_norm": 77.25183113273191, "kl": 7.546875, "learning_rate": 5.102031504140842e-07, "loss": 0.3659, "reward": 0.19230081140995026, "reward_std": 0.02740845491643995, "rewards/accuracy_reward": 0.19230081140995026, "step": 738 }, { "clip_ratio": 0.0, "completion_length": 7.395833492279053, "epoch": 0.5762183235867446, "grad_norm": 7.000475441817, "kl": 2.9765625, "learning_rate": 5.089819836291954e-07, "loss": 0.1014, "reward": 0.18896903097629547, "reward_std": 0.014748050831258297, "rewards/accuracy_reward": 0.18896903097629547, "step": 739 }, { "clip_ratio": 0.0, "completion_length": 7.625000238418579, "epoch": 0.5769980506822612, "grad_norm": 28.470134149326157, "kl": 3.8828125, "learning_rate": 5.077611213640928e-07, "loss": 0.2367, "reward": 0.1828887090086937, "reward_std": 0.01444562803953886, "rewards/accuracy_reward": 0.1828887090086937, "step": 740 }, { "clip_ratio": 0.0, "completion_length": 7.8125, "epoch": 0.5777777777777777, "grad_norm": 25.482433222694905, "kl": 6.109375, "learning_rate": 5.065405726825182e-07, "loss": 0.2564, "reward": 0.2014196440577507, "reward_std": 0.024420201312750578, "rewards/accuracy_reward": 0.2014196440577507, "step": 741 }, { "clip_ratio": 0.0, "completion_length": 7.3125, "epoch": 0.5785575048732944, "grad_norm": 833.6018682132706, "kl": 23.53515625, "learning_rate": 5.053203466458858e-07, "loss": 1.9449, "reward": 0.18860293924808502, "reward_std": 0.019932100549340248, "rewards/accuracy_reward": 0.18860293924808502, "step": 742 }, { "clip_ratio": 0.0, "completion_length": 10.729166984558105, "epoch": 0.5793372319688109, "grad_norm": 118.74058673592324, "kl": 12.5625, "learning_rate": 5.041004523132139e-07, "loss": 0.6016, "reward": 0.17768223583698273, "reward_std": 0.0218703243881464, "rewards/accuracy_reward": 0.17768223583698273, "step": 743 }, { "clip_ratio": 0.0, "completion_length": 7.145833492279053, "epoch": 0.5801169590643275, "grad_norm": 5.797584009416236, "kl": 3.09375, "learning_rate": 5.028808987410586e-07, "loss": 0.1029, "reward": 0.2204861119389534, "reward_std": 0.012501556426286697, "rewards/accuracy_reward": 0.2204861119389534, "step": 744 }, { "clip_ratio": 0.0, "completion_length": 8.291666984558105, "epoch": 0.580896686159844, "grad_norm": 33.51764348931488, "kl": 6.3359375, "learning_rate": 5.01661694983446e-07, "loss": 0.3254, "reward": 0.17308900505304337, "reward_std": 0.01642521470785141, "rewards/accuracy_reward": 0.17308900505304337, "step": 745 }, { "clip_ratio": 0.0, "completion_length": 8.125, "epoch": 0.5816764132553606, "grad_norm": 32.4677377313916, "kl": 6.046875, "learning_rate": 5.004428500918054e-07, "loss": 0.587, "reward": 0.16805507242679596, "reward_std": 0.027270404621958733, "rewards/accuracy_reward": 0.16805507242679596, "step": 746 }, { "clip_ratio": 0.0, "completion_length": 8.750000238418579, "epoch": 0.5824561403508772, "grad_norm": 12.9733401908474, "kl": 3.375, "learning_rate": 4.992243731149014e-07, "loss": 0.2741, "reward": 0.19418112188577652, "reward_std": 0.015036219730973244, "rewards/accuracy_reward": 0.19418112188577652, "step": 747 }, { "clip_ratio": 0.0, "completion_length": 9.562500476837158, "epoch": 0.5832358674463938, "grad_norm": 12.604818759759834, "kl": 4.96875, "learning_rate": 4.980062730987677e-07, "loss": 0.1591, "reward": 0.18438390642404556, "reward_std": 0.03260108456015587, "rewards/accuracy_reward": 0.18438390642404556, "step": 748 }, { "clip_ratio": 0.0, "completion_length": 8.541666984558105, "epoch": 0.5840155945419103, "grad_norm": 17.315812431905435, "kl": 4.796875, "learning_rate": 4.967885590866388e-07, "loss": 0.3777, "reward": 0.16472266614437103, "reward_std": 0.04542579501867294, "rewards/accuracy_reward": 0.16472266614437103, "step": 749 }, { "clip_ratio": 0.0, "completion_length": 8.250000238418579, "epoch": 0.5847953216374269, "grad_norm": 513.1154296842493, "kl": 35.796875, "learning_rate": 4.955712401188844e-07, "loss": 1.4642, "reward": 0.18676988035440445, "reward_std": 0.02362931426614523, "rewards/accuracy_reward": 0.18676988035440445, "step": 750 }, { "clip_ratio": 0.0, "completion_length": 8.041666984558105, "epoch": 0.5855750487329434, "grad_norm": 280.46370623111204, "kl": 11.25, "learning_rate": 4.943543252329406e-07, "loss": 1.4801, "reward": 0.19414883852005005, "reward_std": 0.03875085152685642, "rewards/accuracy_reward": 0.19414883852005005, "step": 751 }, { "clip_ratio": 0.0, "completion_length": 7.708333492279053, "epoch": 0.5863547758284601, "grad_norm": 29.89352853771839, "kl": 3.359375, "learning_rate": 4.931378234632438e-07, "loss": 0.254, "reward": 0.21168728172779083, "reward_std": 0.01946496916934848, "rewards/accuracy_reward": 0.21168728172779083, "step": 752 }, { "clip_ratio": 0.0, "completion_length": 10.187500238418579, "epoch": 0.5871345029239766, "grad_norm": 18.12528820630025, "kl": 3.609375, "learning_rate": 4.919217438411633e-07, "loss": 0.2833, "reward": 0.1397571787238121, "reward_std": 0.016146399080753326, "rewards/accuracy_reward": 0.1397571787238121, "step": 753 }, { "clip_ratio": 0.0, "completion_length": 8.291666984558105, "epoch": 0.5879142300194932, "grad_norm": 106.80881891467668, "kl": 5.22265625, "learning_rate": 4.907060953949347e-07, "loss": 0.666, "reward": 0.19727333635091782, "reward_std": 0.02822993416339159, "rewards/accuracy_reward": 0.19727333635091782, "step": 754 }, { "clip_ratio": 0.0, "completion_length": 9.083333492279053, "epoch": 0.5886939571150097, "grad_norm": 27.75147661184074, "kl": 3.3671875, "learning_rate": 4.894908871495922e-07, "loss": 0.4653, "reward": 0.1966698244214058, "reward_std": 0.01028318889439106, "rewards/accuracy_reward": 0.1966698244214058, "step": 755 }, { "clip_ratio": 0.0, "completion_length": 8.062500476837158, "epoch": 0.5894736842105263, "grad_norm": 10.722757471616433, "kl": 1.58984375, "learning_rate": 4.882761281269019e-07, "loss": 0.0505, "reward": 0.1849554032087326, "reward_std": 0.009616607800126076, "rewards/accuracy_reward": 0.1849554032087326, "step": 756 }, { "clip_ratio": 0.0, "completion_length": 6.7291669845581055, "epoch": 0.5902534113060429, "grad_norm": 102.8928844801119, "kl": 12.3125, "learning_rate": 4.870618273452952e-07, "loss": 0.6361, "reward": 0.2091982513666153, "reward_std": 0.04147295653820038, "rewards/accuracy_reward": 0.2091982513666153, "step": 757 }, { "clip_ratio": 0.0, "completion_length": 10.270833969116211, "epoch": 0.5910331384015595, "grad_norm": 12.046063435267275, "kl": 1.85546875, "learning_rate": 4.858479938198012e-07, "loss": 0.1541, "reward": 0.1697048619389534, "reward_std": 0.016207124979700893, "rewards/accuracy_reward": 0.1697048619389534, "step": 758 }, { "clip_ratio": 0.0, "completion_length": 7.8541669845581055, "epoch": 0.591812865497076, "grad_norm": 39.507207248358554, "kl": 6.3984375, "learning_rate": 4.8463463656198e-07, "loss": 0.4891, "reward": 0.1688900664448738, "reward_std": 0.040857626125216484, "rewards/accuracy_reward": 0.1688900664448738, "step": 759 }, { "clip_ratio": 0.0, "completion_length": 7.729166746139526, "epoch": 0.5925925925925926, "grad_norm": 11.626844783892748, "kl": 2.19140625, "learning_rate": 4.834217645798564e-07, "loss": 0.1351, "reward": 0.19343341886997223, "reward_std": 0.01102245319634676, "rewards/accuracy_reward": 0.19343341886997223, "step": 760 }, { "clip_ratio": 0.0, "completion_length": 7.958333492279053, "epoch": 0.5933723196881091, "grad_norm": 57.713097720505786, "kl": 7.0, "learning_rate": 4.822093868778517e-07, "loss": 0.5137, "reward": 0.19546708464622498, "reward_std": 0.03336811903864145, "rewards/accuracy_reward": 0.19546708464622498, "step": 761 }, { "clip_ratio": 0.0, "completion_length": 10.25, "epoch": 0.5941520467836258, "grad_norm": 11.053631456072825, "kl": 3.890625, "learning_rate": 4.809975124567184e-07, "loss": 0.1512, "reward": 0.17614223062992096, "reward_std": 0.029737313278019428, "rewards/accuracy_reward": 0.17614223062992096, "step": 762 }, { "clip_ratio": 0.0, "completion_length": 7.875000476837158, "epoch": 0.5949317738791423, "grad_norm": 13.964064600395327, "kl": 5.109375, "learning_rate": 4.797861503134719e-07, "loss": 0.4196, "reward": 0.19560137391090393, "reward_std": 0.0304703451693058, "rewards/accuracy_reward": 0.19560137391090393, "step": 763 }, { "clip_ratio": 0.0, "completion_length": 7.791666746139526, "epoch": 0.5957115009746589, "grad_norm": 5.810858456372481, "kl": 2.6953125, "learning_rate": 4.78575309441325e-07, "loss": 0.0868, "reward": 0.20079366117715836, "reward_std": 0.009622504934668541, "rewards/accuracy_reward": 0.20079366117715836, "step": 764 }, { "clip_ratio": 0.0, "completion_length": 7.395833492279053, "epoch": 0.5964912280701754, "grad_norm": 1049.0042068010284, "kl": 59.2265625, "learning_rate": 4.773649988296201e-07, "loss": 2.2499, "reward": 0.21168051660060883, "reward_std": 0.03866158239543438, "rewards/accuracy_reward": 0.21168051660060883, "step": 765 }, { "clip_ratio": 0.0, "completion_length": 7.416666746139526, "epoch": 0.597270955165692, "grad_norm": 21.85254158598344, "kl": 3.078125, "learning_rate": 4.7615522746376345e-07, "loss": 0.2842, "reward": 0.2145531252026558, "reward_std": 0.016547015868127346, "rewards/accuracy_reward": 0.2145531252026558, "step": 766 }, { "clip_ratio": 0.0, "completion_length": 7.25, "epoch": 0.5980506822612086, "grad_norm": 10.024992837660362, "kl": 4.03125, "learning_rate": 4.7494600432515754e-07, "loss": 0.193, "reward": 0.17795942723751068, "reward_std": 0.022083266347181052, "rewards/accuracy_reward": 0.17795942723751068, "step": 767 }, { "clip_ratio": 0.0, "completion_length": 10.000000476837158, "epoch": 0.5988304093567252, "grad_norm": 7.8565078659367, "kl": 1.14453125, "learning_rate": 4.737373383911347e-07, "loss": 0.0518, "reward": 0.1793087124824524, "reward_std": 0.0058309040032327175, "rewards/accuracy_reward": 0.1793087124824524, "step": 768 }, { "clip_ratio": 0.0, "completion_length": 9.354166984558105, "epoch": 0.5996101364522417, "grad_norm": 79.39213074490343, "kl": 7.0625, "learning_rate": 4.72529238634891e-07, "loss": 0.4959, "reward": 0.16901394724845886, "reward_std": 0.01649160380475223, "rewards/accuracy_reward": 0.16901394724845886, "step": 769 }, { "clip_ratio": 0.0, "completion_length": 6.791666746139526, "epoch": 0.6003898635477583, "grad_norm": 17.942624401665725, "kl": 4.75, "learning_rate": 4.7132171402541865e-07, "loss": 0.1882, "reward": 0.22621528804302216, "reward_std": 0.025411183014512062, "rewards/accuracy_reward": 0.22621528804302216, "step": 770 }, { "clip_ratio": 0.0, "completion_length": 8.166666746139526, "epoch": 0.6011695906432749, "grad_norm": 29.255201760504846, "kl": 4.4375, "learning_rate": 4.7011477352744034e-07, "loss": 0.3436, "reward": 0.1923586130142212, "reward_std": 0.012202110141515732, "rewards/accuracy_reward": 0.1923586130142212, "step": 771 }, { "clip_ratio": 0.0, "completion_length": 7.833333492279053, "epoch": 0.6019493177387915, "grad_norm": 59.112660107637076, "kl": 10.53125, "learning_rate": 4.689084261013421e-07, "loss": 0.6029, "reward": 0.20741159468889236, "reward_std": 0.02666310779750347, "rewards/accuracy_reward": 0.20741159468889236, "step": 772 }, { "clip_ratio": 0.0, "completion_length": 6.958333492279053, "epoch": 0.602729044834308, "grad_norm": 68.00304081397594, "kl": 10.71875, "learning_rate": 4.677026807031067e-07, "loss": 0.55, "reward": 0.20736417174339294, "reward_std": 0.03963995724916458, "rewards/accuracy_reward": 0.20736417174339294, "step": 773 }, { "clip_ratio": 0.0, "completion_length": 7.3541669845581055, "epoch": 0.6035087719298246, "grad_norm": 15.505441590336915, "kl": 4.671875, "learning_rate": 4.664975462842483e-07, "loss": 0.2714, "reward": 0.1954365149140358, "reward_std": 0.007877668365836143, "rewards/accuracy_reward": 0.1954365149140358, "step": 774 }, { "clip_ratio": 0.0, "completion_length": 8.750000476837158, "epoch": 0.6042884990253411, "grad_norm": 45.13047924003801, "kl": 5.09375, "learning_rate": 4.652930317917443e-07, "loss": 0.448, "reward": 0.19026663899421692, "reward_std": 0.02981062512844801, "rewards/accuracy_reward": 0.19026663899421692, "step": 775 }, { "clip_ratio": 0.0, "completion_length": 7.7916669845581055, "epoch": 0.6050682261208578, "grad_norm": 7.958277092307518, "kl": 1.8984375, "learning_rate": 4.640891461679698e-07, "loss": 0.0864, "reward": 0.19340278208255768, "reward_std": 0.006357727572321892, "rewards/accuracy_reward": 0.19340278208255768, "step": 776 }, { "clip_ratio": 0.0, "completion_length": 6.916666746139526, "epoch": 0.6058479532163743, "grad_norm": 560.4114274304104, "kl": 19.1328125, "learning_rate": 4.6288589835063155e-07, "loss": 1.3135, "reward": 0.19378283619880676, "reward_std": 0.017120971344411373, "rewards/accuracy_reward": 0.19378283619880676, "step": 777 }, { "clip_ratio": 0.0, "completion_length": 7.333333492279053, "epoch": 0.6066276803118908, "grad_norm": 14.676701775679524, "kl": 4.375, "learning_rate": 4.616832972727009e-07, "loss": 0.3108, "reward": 0.19689543545246124, "reward_std": 0.024047876708209515, "rewards/accuracy_reward": 0.19689543545246124, "step": 778 }, { "clip_ratio": 0.0, "completion_length": 6.541666746139526, "epoch": 0.6074074074074074, "grad_norm": 28.79449619848036, "kl": 5.3671875, "learning_rate": 4.604813518623478e-07, "loss": 0.1833, "reward": 0.2140377089381218, "reward_std": 0.02147880382835865, "rewards/accuracy_reward": 0.2140377089381218, "step": 779 }, { "clip_ratio": 0.0, "completion_length": 7.083333492279053, "epoch": 0.6081871345029239, "grad_norm": 3.4839908125831687, "kl": 3.8359375, "learning_rate": 4.592800710428747e-07, "loss": 0.1725, "reward": 0.2162698581814766, "reward_std": 0.007877668365836143, "rewards/accuracy_reward": 0.2162698581814766, "step": 780 }, { "clip_ratio": 0.0, "completion_length": 7.375000238418579, "epoch": 0.6089668615984406, "grad_norm": 11.339906943788513, "kl": 2.7265625, "learning_rate": 4.580794637326497e-07, "loss": 0.1529, "reward": 0.195563443005085, "reward_std": 0.017552451230585575, "rewards/accuracy_reward": 0.195563443005085, "step": 781 }, { "clip_ratio": 0.0, "completion_length": 10.375, "epoch": 0.6097465886939571, "grad_norm": 16.706542975142643, "kl": 3.0859375, "learning_rate": 4.5687953884504117e-07, "loss": 0.2591, "reward": 0.1861708089709282, "reward_std": 0.033440083265304565, "rewards/accuracy_reward": 0.1861708089709282, "step": 782 }, { "clip_ratio": 0.0, "completion_length": 9.166666746139526, "epoch": 0.6105263157894737, "grad_norm": 10.366552350377624, "kl": 2.984375, "learning_rate": 4.556803052883508e-07, "loss": 0.3351, "reward": 0.1939774826169014, "reward_std": 0.01127982884645462, "rewards/accuracy_reward": 0.1939774826169014, "step": 783 }, { "clip_ratio": 0.0, "completion_length": 7.1666669845581055, "epoch": 0.6113060428849902, "grad_norm": 6.820595275329998, "kl": 2.48828125, "learning_rate": 4.54481771965748e-07, "loss": 0.0946, "reward": 0.2082669585943222, "reward_std": 0.013235677033662796, "rewards/accuracy_reward": 0.2082669585943222, "step": 784 }, { "clip_ratio": 0.0, "completion_length": 9.187500476837158, "epoch": 0.6120857699805068, "grad_norm": 8.114162777111806, "kl": 2.3046875, "learning_rate": 4.532839477752035e-07, "loss": 0.0906, "reward": 0.19740156829357147, "reward_std": 0.014632525853812695, "rewards/accuracy_reward": 0.19740156829357147, "step": 785 }, { "clip_ratio": 0.0, "completion_length": 7.2916669845581055, "epoch": 0.6128654970760234, "grad_norm": 51.445169185273556, "kl": 7.5078125, "learning_rate": 4.520868416094236e-07, "loss": 0.3441, "reward": 0.20311260968446732, "reward_std": 0.026415514759719372, "rewards/accuracy_reward": 0.20311260968446732, "step": 786 }, { "clip_ratio": 0.0, "completion_length": 7.458333730697632, "epoch": 0.61364522417154, "grad_norm": 24.493535986190917, "kl": 3.625, "learning_rate": 4.5089046235578345e-07, "loss": 0.19, "reward": 0.19780371338129044, "reward_std": 0.016349583864212036, "rewards/accuracy_reward": 0.19780371338129044, "step": 787 }, { "clip_ratio": 0.0, "completion_length": 10.208333492279053, "epoch": 0.6144249512670565, "grad_norm": 6.395559042532656, "kl": 2.34375, "learning_rate": 4.4969481889626224e-07, "loss": 0.0852, "reward": 0.19479166716337204, "reward_std": 0.006966505199670792, "rewards/accuracy_reward": 0.19479166716337204, "step": 788 }, { "clip_ratio": 0.0, "completion_length": 7.5416669845581055, "epoch": 0.6152046783625731, "grad_norm": 9.290677501031373, "kl": 3.5390625, "learning_rate": 4.4849992010737625e-07, "loss": 0.229, "reward": 0.17592810839414597, "reward_std": 0.022496329620480537, "rewards/accuracy_reward": 0.17592810839414597, "step": 789 }, { "clip_ratio": 0.0, "completion_length": 9.916666746139526, "epoch": 0.6159844054580896, "grad_norm": 94.37831153244247, "kl": 8.9296875, "learning_rate": 4.4730577486011333e-07, "loss": 0.4132, "reward": 0.19950980693101883, "reward_std": 0.03226372506469488, "rewards/accuracy_reward": 0.19950980693101883, "step": 790 }, { "clip_ratio": 0.0, "completion_length": 6.5625, "epoch": 0.6167641325536063, "grad_norm": 5.581001454717509, "kl": 3.0, "learning_rate": 4.4611239201986685e-07, "loss": 0.1298, "reward": 0.20035651326179504, "reward_std": 0.0013121606316417456, "rewards/accuracy_reward": 0.20035651326179504, "step": 791 }, { "clip_ratio": 0.0, "completion_length": 7.770833492279053, "epoch": 0.6175438596491228, "grad_norm": 15.291703216686232, "kl": 3.1640625, "learning_rate": 4.4491978044637015e-07, "loss": 0.2724, "reward": 0.20971546322107315, "reward_std": 0.01972260605543852, "rewards/accuracy_reward": 0.20971546322107315, "step": 792 }, { "clip_ratio": 0.0, "completion_length": 7.5416669845581055, "epoch": 0.6183235867446394, "grad_norm": 84.93749548444256, "kl": 13.34375, "learning_rate": 4.437279489936306e-07, "loss": 0.5183, "reward": 0.20228589326143265, "reward_std": 0.030025272630155087, "rewards/accuracy_reward": 0.20228589326143265, "step": 793 }, { "clip_ratio": 0.0, "completion_length": 8.083333492279053, "epoch": 0.6191033138401559, "grad_norm": 29.10800078204113, "kl": 5.296875, "learning_rate": 4.4253690650986417e-07, "loss": 0.4785, "reward": 0.19759321957826614, "reward_std": 0.019533390179276466, "rewards/accuracy_reward": 0.19759321957826614, "step": 794 }, { "clip_ratio": 0.0, "completion_length": 10.270833492279053, "epoch": 0.6198830409356725, "grad_norm": 383.067637919153, "kl": 17.21875, "learning_rate": 4.413466618374291e-07, "loss": 1.3129, "reward": 0.19665924459695816, "reward_std": 0.03253862354904413, "rewards/accuracy_reward": 0.19665924459695816, "step": 795 }, { "clip_ratio": 0.0, "completion_length": 7.7916669845581055, "epoch": 0.6206627680311891, "grad_norm": 10.216804573778802, "kl": 2.7578125, "learning_rate": 4.4015722381276086e-07, "loss": 0.1883, "reward": 0.2028769925236702, "reward_std": 0.0024056262336671352, "rewards/accuracy_reward": 0.2028769925236702, "step": 796 }, { "clip_ratio": 0.0, "completion_length": 8.395833492279053, "epoch": 0.6214424951267057, "grad_norm": 385.8434929742846, "kl": 25.375, "learning_rate": 4.389686012663063e-07, "loss": 1.3037, "reward": 0.2043689414858818, "reward_std": 0.023358877166174352, "rewards/accuracy_reward": 0.2043689414858818, "step": 797 }, { "clip_ratio": 0.0, "completion_length": 8.187500238418579, "epoch": 0.6222222222222222, "grad_norm": 38.59957111208649, "kl": 6.484375, "learning_rate": 4.377808030224581e-07, "loss": 0.3835, "reward": 0.2030586376786232, "reward_std": 0.030849136412143707, "rewards/accuracy_reward": 0.2030586376786232, "step": 798 }, { "clip_ratio": 0.0, "completion_length": 8.333333492279053, "epoch": 0.6230019493177388, "grad_norm": 107.42478125369642, "kl": 9.265625, "learning_rate": 4.3659383789948934e-07, "loss": 0.539, "reward": 0.19165126979351044, "reward_std": 0.01961417868733406, "rewards/accuracy_reward": 0.19165126979351044, "step": 799 }, { "clip_ratio": 0.0, "completion_length": 8.958333492279053, "epoch": 0.6237816764132553, "grad_norm": 19.352124542532792, "kl": 5.921875, "learning_rate": 4.35407714709488e-07, "loss": 0.2386, "reward": 0.1721200942993164, "reward_std": 0.047283776104450226, "rewards/accuracy_reward": 0.1721200942993164, "step": 800 }, { "clip_ratio": 0.0, "completion_length": 7.229166746139526, "epoch": 0.624561403508772, "grad_norm": 42.39111872358708, "kl": 5.1875, "learning_rate": 4.342224422582911e-07, "loss": 0.3254, "reward": 0.2216346263885498, "reward_std": 0.011658036150038242, "rewards/accuracy_reward": 0.2216346263885498, "step": 801 }, { "clip_ratio": 0.0, "completion_length": 7.5, "epoch": 0.6253411306042885, "grad_norm": 42.29530611712892, "kl": 5.296875, "learning_rate": 4.330380293454202e-07, "loss": 0.2243, "reward": 0.19074634462594986, "reward_std": 0.023415545001626015, "rewards/accuracy_reward": 0.19074634462594986, "step": 802 }, { "clip_ratio": 0.0, "completion_length": 7.25, "epoch": 0.6261208576998051, "grad_norm": 19.1864664407353, "kl": 5.828125, "learning_rate": 4.318544847640158e-07, "loss": 0.3136, "reward": 0.2264232411980629, "reward_std": 0.0345494095236063, "rewards/accuracy_reward": 0.2264232411980629, "step": 803 }, { "clip_ratio": 0.0, "completion_length": 8.104166984558105, "epoch": 0.6269005847953216, "grad_norm": 12.053964076178366, "kl": 5.96875, "learning_rate": 4.30671817300771e-07, "loss": 0.401, "reward": 0.19188113510608673, "reward_std": 0.04166575148701668, "rewards/accuracy_reward": 0.19188113510608673, "step": 804 }, { "clip_ratio": 0.0, "completion_length": 7.875000238418579, "epoch": 0.6276803118908382, "grad_norm": 15.369913762064051, "kl": 2.7109375, "learning_rate": 4.2949003573586785e-07, "loss": 0.1415, "reward": 0.1992822140455246, "reward_std": 0.017546577379107475, "rewards/accuracy_reward": 0.1992822140455246, "step": 805 }, { "clip_ratio": 0.0, "completion_length": 8.312500476837158, "epoch": 0.6284600389863548, "grad_norm": 20.08835528033312, "kl": 3.359375, "learning_rate": 4.283091488429115e-07, "loss": 0.3665, "reward": 0.19808261841535568, "reward_std": 0.019510054029524326, "rewards/accuracy_reward": 0.19808261841535568, "step": 806 }, { "clip_ratio": 0.0, "completion_length": 8.041666984558105, "epoch": 0.6292397660818714, "grad_norm": 14.746151854082994, "kl": 3.1484375, "learning_rate": 4.271291653888642e-07, "loss": 0.2432, "reward": 0.20466720312833786, "reward_std": 0.014170052949339151, "rewards/accuracy_reward": 0.20466720312833786, "step": 807 }, { "clip_ratio": 0.0, "completion_length": 7.375000238418579, "epoch": 0.6300194931773879, "grad_norm": 9.651169896639045, "kl": 3.734375, "learning_rate": 4.259500941339822e-07, "loss": 0.2547, "reward": 0.20700864493846893, "reward_std": 0.014448235742747784, "rewards/accuracy_reward": 0.20700864493846893, "step": 808 }, { "clip_ratio": 0.0, "completion_length": 7.770833492279053, "epoch": 0.6307992202729045, "grad_norm": 136.78566738068835, "kl": 11.65625, "learning_rate": 4.2477194383174863e-07, "loss": 0.6354, "reward": 0.20887459069490433, "reward_std": 0.02471503522247076, "rewards/accuracy_reward": 0.20887459069490433, "step": 809 }, { "clip_ratio": 0.0, "completion_length": 50.04166793823242, "epoch": 0.631578947368421, "grad_norm": 167.52586776433836, "kl": 8.140625, "learning_rate": 4.2359472322880973e-07, "loss": 1.1554, "reward": 0.20988915115594864, "reward_std": 0.03373613581061363, "rewards/accuracy_reward": 0.20988915115594864, "step": 810 }, { "clip_ratio": 0.0, "completion_length": 6.729166746139526, "epoch": 0.6323586744639377, "grad_norm": 11.30581276764659, "kl": 3.875, "learning_rate": 4.2241844106490967e-07, "loss": 0.1823, "reward": 0.21582088619470596, "reward_std": 0.01094868266955018, "rewards/accuracy_reward": 0.21582088619470596, "step": 811 }, { "clip_ratio": 0.0, "completion_length": 7.7916669845581055, "epoch": 0.6331384015594542, "grad_norm": 280.4746717335008, "kl": 15.515625, "learning_rate": 4.2124310607282543e-07, "loss": 1.2256, "reward": 0.20998981595039368, "reward_std": 0.034942809492349625, "rewards/accuracy_reward": 0.20998981595039368, "step": 812 }, { "clip_ratio": 0.0, "completion_length": 8.208333969116211, "epoch": 0.6339181286549708, "grad_norm": 14.719046726718693, "kl": 2.52734375, "learning_rate": 4.200687269783023e-07, "loss": 0.1315, "reward": 0.1912563219666481, "reward_std": 0.018876119516789913, "rewards/accuracy_reward": 0.1912563219666481, "step": 813 }, { "clip_ratio": 0.0, "completion_length": 9.083333492279053, "epoch": 0.6346978557504873, "grad_norm": 80.5964186128411, "kl": 12.34375, "learning_rate": 4.188953124999888e-07, "loss": 0.8233, "reward": 0.1867903769016266, "reward_std": 0.02752780169248581, "rewards/accuracy_reward": 0.1867903769016266, "step": 814 }, { "clip_ratio": 0.0, "completion_length": 7.750000476837158, "epoch": 0.6354775828460039, "grad_norm": 39.42691012375579, "kl": 8.1640625, "learning_rate": 4.1772287134937223e-07, "loss": 0.4886, "reward": 0.19144070148468018, "reward_std": 0.020385267212986946, "rewards/accuracy_reward": 0.19144070148468018, "step": 815 }, { "clip_ratio": 0.0, "completion_length": 49.41666793823242, "epoch": 0.6362573099415205, "grad_norm": 11.547947822180952, "kl": 4.609375, "learning_rate": 4.1655141223071366e-07, "loss": 0.6891, "reward": 0.20407475531101227, "reward_std": 0.03265130240470171, "rewards/accuracy_reward": 0.20407475531101227, "step": 816 }, { "clip_ratio": 0.0, "completion_length": 8.187500476837158, "epoch": 0.6370370370370371, "grad_norm": 24.576295798710138, "kl": 7.1015625, "learning_rate": 4.1538094384098363e-07, "loss": 0.4078, "reward": 0.1960323303937912, "reward_std": 0.025398317025974393, "rewards/accuracy_reward": 0.1960323303937912, "step": 817 }, { "clip_ratio": 0.0, "completion_length": 6.208333492279053, "epoch": 0.6378167641325536, "grad_norm": 16.855327869999773, "kl": 4.71875, "learning_rate": 4.142114748697979e-07, "loss": 0.2384, "reward": 0.21702534705400467, "reward_std": 0.02624906785786152, "rewards/accuracy_reward": 0.21702534705400467, "step": 818 }, { "clip_ratio": 0.0, "completion_length": 7.145833492279053, "epoch": 0.6385964912280702, "grad_norm": 18.606683912213757, "kl": 4.984375, "learning_rate": 4.130430139993516e-07, "loss": 0.3394, "reward": 0.2314903885126114, "reward_std": 0.016296265181154013, "rewards/accuracy_reward": 0.2314903885126114, "step": 819 }, { "clip_ratio": 0.0, "completion_length": 7.625000238418579, "epoch": 0.6393762183235867, "grad_norm": 35.95326779247107, "kl": 5.5, "learning_rate": 4.1187556990435635e-07, "loss": 0.3098, "reward": 0.2029413878917694, "reward_std": 0.01674424426164478, "rewards/accuracy_reward": 0.2029413878917694, "step": 820 }, { "clip_ratio": 0.0, "completion_length": 8.729166984558105, "epoch": 0.6401559454191034, "grad_norm": 196.53792273316563, "kl": 15.46875, "learning_rate": 4.1070915125197504e-07, "loss": 0.8768, "reward": 0.19404908269643784, "reward_std": 0.02998365741223097, "rewards/accuracy_reward": 0.19404908269643784, "step": 821 }, { "clip_ratio": 0.0, "completion_length": 6.8125, "epoch": 0.6409356725146199, "grad_norm": 5.60700152481248, "kl": 3.3671875, "learning_rate": 4.0954376670175727e-07, "loss": 0.1593, "reward": 0.2088274285197258, "reward_std": 0.004716914147138596, "rewards/accuracy_reward": 0.2088274285197258, "step": 822 }, { "clip_ratio": 0.0, "completion_length": 7.145833492279053, "epoch": 0.6417153996101365, "grad_norm": 112.55388322425284, "kl": 12.453125, "learning_rate": 4.0837942490557577e-07, "loss": 0.6601, "reward": 0.22587250918149948, "reward_std": 0.034454891458153725, "rewards/accuracy_reward": 0.22587250918149948, "step": 823 }, { "clip_ratio": 0.0, "completion_length": 7.520833492279053, "epoch": 0.642495126705653, "grad_norm": 20.777146687622558, "kl": 4.109375, "learning_rate": 4.0721613450756196e-07, "loss": 0.2603, "reward": 0.22010359913110733, "reward_std": 0.02082142001017928, "rewards/accuracy_reward": 0.22010359913110733, "step": 824 }, { "clip_ratio": 0.0, "completion_length": 7.979166746139526, "epoch": 0.6432748538011696, "grad_norm": 51.26771572352514, "kl": 5.203125, "learning_rate": 4.0605390414404117e-07, "loss": 0.2251, "reward": 0.19832590222358704, "reward_std": 0.017583255423232913, "rewards/accuracy_reward": 0.19832590222358704, "step": 825 }, { "clip_ratio": 0.0, "completion_length": 8.083333492279053, "epoch": 0.6440545808966862, "grad_norm": 69.41846787049884, "kl": 6.06640625, "learning_rate": 4.0489274244346925e-07, "loss": 0.2655, "reward": 0.19951199740171432, "reward_std": 0.020145550835877657, "rewards/accuracy_reward": 0.19951199740171432, "step": 826 }, { "clip_ratio": 0.0, "completion_length": 7.812500238418579, "epoch": 0.6448343079922028, "grad_norm": 23.910114426636852, "kl": 4.71875, "learning_rate": 4.037326580263682e-07, "loss": 0.3594, "reward": 0.19929338246583939, "reward_std": 0.019803845789283514, "rewards/accuracy_reward": 0.19929338246583939, "step": 827 }, { "clip_ratio": 0.0, "completion_length": 7.125000238418579, "epoch": 0.6456140350877193, "grad_norm": 19.421358737779105, "kl": 3.2421875, "learning_rate": 4.0257365950526243e-07, "loss": 0.1116, "reward": 0.19143907725811005, "reward_std": 0.013990202452987432, "rewards/accuracy_reward": 0.19143907725811005, "step": 828 }, { "clip_ratio": 0.0, "completion_length": 7.583333492279053, "epoch": 0.6463937621832359, "grad_norm": 7.941813633380404, "kl": 2.8671875, "learning_rate": 4.0141575548461403e-07, "loss": 0.1071, "reward": 0.2020089402794838, "reward_std": 0.005412658676505089, "rewards/accuracy_reward": 0.2020089402794838, "step": 829 }, { "clip_ratio": 0.0, "completion_length": 8.666666984558105, "epoch": 0.6471734892787524, "grad_norm": 11.737896987154013, "kl": 3.390625, "learning_rate": 4.002589545607602e-07, "loss": 0.1886, "reward": 0.19675233960151672, "reward_std": 0.013309437199495733, "rewards/accuracy_reward": 0.19675233960151672, "step": 830 }, { "clip_ratio": 0.0, "completion_length": 17.812500476837158, "epoch": 0.6479532163742691, "grad_norm": 18.366980925881403, "kl": 5.046875, "learning_rate": 3.99103265321848e-07, "loss": 0.496, "reward": 0.20842014253139496, "reward_std": 0.05011184327304363, "rewards/accuracy_reward": 0.20842014253139496, "step": 831 }, { "clip_ratio": 0.0, "completion_length": 9.020833969116211, "epoch": 0.6487329434697856, "grad_norm": 13.405002609765074, "kl": 2.76171875, "learning_rate": 3.9794869634777174e-07, "loss": 0.2071, "reward": 0.1993274837732315, "reward_std": 0.012340146116912365, "rewards/accuracy_reward": 0.1993274837732315, "step": 832 }, { "clip_ratio": 0.0, "completion_length": 6.791666746139526, "epoch": 0.6495126705653022, "grad_norm": 15.494509868888603, "kl": 4.234375, "learning_rate": 3.967952562101088e-07, "loss": 0.1557, "reward": 0.22708333283662796, "reward_std": 0.0051549142226576805, "rewards/accuracy_reward": 0.22708333283662796, "step": 833 }, { "clip_ratio": 0.0, "completion_length": 7.104166746139526, "epoch": 0.6502923976608187, "grad_norm": 2.084070941523775, "kl": 3.0078125, "learning_rate": 3.956429534720559e-07, "loss": 0.1195, "reward": 0.21607143431901932, "reward_std": 0.0, "rewards/accuracy_reward": 0.21607143431901932, "step": 834 }, { "clip_ratio": 0.0, "completion_length": 8.187500476837158, "epoch": 0.6510721247563352, "grad_norm": 8.08640444249246, "kl": 2.2421875, "learning_rate": 3.944917966883655e-07, "loss": 0.1531, "reward": 0.2024982050061226, "reward_std": 0.0026334156282246113, "rewards/accuracy_reward": 0.2024982050061226, "step": 835 }, { "clip_ratio": 0.0, "completion_length": 7.312500238418579, "epoch": 0.6518518518518519, "grad_norm": 13.786405653279184, "kl": 3.6796875, "learning_rate": 3.933417944052828e-07, "loss": 0.1478, "reward": 0.22604165971279144, "reward_std": 0.01539972797036171, "rewards/accuracy_reward": 0.22604165971279144, "step": 836 }, { "clip_ratio": 0.0, "completion_length": 6.041666746139526, "epoch": 0.6526315789473685, "grad_norm": 1.1748814247215527, "kl": 3.6015625, "learning_rate": 3.9219295516048155e-07, "loss": 0.1442, "reward": 0.19485295563936234, "reward_std": 0.0, "rewards/accuracy_reward": 0.19485295563936234, "step": 837 }, { "clip_ratio": 0.0, "completion_length": 8.145833492279053, "epoch": 0.653411306042885, "grad_norm": 2237.912229922971, "kl": 139.1875, "learning_rate": 3.910452874830015e-07, "loss": 4.8967, "reward": 0.1780773475766182, "reward_std": 0.03933209553360939, "rewards/accuracy_reward": 0.1780773475766182, "step": 838 }, { "clip_ratio": 0.0, "completion_length": 7.6666669845581055, "epoch": 0.6541910331384015, "grad_norm": 10.473272553926513, "kl": 4.1953125, "learning_rate": 3.8989879989318407e-07, "loss": 0.1865, "reward": 0.20865502208471298, "reward_std": 0.023965464439243078, "rewards/accuracy_reward": 0.20865502208471298, "step": 839 }, { "clip_ratio": 0.0, "completion_length": 7.604166746139526, "epoch": 0.6549707602339181, "grad_norm": 83.32627325211686, "kl": 8.25, "learning_rate": 3.8875350090261014e-07, "loss": 0.3729, "reward": 0.21747076511383057, "reward_std": 0.016058134846389294, "rewards/accuracy_reward": 0.21747076511383057, "step": 840 }, { "clip_ratio": 0.0, "completion_length": 7.4791669845581055, "epoch": 0.6557504873294348, "grad_norm": 58.93143334333492, "kl": 7.03125, "learning_rate": 3.8760939901403613e-07, "loss": 0.5712, "reward": 0.21835222840309143, "reward_std": 0.030122676864266396, "rewards/accuracy_reward": 0.21835222840309143, "step": 841 }, { "clip_ratio": 0.0, "completion_length": 7.229166746139526, "epoch": 0.6565302144249513, "grad_norm": 71.40128600202888, "kl": 8.390625, "learning_rate": 3.864665027213311e-07, "loss": 0.5488, "reward": 0.20483766496181488, "reward_std": 0.023112010210752487, "rewards/accuracy_reward": 0.20483766496181488, "step": 842 }, { "clip_ratio": 0.0, "completion_length": 8.916666984558105, "epoch": 0.6573099415204678, "grad_norm": 81.00529931390395, "kl": 12.046875, "learning_rate": 3.8532482050941374e-07, "loss": 0.7515, "reward": 0.186067596077919, "reward_std": 0.03991219587624073, "rewards/accuracy_reward": 0.186067596077919, "step": 843 }, { "clip_ratio": 0.0, "completion_length": 8.000000238418579, "epoch": 0.6580896686159844, "grad_norm": 28.540090037116666, "kl": 5.8125, "learning_rate": 3.8418436085418935e-07, "loss": 0.3534, "reward": 0.1989043578505516, "reward_std": 0.026317469775676727, "rewards/accuracy_reward": 0.1989043578505516, "step": 844 }, { "clip_ratio": 0.0, "completion_length": 9.375000476837158, "epoch": 0.6588693957115009, "grad_norm": 49.22818092955027, "kl": 10.984375, "learning_rate": 3.8304513222248713e-07, "loss": 0.4965, "reward": 0.18014463782310486, "reward_std": 0.014690164476633072, "rewards/accuracy_reward": 0.18014463782310486, "step": 845 }, { "clip_ratio": 0.0, "completion_length": 7.2916669845581055, "epoch": 0.6596491228070176, "grad_norm": 41.01620676230909, "kl": 5.2421875, "learning_rate": 3.8190714307199655e-07, "loss": 0.353, "reward": 0.20514292269945145, "reward_std": 0.01859601680189371, "rewards/accuracy_reward": 0.20514292269945145, "step": 846 }, { "clip_ratio": 0.0, "completion_length": 8.145833730697632, "epoch": 0.6604288499025341, "grad_norm": 15.50305603216997, "kl": 4.171875, "learning_rate": 3.807704018512059e-07, "loss": 0.2939, "reward": 0.2043001651763916, "reward_std": 0.023173989728093147, "rewards/accuracy_reward": 0.2043001651763916, "step": 847 }, { "clip_ratio": 0.0, "completion_length": 6.895833492279053, "epoch": 0.6612085769980507, "grad_norm": 9.893991042900543, "kl": 3.4296875, "learning_rate": 3.79634916999338e-07, "loss": 0.1809, "reward": 0.21078431606292725, "reward_std": 0.005943312309682369, "rewards/accuracy_reward": 0.21078431606292725, "step": 848 }, { "clip_ratio": 0.0, "completion_length": 7.520833492279053, "epoch": 0.6619883040935672, "grad_norm": 31.156907102921853, "kl": 5.1015625, "learning_rate": 3.7850069694628884e-07, "loss": 0.3965, "reward": 0.20739000290632248, "reward_std": 0.025404288433492184, "rewards/accuracy_reward": 0.20739000290632248, "step": 849 }, { "clip_ratio": 0.0, "completion_length": 7.083333492279053, "epoch": 0.6627680311890838, "grad_norm": 11.988670132221133, "kl": 4.5625, "learning_rate": 3.7736775011256415e-07, "loss": 0.1814, "reward": 0.20704366266727448, "reward_std": 0.020715244114398956, "rewards/accuracy_reward": 0.20704366266727448, "step": 850 }, { "clip_ratio": 0.0, "completion_length": 6.729166746139526, "epoch": 0.6635477582846004, "grad_norm": 15.094477497666018, "kl": 5.875, "learning_rate": 3.762360849092175e-07, "loss": 0.2685, "reward": 0.1950061321258545, "reward_std": 0.027128692716360092, "rewards/accuracy_reward": 0.1950061321258545, "step": 851 }, { "clip_ratio": 0.0, "completion_length": 8.666666984558105, "epoch": 0.664327485380117, "grad_norm": 47.49385296307977, "kl": 5.59375, "learning_rate": 3.7510570973778743e-07, "loss": 0.5546, "reward": 0.19470486044883728, "reward_std": 0.04438193514943123, "rewards/accuracy_reward": 0.19470486044883728, "step": 852 }, { "clip_ratio": 0.0, "completion_length": 9.041666984558105, "epoch": 0.6651072124756335, "grad_norm": 49.403938914448794, "kl": 5.109375, "learning_rate": 3.7397663299023507e-07, "loss": 0.5808, "reward": 0.18113365024328232, "reward_std": 0.027653340250253677, "rewards/accuracy_reward": 0.18113365024328232, "step": 853 }, { "clip_ratio": 0.0, "completion_length": 8.833333730697632, "epoch": 0.6658869395711501, "grad_norm": 115.05846461575804, "kl": 5.15625, "learning_rate": 3.728488630488823e-07, "loss": 0.1882, "reward": 0.17945753037929535, "reward_std": 0.01749674673192203, "rewards/accuracy_reward": 0.17945753037929535, "step": 854 }, { "clip_ratio": 0.0, "completion_length": 21.562500715255737, "epoch": 0.6666666666666666, "grad_norm": 19.08448230040922, "kl": 4.859375, "learning_rate": 3.7172240828634907e-07, "loss": 0.5295, "reward": 0.2150610387325287, "reward_std": 0.02549346862360835, "rewards/accuracy_reward": 0.2150610387325287, "step": 855 }, { "clip_ratio": 0.0, "completion_length": 7.9791669845581055, "epoch": 0.6674463937621833, "grad_norm": 583.5786239380601, "kl": 23.7421875, "learning_rate": 3.705972770654912e-07, "loss": 1.3665, "reward": 0.2295539528131485, "reward_std": 0.02044660970568657, "rewards/accuracy_reward": 0.2295539528131485, "step": 856 }, { "clip_ratio": 0.0, "completion_length": 8.187500476837158, "epoch": 0.6682261208576998, "grad_norm": 11.440404797312098, "kl": 3.9921875, "learning_rate": 3.694734777393387e-07, "loss": 0.2865, "reward": 0.21353499591350555, "reward_std": 0.014220003969967365, "rewards/accuracy_reward": 0.21353499591350555, "step": 857 }, { "clip_ratio": 0.0, "completion_length": 7.4166669845581055, "epoch": 0.6690058479532164, "grad_norm": 18.288574984719386, "kl": 5.3515625, "learning_rate": 3.6835101865103347e-07, "loss": 0.2077, "reward": 0.21395864337682724, "reward_std": 0.029082434251904488, "rewards/accuracy_reward": 0.21395864337682724, "step": 858 }, { "clip_ratio": 0.0, "completion_length": 8.833333492279053, "epoch": 0.6697855750487329, "grad_norm": 99.72355524750172, "kl": 7.65625, "learning_rate": 3.6722990813376744e-07, "loss": 0.7471, "reward": 0.19210276752710342, "reward_std": 0.030884217470884323, "rewards/accuracy_reward": 0.19210276752710342, "step": 859 }, { "clip_ratio": 0.0, "completion_length": 10.1875, "epoch": 0.6705653021442495, "grad_norm": 62.567748839286416, "kl": 6.34375, "learning_rate": 3.661101545107207e-07, "loss": 0.6136, "reward": 0.1789841502904892, "reward_std": 0.044487953186035156, "rewards/accuracy_reward": 0.1789841502904892, "step": 860 }, { "clip_ratio": 0.0, "completion_length": 7.8541669845581055, "epoch": 0.6713450292397661, "grad_norm": 9.421069945210936, "kl": 3.1484375, "learning_rate": 3.649917660949995e-07, "loss": 0.1673, "reward": 0.2123817279934883, "reward_std": 0.012781540397554636, "rewards/accuracy_reward": 0.2123817279934883, "step": 861 }, { "clip_ratio": 0.0, "completion_length": 7.104166746139526, "epoch": 0.6721247563352827, "grad_norm": 11.37393268822973, "kl": 3.3671875, "learning_rate": 3.638747511895748e-07, "loss": 0.1343, "reward": 0.20081845670938492, "reward_std": 0.020817924290895462, "rewards/accuracy_reward": 0.20081845670938492, "step": 862 }, { "clip_ratio": 0.0, "completion_length": 8.229166984558105, "epoch": 0.6729044834307992, "grad_norm": 19.10937238360906, "kl": 4.328125, "learning_rate": 3.627591180872209e-07, "loss": 0.2971, "reward": 0.19378720968961716, "reward_std": 0.033046112395823, "rewards/accuracy_reward": 0.19378720968961716, "step": 863 }, { "clip_ratio": 0.0, "completion_length": 8.166666984558105, "epoch": 0.6736842105263158, "grad_norm": 16.243544201159025, "kl": 5.0, "learning_rate": 3.616448750704529e-07, "loss": 0.294, "reward": 0.2008969932794571, "reward_std": 0.025891023222357035, "rewards/accuracy_reward": 0.2008969932794571, "step": 864 }, { "clip_ratio": 0.0, "completion_length": 8.375000476837158, "epoch": 0.6744639376218323, "grad_norm": 36.36679969189664, "kl": 6.515625, "learning_rate": 3.605320304114665e-07, "loss": 0.2803, "reward": 0.18615100532770157, "reward_std": 0.026490109972655773, "rewards/accuracy_reward": 0.18615100532770157, "step": 865 }, { "clip_ratio": 0.0, "completion_length": 7.5, "epoch": 0.675243664717349, "grad_norm": 10.953750395305315, "kl": 4.046875, "learning_rate": 3.594205923720754e-07, "loss": 0.2638, "reward": 0.2095734253525734, "reward_std": 0.026855789124965668, "rewards/accuracy_reward": 0.2095734253525734, "step": 866 }, { "clip_ratio": 0.0, "completion_length": 7.75, "epoch": 0.6760233918128655, "grad_norm": 19.11690069447073, "kl": 5.9921875, "learning_rate": 3.583105692036508e-07, "loss": 0.2916, "reward": 0.17860960215330124, "reward_std": 0.014079881832003593, "rewards/accuracy_reward": 0.17860960215330124, "step": 867 }, { "clip_ratio": 0.0, "completion_length": 9.416666984558105, "epoch": 0.6768031189083821, "grad_norm": 26.351598894194492, "kl": 5.15625, "learning_rate": 3.572019691470596e-07, "loss": 0.3035, "reward": 0.18929192423820496, "reward_std": 0.01562455779640004, "rewards/accuracy_reward": 0.18929192423820496, "step": 868 }, { "clip_ratio": 0.0, "completion_length": 7.708333730697632, "epoch": 0.6775828460038986, "grad_norm": 24.05928915180133, "kl": 5.03125, "learning_rate": 3.5609480043260355e-07, "loss": 0.3947, "reward": 0.20867124199867249, "reward_std": 0.02751572383567691, "rewards/accuracy_reward": 0.20867124199867249, "step": 869 }, { "clip_ratio": 0.0, "completion_length": 9.520833492279053, "epoch": 0.6783625730994152, "grad_norm": 10.54519917794754, "kl": 4.078125, "learning_rate": 3.5498907127995804e-07, "loss": 0.2366, "reward": 0.1898021697998047, "reward_std": 0.014521957375109196, "rewards/accuracy_reward": 0.1898021697998047, "step": 870 }, { "clip_ratio": 0.0, "completion_length": 7.3541669845581055, "epoch": 0.6791423001949318, "grad_norm": 9.150597797246903, "kl": 3.984375, "learning_rate": 3.538847898981112e-07, "loss": 0.193, "reward": 0.2298201099038124, "reward_std": 0.009535149089060724, "rewards/accuracy_reward": 0.2298201099038124, "step": 871 }, { "clip_ratio": 0.0, "completion_length": 8.187500238418579, "epoch": 0.6799220272904484, "grad_norm": 11.265811806168355, "kl": 4.4921875, "learning_rate": 3.527819644853026e-07, "loss": 0.3085, "reward": 0.18132660537958145, "reward_std": 0.013307837769389153, "rewards/accuracy_reward": 0.18132660537958145, "step": 872 }, { "clip_ratio": 0.0, "completion_length": 8.270833730697632, "epoch": 0.6807017543859649, "grad_norm": 27.280552585994847, "kl": 6.296875, "learning_rate": 3.5168060322896296e-07, "loss": 0.4025, "reward": 0.1876101717352867, "reward_std": 0.030521446838974953, "rewards/accuracy_reward": 0.1876101717352867, "step": 873 }, { "clip_ratio": 0.0, "completion_length": 7.375000238418579, "epoch": 0.6814814814814815, "grad_norm": 29.09441002295356, "kl": 5.125, "learning_rate": 3.5058071430565264e-07, "loss": 0.309, "reward": 0.18311680853366852, "reward_std": 0.01297963922843337, "rewards/accuracy_reward": 0.18311680853366852, "step": 874 }, { "clip_ratio": 0.0, "completion_length": 8.479166984558105, "epoch": 0.682261208576998, "grad_norm": 17.95798638973203, "kl": 4.703125, "learning_rate": 3.4948230588100154e-07, "loss": 0.4875, "reward": 0.2000204399228096, "reward_std": 0.03611239977180958, "rewards/accuracy_reward": 0.2000204399228096, "step": 875 }, { "clip_ratio": 0.0, "completion_length": 8.062500476837158, "epoch": 0.6830409356725147, "grad_norm": 16.598550671346135, "kl": 4.96875, "learning_rate": 3.4838538610964805e-07, "loss": 0.241, "reward": 0.18047089129686356, "reward_std": 0.03364178724586964, "rewards/accuracy_reward": 0.18047089129686356, "step": 876 }, { "clip_ratio": 0.0, "completion_length": 8.083333730697632, "epoch": 0.6838206627680312, "grad_norm": 10.380660860394476, "kl": 2.2578125, "learning_rate": 3.4728996313517937e-07, "loss": 0.169, "reward": 0.211860790848732, "reward_std": 0.0022142704110592604, "rewards/accuracy_reward": 0.211860790848732, "step": 877 }, { "clip_ratio": 0.0, "completion_length": 10.354166984558105, "epoch": 0.6846003898635478, "grad_norm": 14.396135490520734, "kl": 3.9296875, "learning_rate": 3.4619604509006974e-07, "loss": 0.2923, "reward": 0.19804134964942932, "reward_std": 0.023806564509868622, "rewards/accuracy_reward": 0.19804134964942932, "step": 878 }, { "clip_ratio": 0.0, "completion_length": 11.479166984558105, "epoch": 0.6853801169590643, "grad_norm": 12.399009445589902, "kl": 4.0859375, "learning_rate": 3.451036400956208e-07, "loss": 0.2746, "reward": 0.18712309002876282, "reward_std": 0.02333628199994564, "rewards/accuracy_reward": 0.18712309002876282, "step": 879 }, { "clip_ratio": 0.0, "completion_length": 9.729166984558105, "epoch": 0.6861598440545809, "grad_norm": 11.78205400828776, "kl": 3.6015625, "learning_rate": 3.4401275626190144e-07, "loss": 0.148, "reward": 0.17492123693227768, "reward_std": 0.014532366767525673, "rewards/accuracy_reward": 0.17492123693227768, "step": 880 }, { "clip_ratio": 0.0, "completion_length": 8.145833730697632, "epoch": 0.6869395711500975, "grad_norm": 338.0151734976019, "kl": 24.6875, "learning_rate": 3.429234016876874e-07, "loss": 0.6819, "reward": 0.21351857483386993, "reward_std": 0.03548992797732353, "rewards/accuracy_reward": 0.21351857483386993, "step": 881 }, { "clip_ratio": 0.0, "completion_length": 8.812500476837158, "epoch": 0.6877192982456141, "grad_norm": 151.99267479623956, "kl": 10.328125, "learning_rate": 3.4183558446040096e-07, "loss": 0.8245, "reward": 0.1790117770433426, "reward_std": 0.027704279869794846, "rewards/accuracy_reward": 0.1790117770433426, "step": 882 }, { "clip_ratio": 0.0, "completion_length": 8.166666984558105, "epoch": 0.6884990253411306, "grad_norm": 13.12288886853419, "kl": 3.046875, "learning_rate": 3.40749312656051e-07, "loss": 0.2597, "reward": 0.1871459111571312, "reward_std": 0.020442471373826265, "rewards/accuracy_reward": 0.1871459111571312, "step": 883 }, { "clip_ratio": 0.0, "completion_length": 8.708333969116211, "epoch": 0.6892787524366472, "grad_norm": 125.94566298406964, "kl": 8.29296875, "learning_rate": 3.396645943391734e-07, "loss": 0.5188, "reward": 0.1927696168422699, "reward_std": 0.010672137141227722, "rewards/accuracy_reward": 0.1927696168422699, "step": 884 }, { "clip_ratio": 0.0, "completion_length": 7.270833492279053, "epoch": 0.6900584795321637, "grad_norm": 18.80558081305964, "kl": 4.96875, "learning_rate": 3.3858143756277057e-07, "loss": 0.3059, "reward": 0.20359624177217484, "reward_std": 0.012285875156521797, "rewards/accuracy_reward": 0.20359624177217484, "step": 885 }, { "clip_ratio": 0.0, "completion_length": 7.083333492279053, "epoch": 0.6908382066276804, "grad_norm": 9.88574551897838, "kl": 3.96875, "learning_rate": 3.374998503682522e-07, "loss": 0.2362, "reward": 0.22083333134651184, "reward_std": 0.014433756470680237, "rewards/accuracy_reward": 0.22083333134651184, "step": 886 }, { "clip_ratio": 0.0, "completion_length": 10.3125, "epoch": 0.6916179337231969, "grad_norm": 956.8158767549055, "kl": 55.71484375, "learning_rate": 3.3641984078537524e-07, "loss": 3.2511, "reward": 0.17928192764520645, "reward_std": 0.010196511633694172, "rewards/accuracy_reward": 0.17928192764520645, "step": 887 }, { "clip_ratio": 0.0, "completion_length": 8.270833492279053, "epoch": 0.6923976608187135, "grad_norm": 22.18857442314468, "kl": 4.21875, "learning_rate": 3.353414168321842e-07, "loss": 0.3368, "reward": 0.2063341811299324, "reward_std": 0.01536745554767549, "rewards/accuracy_reward": 0.2063341811299324, "step": 888 }, { "clip_ratio": 0.0, "completion_length": 7.166666746139526, "epoch": 0.69317738791423, "grad_norm": 13.289208716880553, "kl": 3.53125, "learning_rate": 3.34264586514952e-07, "loss": 0.1577, "reward": 0.21908482909202576, "reward_std": 0.001933091669343412, "rewards/accuracy_reward": 0.21908482909202576, "step": 889 }, { "clip_ratio": 0.0, "completion_length": 7.833333492279053, "epoch": 0.6939571150097466, "grad_norm": 16.414544697328814, "kl": 5.875, "learning_rate": 3.331893578281203e-07, "loss": 0.3265, "reward": 0.21650509536266327, "reward_std": 0.026580046862363815, "rewards/accuracy_reward": 0.21650509536266327, "step": 890 }, { "clip_ratio": 0.0, "completion_length": 6.875000238418579, "epoch": 0.6947368421052632, "grad_norm": 55.329987616850566, "kl": 6.3984375, "learning_rate": 3.3211573875424005e-07, "loss": 0.3431, "reward": 0.19722440838813782, "reward_std": 0.0016096474137157202, "rewards/accuracy_reward": 0.19722440838813782, "step": 891 }, { "clip_ratio": 0.0, "completion_length": 7.2291669845581055, "epoch": 0.6955165692007798, "grad_norm": 16.370349301666863, "kl": 5.140625, "learning_rate": 3.310437372639128e-07, "loss": 0.4026, "reward": 0.20340585708618164, "reward_std": 0.02194612193852663, "rewards/accuracy_reward": 0.20340585708618164, "step": 892 }, { "clip_ratio": 0.0, "completion_length": 7.041666746139526, "epoch": 0.6962962962962963, "grad_norm": 14.943007133298616, "kl": 5.1875, "learning_rate": 3.2997336131573047e-07, "loss": 0.2734, "reward": 0.20707210153341293, "reward_std": 0.016659541055560112, "rewards/accuracy_reward": 0.20707210153341293, "step": 893 }, { "clip_ratio": 0.0, "completion_length": 7.4166669845581055, "epoch": 0.6970760233918128, "grad_norm": 29.232016253147407, "kl": 5.71875, "learning_rate": 3.289046188562171e-07, "loss": 0.2327, "reward": 0.2031308338046074, "reward_std": 0.039826130494475365, "rewards/accuracy_reward": 0.2031308338046074, "step": 894 }, { "clip_ratio": 0.0, "completion_length": 7.520833492279053, "epoch": 0.6978557504873294, "grad_norm": 25.562862068261172, "kl": 6.53125, "learning_rate": 3.2783751781976986e-07, "loss": 0.3669, "reward": 0.1791870892047882, "reward_std": 0.03576790168881416, "rewards/accuracy_reward": 0.1791870892047882, "step": 895 }, { "clip_ratio": 0.0, "completion_length": 7.6875, "epoch": 0.698635477582846, "grad_norm": 15.20260004328982, "kl": 5.8125, "learning_rate": 3.2677206612859954e-07, "loss": 0.2967, "reward": 0.20109273493289948, "reward_std": 0.021647372283041477, "rewards/accuracy_reward": 0.20109273493289948, "step": 896 }, { "clip_ratio": 0.0, "completion_length": 7.375000238418579, "epoch": 0.6994152046783626, "grad_norm": 6.366388476173305, "kl": 2.8671875, "learning_rate": 3.257082716926724e-07, "loss": 0.1154, "reward": 0.20592321455478668, "reward_std": 0.0024056266993284225, "rewards/accuracy_reward": 0.20592321455478668, "step": 897 }, { "clip_ratio": 0.0, "completion_length": 6.895833492279053, "epoch": 0.7001949317738791, "grad_norm": 9.449159323940716, "kl": 3.9296875, "learning_rate": 3.2464614240965104e-07, "loss": 0.197, "reward": 0.22831101715564728, "reward_std": 0.0009021099540404975, "rewards/accuracy_reward": 0.22831101715564728, "step": 898 }, { "clip_ratio": 0.0, "completion_length": 8.395833492279053, "epoch": 0.7009746588693957, "grad_norm": 814.5576645812628, "kl": 27.8125, "learning_rate": 3.235856861648358e-07, "loss": 1.819, "reward": 0.19169730693101883, "reward_std": 0.04360729083418846, "rewards/accuracy_reward": 0.19169730693101883, "step": 899 }, { "clip_ratio": 0.0, "completion_length": 7.750000476837158, "epoch": 0.7017543859649122, "grad_norm": 13.513138561797795, "kl": 4.1796875, "learning_rate": 3.225269108311066e-07, "loss": 0.3896, "reward": 0.2200138419866562, "reward_std": 0.01727256504818797, "rewards/accuracy_reward": 0.2200138419866562, "step": 900 }, { "clip_ratio": 0.0, "completion_length": 7.937500238418579, "epoch": 0.7025341130604289, "grad_norm": 15.221752544440637, "kl": 6.09375, "learning_rate": 3.214698242688638e-07, "loss": 0.443, "reward": 0.21599837392568588, "reward_std": 0.03508717566728592, "rewards/accuracy_reward": 0.21599837392568588, "step": 901 }, { "clip_ratio": 0.0, "completion_length": 8.604166984558105, "epoch": 0.7033138401559454, "grad_norm": 24.263134555368826, "kl": 9.59375, "learning_rate": 3.204144343259705e-07, "loss": 0.5248, "reward": 0.19001584500074387, "reward_std": 0.051292162388563156, "rewards/accuracy_reward": 0.19001584500074387, "step": 902 }, { "clip_ratio": 0.0, "completion_length": 8.416666984558105, "epoch": 0.704093567251462, "grad_norm": 7.008272689538141, "kl": 4.2109375, "learning_rate": 3.193607488376938e-07, "loss": 0.1621, "reward": 0.2144649550318718, "reward_std": 0.01635934691876173, "rewards/accuracy_reward": 0.2144649550318718, "step": 903 }, { "clip_ratio": 0.0, "completion_length": 6.812500238418579, "epoch": 0.7048732943469785, "grad_norm": 13.788912809415534, "kl": 7.625, "learning_rate": 3.1830877562664715e-07, "loss": 0.3442, "reward": 0.2130710855126381, "reward_std": 0.030132445972412825, "rewards/accuracy_reward": 0.2130710855126381, "step": 904 }, { "clip_ratio": 0.0, "completion_length": 9.166666984558105, "epoch": 0.7056530214424951, "grad_norm": 27.580211413108927, "kl": 4.1640625, "learning_rate": 3.172585225027318e-07, "loss": 0.3393, "reward": 0.1855800747871399, "reward_std": 0.0070287775015458465, "rewards/accuracy_reward": 0.1855800747871399, "step": 905 }, { "clip_ratio": 0.0, "completion_length": 7.7916669845581055, "epoch": 0.7064327485380117, "grad_norm": 107.79419946530524, "kl": 15.25, "learning_rate": 3.162099972630787e-07, "loss": 0.7049, "reward": 0.17611265927553177, "reward_std": 0.03691624291241169, "rewards/accuracy_reward": 0.17611265927553177, "step": 906 }, { "clip_ratio": 0.0, "completion_length": 7.604166746139526, "epoch": 0.7072124756335283, "grad_norm": 13.963377831608838, "kl": 4.34375, "learning_rate": 3.1516320769199157e-07, "loss": 0.254, "reward": 0.20359988510608673, "reward_std": 0.016609287995379418, "rewards/accuracy_reward": 0.20359988510608673, "step": 907 }, { "clip_ratio": 0.0, "completion_length": 7.958333492279053, "epoch": 0.7079922027290448, "grad_norm": 17.817515687699842, "kl": 5.46875, "learning_rate": 3.1411816156088786e-07, "loss": 0.3754, "reward": 0.22689075767993927, "reward_std": 0.02558915875852108, "rewards/accuracy_reward": 0.22689075767993927, "step": 908 }, { "clip_ratio": 0.0, "completion_length": 7.645833730697632, "epoch": 0.7087719298245614, "grad_norm": 10.694574590280249, "kl": 3.484375, "learning_rate": 3.13074866628242e-07, "loss": 0.1257, "reward": 0.2113136574625969, "reward_std": 0.012055137194693089, "rewards/accuracy_reward": 0.2113136574625969, "step": 909 }, { "clip_ratio": 0.0, "completion_length": 6.854166746139526, "epoch": 0.7095516569200779, "grad_norm": 11.213379393985633, "kl": 5.203125, "learning_rate": 3.1203333063952706e-07, "loss": 0.1745, "reward": 0.22336310148239136, "reward_std": 0.018042195588350296, "rewards/accuracy_reward": 0.22336310148239136, "step": 910 }, { "clip_ratio": 0.0, "completion_length": 6.770833492279053, "epoch": 0.7103313840155946, "grad_norm": 6.2183027038896315, "kl": 4.484375, "learning_rate": 3.109935613271579e-07, "loss": 0.1723, "reward": 0.2154718115925789, "reward_std": 0.006640276871621609, "rewards/accuracy_reward": 0.2154718115925789, "step": 911 }, { "clip_ratio": 0.0, "completion_length": 8.187500238418579, "epoch": 0.7111111111111111, "grad_norm": 17.318399561017525, "kl": 4.8515625, "learning_rate": 3.0995556641043353e-07, "loss": 0.2518, "reward": 0.19732142984867096, "reward_std": 0.021650634706020355, "rewards/accuracy_reward": 0.19732142984867096, "step": 912 }, { "clip_ratio": 0.0, "completion_length": 8.562500476837158, "epoch": 0.7118908382066277, "grad_norm": 38.39954013464861, "kl": 8.15625, "learning_rate": 3.0891935359547954e-07, "loss": 0.3732, "reward": 0.18085789680480957, "reward_std": 0.024646949023008347, "rewards/accuracy_reward": 0.18085789680480957, "step": 913 }, { "clip_ratio": 0.0, "completion_length": 7.6666669845581055, "epoch": 0.7126705653021442, "grad_norm": 20.332917727034783, "kl": 4.6484375, "learning_rate": 3.078849305751913e-07, "loss": 0.2587, "reward": 0.2028120830655098, "reward_std": 0.005516134202480316, "rewards/accuracy_reward": 0.2028120830655098, "step": 914 }, { "clip_ratio": 0.0, "completion_length": 6.6041669845581055, "epoch": 0.7134502923976608, "grad_norm": 2.159210519753522, "kl": 3.6328125, "learning_rate": 3.0685230502917644e-07, "loss": 0.1445, "reward": 0.25, "reward_std": 0.0, "rewards/accuracy_reward": 0.25, "step": 915 }, { "clip_ratio": 0.0, "completion_length": 7.5416669845581055, "epoch": 0.7142300194931774, "grad_norm": 15.310858422287343, "kl": 4.0, "learning_rate": 3.0582148462369826e-07, "loss": 0.2431, "reward": 0.2052827551960945, "reward_std": 0.006443641148507595, "rewards/accuracy_reward": 0.2052827551960945, "step": 916 }, { "clip_ratio": 0.0, "completion_length": 8.041666746139526, "epoch": 0.715009746588694, "grad_norm": 20.393041611437226, "kl": 5.9375, "learning_rate": 3.047924770116186e-07, "loss": 0.3657, "reward": 0.18276400119066238, "reward_std": 0.023592438083142042, "rewards/accuracy_reward": 0.18276400119066238, "step": 917 }, { "clip_ratio": 0.0, "completion_length": 7.3125, "epoch": 0.7157894736842105, "grad_norm": 1496.820887608113, "kl": 36.15625, "learning_rate": 3.037652898323408e-07, "loss": 3.3766, "reward": 0.21443071961402893, "reward_std": 0.04821654036641121, "rewards/accuracy_reward": 0.21443071961402893, "step": 918 }, { "clip_ratio": 0.0, "completion_length": 6.937500238418579, "epoch": 0.7165692007797271, "grad_norm": 15.71162748438164, "kl": 4.984375, "learning_rate": 3.027399307117535e-07, "loss": 0.3712, "reward": 0.19430500268936157, "reward_std": 0.02692793821915984, "rewards/accuracy_reward": 0.19430500268936157, "step": 919 }, { "clip_ratio": 0.0, "completion_length": 8.250000238418579, "epoch": 0.7173489278752436, "grad_norm": 37.41427053580005, "kl": 7.28125, "learning_rate": 3.017164072621735e-07, "loss": 0.6852, "reward": 0.20742052048444748, "reward_std": 0.03735269792377949, "rewards/accuracy_reward": 0.20742052048444748, "step": 920 }, { "clip_ratio": 0.0, "completion_length": 9.270833492279053, "epoch": 0.7181286549707603, "grad_norm": 22.46056149415884, "kl": 6.1484375, "learning_rate": 3.006947270822898e-07, "loss": 0.4526, "reward": 0.1538393869996071, "reward_std": 0.039797255769371986, "rewards/accuracy_reward": 0.1538393869996071, "step": 921 }, { "clip_ratio": 0.0, "completion_length": 7.2291669845581055, "epoch": 0.7189083820662768, "grad_norm": 35.627087156072264, "kl": 9.3125, "learning_rate": 2.996748977571067e-07, "loss": 0.3437, "reward": 0.2000187262892723, "reward_std": 0.05268122907727957, "rewards/accuracy_reward": 0.2000187262892723, "step": 922 }, { "clip_ratio": 0.0, "completion_length": 6.7291669845581055, "epoch": 0.7196881091617934, "grad_norm": 20.407968793066914, "kl": 5.921875, "learning_rate": 2.986569268578875e-07, "loss": 0.2892, "reward": 0.20979664474725723, "reward_std": 0.030791686847805977, "rewards/accuracy_reward": 0.20979664474725723, "step": 923 }, { "clip_ratio": 0.0, "completion_length": 14.645833730697632, "epoch": 0.7204678362573099, "grad_norm": 68.53982486823352, "kl": 8.515625, "learning_rate": 2.976408219420986e-07, "loss": 0.7908, "reward": 0.19621873646974564, "reward_std": 0.041535016149282455, "rewards/accuracy_reward": 0.19621873646974564, "step": 924 }, { "clip_ratio": 0.0, "completion_length": 10.000000476837158, "epoch": 0.7212475633528265, "grad_norm": 11.50355764746805, "kl": 2.875, "learning_rate": 2.966265905533534e-07, "loss": 0.1718, "reward": 0.18029513955116272, "reward_std": 0.0033077369444072247, "rewards/accuracy_reward": 0.18029513955116272, "step": 925 }, { "clip_ratio": 0.0, "completion_length": 7.270833492279053, "epoch": 0.7220272904483431, "grad_norm": 7.8761482950894015, "kl": 3.6953125, "learning_rate": 2.9561424022135577e-07, "loss": 0.1141, "reward": 0.19398538768291473, "reward_std": 0.016037508845329285, "rewards/accuracy_reward": 0.19398538768291473, "step": 926 }, { "clip_ratio": 0.0, "completion_length": 10.145833969116211, "epoch": 0.7228070175438597, "grad_norm": 16.11975226620772, "kl": 3.3828125, "learning_rate": 2.946037784618449e-07, "loss": 0.2142, "reward": 0.1789299175143242, "reward_std": 0.0071819748263806105, "rewards/accuracy_reward": 0.1789299175143242, "step": 927 }, { "clip_ratio": 0.0, "completion_length": 8.250000476837158, "epoch": 0.7235867446393762, "grad_norm": 741.3386308381091, "kl": 21.2265625, "learning_rate": 2.93595212776539e-07, "loss": 0.8527, "reward": 0.20706408470869064, "reward_std": 0.008988404646515846, "rewards/accuracy_reward": 0.20706408470869064, "step": 928 }, { "clip_ratio": 0.0, "completion_length": 7.312500238418579, "epoch": 0.7243664717348928, "grad_norm": 10.027931688039331, "kl": 3.9609375, "learning_rate": 2.9258855065307954e-07, "loss": 0.1904, "reward": 0.19589121639728546, "reward_std": 0.012564446311444044, "rewards/accuracy_reward": 0.19589121639728546, "step": 929 }, { "clip_ratio": 0.0, "completion_length": 7.770833492279053, "epoch": 0.7251461988304093, "grad_norm": 31.635156071602413, "kl": 7.125, "learning_rate": 2.9158379956497615e-07, "loss": 0.4183, "reward": 0.17945359647274017, "reward_std": 0.04685807507485151, "rewards/accuracy_reward": 0.17945359647274017, "step": 930 }, { "clip_ratio": 0.0, "completion_length": 8.583333730697632, "epoch": 0.725925925925926, "grad_norm": 17.605759226467647, "kl": 5.859375, "learning_rate": 2.9058096697155056e-07, "loss": 0.4898, "reward": 0.16475240886211395, "reward_std": 0.04607238993048668, "rewards/accuracy_reward": 0.16475240886211395, "step": 931 }, { "clip_ratio": 0.0, "completion_length": 10.083333492279053, "epoch": 0.7267056530214425, "grad_norm": 6.81837736998544, "kl": 2.58984375, "learning_rate": 2.8958006031788187e-07, "loss": 0.1037, "reward": 0.19114582240581512, "reward_std": 0.006082022562623024, "rewards/accuracy_reward": 0.19114582240581512, "step": 932 }, { "clip_ratio": 0.0, "completion_length": 8.0, "epoch": 0.7274853801169591, "grad_norm": 9.799916814961083, "kl": 3.6953125, "learning_rate": 2.885810870347503e-07, "loss": 0.1867, "reward": 0.21904762089252472, "reward_std": 0.0013901765923947096, "rewards/accuracy_reward": 0.21904762089252472, "step": 933 }, { "clip_ratio": 0.0, "completion_length": 7.500000238418579, "epoch": 0.7282651072124756, "grad_norm": 15.322958836916309, "kl": 5.3203125, "learning_rate": 2.875840545385834e-07, "loss": 0.2221, "reward": 0.20704365521669388, "reward_std": 0.01928596943616867, "rewards/accuracy_reward": 0.20704365521669388, "step": 934 }, { "clip_ratio": 0.0, "completion_length": 7.770833492279053, "epoch": 0.7290448343079922, "grad_norm": 62.89779119693976, "kl": 9.28125, "learning_rate": 2.865889702313995e-07, "loss": 0.5436, "reward": 0.2226661890745163, "reward_std": 0.026209156028926373, "rewards/accuracy_reward": 0.2226661890745163, "step": 935 }, { "clip_ratio": 0.0, "completion_length": 8.3125, "epoch": 0.7298245614035088, "grad_norm": 45.966182833692066, "kl": 7.5546875, "learning_rate": 2.855958415007541e-07, "loss": 0.3375, "reward": 0.20999066531658173, "reward_std": 0.03812823537737131, "rewards/accuracy_reward": 0.20999066531658173, "step": 936 }, { "clip_ratio": 0.0, "completion_length": 8.520833730697632, "epoch": 0.7306042884990254, "grad_norm": 294.342479205245, "kl": 10.828125, "learning_rate": 2.8460467571968394e-07, "loss": 1.4089, "reward": 0.18870870769023895, "reward_std": 0.0448327511548996, "rewards/accuracy_reward": 0.18870870769023895, "step": 937 }, { "clip_ratio": 0.0, "completion_length": 6.895833492279053, "epoch": 0.7313840155945419, "grad_norm": 11.916320863886996, "kl": 3.7109375, "learning_rate": 2.8361548024665293e-07, "loss": 0.1631, "reward": 0.20257259905338287, "reward_std": 0.027222948148846626, "rewards/accuracy_reward": 0.20257259905338287, "step": 938 }, { "clip_ratio": 0.0, "completion_length": 7.645833492279053, "epoch": 0.7321637426900585, "grad_norm": 8.11821760487084, "kl": 3.5234375, "learning_rate": 2.8262826242549735e-07, "loss": 0.1513, "reward": 0.22462120652198792, "reward_std": 0.0013121606316417456, "rewards/accuracy_reward": 0.22462120652198792, "step": 939 }, { "clip_ratio": 0.0, "completion_length": 7.625, "epoch": 0.732943469785575, "grad_norm": 4.6369447947782305, "kl": 3.546875, "learning_rate": 2.816430295853712e-07, "loss": 0.1411, "reward": 0.20333995670080185, "reward_std": 0.007151787634938955, "rewards/accuracy_reward": 0.20333995670080185, "step": 940 }, { "clip_ratio": 0.0, "completion_length": 7.895833492279053, "epoch": 0.7337231968810917, "grad_norm": 73.24128593498158, "kl": 9.703125, "learning_rate": 2.8065978904069184e-07, "loss": 0.5183, "reward": 0.2027934491634369, "reward_std": 0.026852930895984173, "rewards/accuracy_reward": 0.2027934491634369, "step": 941 }, { "clip_ratio": 0.0, "completion_length": 7.0, "epoch": 0.7345029239766082, "grad_norm": 15.39818905969451, "kl": 4.5, "learning_rate": 2.796785480910858e-07, "loss": 0.3448, "reward": 0.22633929550647736, "reward_std": 0.007732369937002659, "rewards/accuracy_reward": 0.22633929550647736, "step": 942 }, { "clip_ratio": 0.0, "completion_length": 7.312500238418579, "epoch": 0.7352826510721248, "grad_norm": 9.638816748497959, "kl": 3.859375, "learning_rate": 2.786993140213346e-07, "loss": 0.1543, "reward": 0.22909799963235855, "reward_std": 0.01054774597287178, "rewards/accuracy_reward": 0.22909799963235855, "step": 943 }, { "clip_ratio": 0.0, "completion_length": 8.020833492279053, "epoch": 0.7360623781676413, "grad_norm": 62.37680160046077, "kl": 10.9375, "learning_rate": 2.7772209410132044e-07, "loss": 0.5944, "reward": 0.23030658811330795, "reward_std": 0.018346962984651327, "rewards/accuracy_reward": 0.23030658811330795, "step": 944 }, { "clip_ratio": 0.0, "completion_length": 8.208333730697632, "epoch": 0.7368421052631579, "grad_norm": 40.62400287735523, "kl": 8.75, "learning_rate": 2.767468955859724e-07, "loss": 0.5286, "reward": 0.19600694626569748, "reward_std": 0.050653524696826935, "rewards/accuracy_reward": 0.19600694626569748, "step": 945 }, { "clip_ratio": 0.0, "completion_length": 7.229166746139526, "epoch": 0.7376218323586745, "grad_norm": 75.10386210352726, "kl": 11.09375, "learning_rate": 2.757737257152127e-07, "loss": 0.6793, "reward": 0.21377315372228622, "reward_std": 0.038850562646985054, "rewards/accuracy_reward": 0.21377315372228622, "step": 946 }, { "clip_ratio": 0.0, "completion_length": 6.770833492279053, "epoch": 0.7384015594541911, "grad_norm": 27.891470191757854, "kl": 7.875, "learning_rate": 2.7480259171390253e-07, "loss": 0.4078, "reward": 0.23989494889974594, "reward_std": 0.018198708537966013, "rewards/accuracy_reward": 0.23989494889974594, "step": 947 }, { "clip_ratio": 0.0, "completion_length": 7.145833492279053, "epoch": 0.7391812865497076, "grad_norm": 12.28104341981631, "kl": 4.234375, "learning_rate": 2.73833500791789e-07, "loss": 0.2202, "reward": 0.20513416081666946, "reward_std": 0.00513895507901907, "rewards/accuracy_reward": 0.20513416081666946, "step": 948 }, { "clip_ratio": 0.0, "completion_length": 8.8125, "epoch": 0.7399610136452242, "grad_norm": 121.38826739010007, "kl": 9.578125, "learning_rate": 2.728664601434508e-07, "loss": 0.763, "reward": 0.19558724761009216, "reward_std": 0.027047929354012012, "rewards/accuracy_reward": 0.19558724761009216, "step": 949 }, { "clip_ratio": 0.0, "completion_length": 8.020833492279053, "epoch": 0.7407407407407407, "grad_norm": 11.223203852179523, "kl": 4.828125, "learning_rate": 2.719014769482462e-07, "loss": 0.2253, "reward": 0.20562995970249176, "reward_std": 0.03450433351099491, "rewards/accuracy_reward": 0.20562995970249176, "step": 950 }, { "clip_ratio": 0.0, "completion_length": 7.5, "epoch": 0.7415204678362574, "grad_norm": 15.959557591894722, "kl": 5.078125, "learning_rate": 2.709385583702578e-07, "loss": 0.1727, "reward": 0.19545455276966095, "reward_std": 0.015745919197797775, "rewards/accuracy_reward": 0.19545455276966095, "step": 951 }, { "clip_ratio": 0.0, "completion_length": 7.041666746139526, "epoch": 0.7423001949317739, "grad_norm": 8.840969631310863, "kl": 4.625, "learning_rate": 2.6997771155824103e-07, "loss": 0.2411, "reward": 0.2261243388056755, "reward_std": 0.014716084115207195, "rewards/accuracy_reward": 0.2261243388056755, "step": 952 }, { "clip_ratio": 0.0, "completion_length": 6.8541669845581055, "epoch": 0.7430799220272905, "grad_norm": 8.845918098964946, "kl": 3.7734375, "learning_rate": 2.690189436455702e-07, "loss": 0.2014, "reward": 0.22633928805589676, "reward_std": 0.007732370402663946, "rewards/accuracy_reward": 0.22633928805589676, "step": 953 }, { "clip_ratio": 0.0, "completion_length": 7.708333492279053, "epoch": 0.743859649122807, "grad_norm": 6.947694379421619, "kl": 2.63671875, "learning_rate": 2.680622617501858e-07, "loss": 0.1329, "reward": 0.21049681305885315, "reward_std": 0.006939306389540434, "rewards/accuracy_reward": 0.21049681305885315, "step": 954 }, { "clip_ratio": 0.0, "completion_length": 7.041666746139526, "epoch": 0.7446393762183235, "grad_norm": 7.436401175020747, "kl": 3.1796875, "learning_rate": 2.671076729745417e-07, "loss": 0.136, "reward": 0.21041666716337204, "reward_std": 0.0010309829376637936, "rewards/accuracy_reward": 0.21041666716337204, "step": 955 }, { "clip_ratio": 0.0, "completion_length": 6.979166746139526, "epoch": 0.7454191033138402, "grad_norm": 4.864689427629056, "kl": 4.359375, "learning_rate": 2.661551844055521e-07, "loss": 0.2003, "reward": 0.20481150597333908, "reward_std": 0.00874157901853323, "rewards/accuracy_reward": 0.20481150597333908, "step": 956 }, { "clip_ratio": 0.0, "completion_length": 6.645833492279053, "epoch": 0.7461988304093568, "grad_norm": 8.507768498196715, "kl": 3.9921875, "learning_rate": 2.652048031145395e-07, "loss": 0.148, "reward": 0.22603785246610641, "reward_std": 0.013582898303866386, "rewards/accuracy_reward": 0.22603785246610641, "step": 957 }, { "clip_ratio": 0.0, "completion_length": 51.187501430511475, "epoch": 0.7469785575048733, "grad_norm": 32.27812582134814, "kl": 4.6484375, "learning_rate": 2.642565361571816e-07, "loss": 0.9699, "reward": 0.18459384143352509, "reward_std": 0.029080177657306194, "rewards/accuracy_reward": 0.18459384143352509, "step": 958 }, { "clip_ratio": 0.0, "completion_length": 7.791666746139526, "epoch": 0.7477582846003898, "grad_norm": 160.75045899960304, "kl": 10.28515625, "learning_rate": 2.633103905734593e-07, "loss": 0.611, "reward": 0.21790007501840591, "reward_std": 0.019832175225019455, "rewards/accuracy_reward": 0.21790007501840591, "step": 959 }, { "clip_ratio": 0.0, "completion_length": 8.104166984558105, "epoch": 0.7485380116959064, "grad_norm": 20.296101176051536, "kl": 4.078125, "learning_rate": 2.6236637338760435e-07, "loss": 0.419, "reward": 0.1891876682639122, "reward_std": 0.020885420497506857, "rewards/accuracy_reward": 0.1891876682639122, "step": 960 }, { "clip_ratio": 0.0, "completion_length": 7.416666746139526, "epoch": 0.749317738791423, "grad_norm": 10.551977697203167, "kl": 4.78125, "learning_rate": 2.6142449160804716e-07, "loss": 0.2533, "reward": 0.22309602051973343, "reward_std": 0.014773918315768242, "rewards/accuracy_reward": 0.22309602051973343, "step": 961 }, { "clip_ratio": 0.0, "completion_length": 6.895833492279053, "epoch": 0.7500974658869396, "grad_norm": 19.491390891705862, "kl": 6.625, "learning_rate": 2.6048475222736465e-07, "loss": 0.3948, "reward": 0.21663805097341537, "reward_std": 0.03417298384010792, "rewards/accuracy_reward": 0.21663805097341537, "step": 962 }, { "clip_ratio": 0.0, "completion_length": 6.729166746139526, "epoch": 0.7508771929824561, "grad_norm": 8.890387601574608, "kl": 3.8828125, "learning_rate": 2.5954716222222856e-07, "loss": 0.1911, "reward": 0.22276785969734192, "reward_std": 0.007732369937002659, "rewards/accuracy_reward": 0.22276785969734192, "step": 963 }, { "clip_ratio": 0.0, "completion_length": 7.750000238418579, "epoch": 0.7516569200779727, "grad_norm": 10.851551062812241, "kl": 4.46875, "learning_rate": 2.586117285533536e-07, "loss": 0.2524, "reward": 0.19119106233119965, "reward_std": 0.017348892986774445, "rewards/accuracy_reward": 0.19119106233119965, "step": 964 }, { "clip_ratio": 0.0, "completion_length": 8.520833492279053, "epoch": 0.7524366471734892, "grad_norm": 18.43504329114516, "kl": 6.328125, "learning_rate": 2.576784581654456e-07, "loss": 0.4014, "reward": 0.20369762182235718, "reward_std": 0.034923356026411057, "rewards/accuracy_reward": 0.20369762182235718, "step": 965 }, { "clip_ratio": 0.0, "completion_length": 8.0, "epoch": 0.7532163742690059, "grad_norm": 18.8201577021743, "kl": 4.6796875, "learning_rate": 2.567473579871502e-07, "loss": 0.2747, "reward": 0.19305557012557983, "reward_std": 0.0164988711476326, "rewards/accuracy_reward": 0.19305557012557983, "step": 966 }, { "clip_ratio": 0.0, "completion_length": 11.020833730697632, "epoch": 0.7539961013645224, "grad_norm": 11.201803955091007, "kl": 5.5078125, "learning_rate": 2.558184349310013e-07, "loss": 0.2001, "reward": 0.20014499127864838, "reward_std": 0.013582898303866386, "rewards/accuracy_reward": 0.20014499127864838, "step": 967 }, { "clip_ratio": 0.0, "completion_length": 9.833333969116211, "epoch": 0.754775828460039, "grad_norm": 25.199604766050193, "kl": 4.984375, "learning_rate": 2.5489169589336966e-07, "loss": 0.2482, "reward": 0.17346681654453278, "reward_std": 0.021422578021883965, "rewards/accuracy_reward": 0.17346681654453278, "step": 968 }, { "clip_ratio": 0.0, "completion_length": 7.312500238418579, "epoch": 0.7555555555555555, "grad_norm": 518.0749557950661, "kl": 29.7421875, "learning_rate": 2.5396714775441176e-07, "loss": 1.7572, "reward": 0.20140931755304337, "reward_std": 0.029014225117862225, "rewards/accuracy_reward": 0.20140931755304337, "step": 969 }, { "clip_ratio": 0.0, "completion_length": 8.500000476837158, "epoch": 0.7563352826510721, "grad_norm": 48.84057430402834, "kl": 5.4296875, "learning_rate": 2.5304479737801903e-07, "loss": 0.4127, "reward": 0.20271387696266174, "reward_std": 0.011814319994300604, "rewards/accuracy_reward": 0.20271387696266174, "step": 970 }, { "clip_ratio": 0.0, "completion_length": 7.145833492279053, "epoch": 0.7571150097465887, "grad_norm": 12.306816337922546, "kl": 3.5546875, "learning_rate": 2.521246516117662e-07, "loss": 0.174, "reward": 0.20942265540361404, "reward_std": 0.009404968470335007, "rewards/accuracy_reward": 0.20942265540361404, "step": 971 }, { "clip_ratio": 0.0, "completion_length": 9.375000476837158, "epoch": 0.7578947368421053, "grad_norm": 124.30793671989524, "kl": 14.03125, "learning_rate": 2.5120671728686103e-07, "loss": 0.5753, "reward": 0.19134657829999924, "reward_std": 0.024854368530213833, "rewards/accuracy_reward": 0.19134657829999924, "step": 972 }, { "clip_ratio": 0.0, "completion_length": 49.291666746139526, "epoch": 0.7586744639376218, "grad_norm": 24.045483087804328, "kl": 5.375, "learning_rate": 2.502910012180935e-07, "loss": 0.8198, "reward": 0.22331849485635757, "reward_std": 0.03475153027102351, "rewards/accuracy_reward": 0.22331849485635757, "step": 973 }, { "clip_ratio": 0.0, "completion_length": 8.375, "epoch": 0.7594541910331384, "grad_norm": 12.934408267280064, "kl": 5.46875, "learning_rate": 2.4937751020378506e-07, "loss": 0.3202, "reward": 0.21127034723758698, "reward_std": 0.018017299007624388, "rewards/accuracy_reward": 0.21127034723758698, "step": 974 }, { "clip_ratio": 0.0, "completion_length": 8.270833492279053, "epoch": 0.7602339181286549, "grad_norm": 17.009590019159607, "kl": 4.5546875, "learning_rate": 2.484662510257382e-07, "loss": 0.2651, "reward": 0.20635724067687988, "reward_std": 0.013679529540240765, "rewards/accuracy_reward": 0.20635724067687988, "step": 975 }, { "clip_ratio": 0.0, "completion_length": 7.6875, "epoch": 0.7610136452241716, "grad_norm": 10.429697339002072, "kl": 3.6015625, "learning_rate": 2.475572304491863e-07, "loss": 0.2304, "reward": 0.22831101715564728, "reward_std": 0.0009021099540404975, "rewards/accuracy_reward": 0.22831101715564728, "step": 976 }, { "clip_ratio": 0.0, "completion_length": 7.145833492279053, "epoch": 0.7617933723196881, "grad_norm": 11.947555394556202, "kl": 4.859375, "learning_rate": 2.4665045522274315e-07, "loss": 0.2264, "reward": 0.20633013546466827, "reward_std": 0.02137306472286582, "rewards/accuracy_reward": 0.20633013546466827, "step": 977 }, { "clip_ratio": 0.0, "completion_length": 7.3541669845581055, "epoch": 0.7625730994152047, "grad_norm": 24.42401875663099, "kl": 5.453125, "learning_rate": 2.45745932078353e-07, "loss": 0.3224, "reward": 0.22212301194667816, "reward_std": 0.01703009381890297, "rewards/accuracy_reward": 0.22212301194667816, "step": 978 }, { "clip_ratio": 0.0, "completion_length": 9.020833492279053, "epoch": 0.7633528265107212, "grad_norm": 19.745012615084153, "kl": 5.6328125, "learning_rate": 2.4484366773124024e-07, "loss": 0.5876, "reward": 0.23024173080921173, "reward_std": 0.03315778076648712, "rewards/accuracy_reward": 0.23024173080921173, "step": 979 }, { "clip_ratio": 0.0, "completion_length": 7.375, "epoch": 0.7641325536062378, "grad_norm": 700.1037250130006, "kl": 33.1875, "learning_rate": 2.4394366887986047e-07, "loss": 1.885, "reward": 0.22899536788463593, "reward_std": 0.030556529760360718, "rewards/accuracy_reward": 0.22899536788463593, "step": 980 }, { "clip_ratio": 0.0, "completion_length": 7.958333492279053, "epoch": 0.7649122807017544, "grad_norm": 12.422299352212708, "kl": 4.46875, "learning_rate": 2.4304594220584965e-07, "loss": 0.3144, "reward": 0.213882215321064, "reward_std": 0.025799294468015432, "rewards/accuracy_reward": 0.213882215321064, "step": 981 }, { "clip_ratio": 0.0, "completion_length": 6.583333492279053, "epoch": 0.765692007797271, "grad_norm": 267.8492873762983, "kl": 18.140625, "learning_rate": 2.421504943739749e-07, "loss": 0.9374, "reward": 0.22915521264076233, "reward_std": 0.02509198058396578, "rewards/accuracy_reward": 0.22915521264076233, "step": 982 }, { "clip_ratio": 0.0, "completion_length": 8.437500476837158, "epoch": 0.7664717348927875, "grad_norm": 32.19371440790225, "kl": 5.578125, "learning_rate": 2.4125733203208554e-07, "loss": 0.3481, "reward": 0.17858626693487167, "reward_std": 0.021845710929483175, "rewards/accuracy_reward": 0.17858626693487167, "step": 983 }, { "clip_ratio": 0.0, "completion_length": 9.791666984558105, "epoch": 0.7672514619883041, "grad_norm": 30.963429361409688, "kl": 6.9296875, "learning_rate": 2.40366461811063e-07, "loss": 0.4352, "reward": 0.1783769130706787, "reward_std": 0.01941643003374338, "rewards/accuracy_reward": 0.1783769130706787, "step": 984 }, { "clip_ratio": 0.0, "completion_length": 7.0, "epoch": 0.7680311890838206, "grad_norm": 9.838602313749337, "kl": 3.546875, "learning_rate": 2.394778903247719e-07, "loss": 0.194, "reward": 0.2005864903330803, "reward_std": 0.009700251743197441, "rewards/accuracy_reward": 0.2005864903330803, "step": 985 }, { "clip_ratio": 0.0, "completion_length": 7.270833492279053, "epoch": 0.7688109161793373, "grad_norm": 7.563139793936384, "kl": 3.6796875, "learning_rate": 2.3859162417001093e-07, "loss": 0.1569, "reward": 0.2123677209019661, "reward_std": 0.006838416215032339, "rewards/accuracy_reward": 0.2123677209019661, "step": 986 }, { "clip_ratio": 0.0, "completion_length": 7.0, "epoch": 0.7695906432748538, "grad_norm": 14.499140694509943, "kl": 4.625, "learning_rate": 2.3770766992646406e-07, "loss": 0.2424, "reward": 0.2105368673801422, "reward_std": 0.006800520233809948, "rewards/accuracy_reward": 0.2105368673801422, "step": 987 }, { "clip_ratio": 0.0, "completion_length": 7.645833492279053, "epoch": 0.7703703703703704, "grad_norm": 14.099907936315352, "kl": 4.2734375, "learning_rate": 2.3682603415665133e-07, "loss": 0.2245, "reward": 0.20190972834825516, "reward_std": 0.009480978362262249, "rewards/accuracy_reward": 0.20190972834825516, "step": 988 }, { "clip_ratio": 0.0, "completion_length": 7.4791669845581055, "epoch": 0.7711500974658869, "grad_norm": 18.01395645045911, "kl": 5.5, "learning_rate": 2.3594672340588042e-07, "loss": 0.3529, "reward": 0.21834935992956161, "reward_std": 0.01715551409870386, "rewards/accuracy_reward": 0.21834935992956161, "step": 989 }, { "clip_ratio": 0.0, "completion_length": 7.645833492279053, "epoch": 0.7719298245614035, "grad_norm": 15.183264664856523, "kl": 4.078125, "learning_rate": 2.3506974420219788e-07, "loss": 0.1894, "reward": 0.18635330349206924, "reward_std": 0.013662159442901611, "rewards/accuracy_reward": 0.18635330349206924, "step": 990 }, { "clip_ratio": 0.0, "completion_length": 6.708333492279053, "epoch": 0.7727095516569201, "grad_norm": 10.698107487049688, "kl": 4.203125, "learning_rate": 2.341951030563408e-07, "loss": 0.2453, "reward": 0.20675770938396454, "reward_std": 0.008013959042727947, "rewards/accuracy_reward": 0.20675770938396454, "step": 991 }, { "clip_ratio": 0.0, "completion_length": 6.708333492279053, "epoch": 0.7734892787524367, "grad_norm": 10.424882318762995, "kl": 6.5, "learning_rate": 2.3332280646168835e-07, "loss": 0.2424, "reward": 0.22024955600500107, "reward_std": 0.026445474475622177, "rewards/accuracy_reward": 0.22024955600500107, "step": 992 }, { "clip_ratio": 0.0, "completion_length": 6.791666746139526, "epoch": 0.7742690058479532, "grad_norm": 10.250433089470693, "kl": 3.28125, "learning_rate": 2.3245286089421379e-07, "loss": 0.1531, "reward": 0.2098102867603302, "reward_std": 0.0013121606316417456, "rewards/accuracy_reward": 0.2098102867603302, "step": 993 }, { "clip_ratio": 0.0, "completion_length": 7.9791669845581055, "epoch": 0.7750487329434698, "grad_norm": 94.41323390877069, "kl": 11.9140625, "learning_rate": 2.315852728124358e-07, "loss": 0.4808, "reward": 0.2062269076704979, "reward_std": 0.027422614162787795, "rewards/accuracy_reward": 0.2062269076704979, "step": 994 }, { "clip_ratio": 0.0, "completion_length": 8.187500476837158, "epoch": 0.7758284600389863, "grad_norm": 11.481468858085083, "kl": 6.09375, "learning_rate": 2.307200486573718e-07, "loss": 0.2584, "reward": 0.2171914055943489, "reward_std": 0.03088666219264269, "rewards/accuracy_reward": 0.2171914055943489, "step": 995 }, { "clip_ratio": 0.0, "completion_length": 7.270833492279053, "epoch": 0.776608187134503, "grad_norm": 8.607359144191236, "kl": 4.953125, "learning_rate": 2.2985719485248846e-07, "loss": 0.2727, "reward": 0.19091825187206268, "reward_std": 0.015199249610304832, "rewards/accuracy_reward": 0.19091825187206268, "step": 996 }, { "clip_ratio": 0.0, "completion_length": 7.583333492279053, "epoch": 0.7773879142300195, "grad_norm": 36.365549208527334, "kl": 4.9453125, "learning_rate": 2.2899671780365525e-07, "loss": 0.2074, "reward": 0.19834958016872406, "reward_std": 0.01723405532538891, "rewards/accuracy_reward": 0.19834958016872406, "step": 997 }, { "clip_ratio": 0.0, "completion_length": 7.958333492279053, "epoch": 0.7781676413255361, "grad_norm": 11.162306979137217, "kl": 5.078125, "learning_rate": 2.2813862389909647e-07, "loss": 0.3154, "reward": 0.21755341440439224, "reward_std": 0.021098894998431206, "rewards/accuracy_reward": 0.21755341440439224, "step": 998 }, { "clip_ratio": 0.0, "completion_length": 7.1041669845581055, "epoch": 0.7789473684210526, "grad_norm": 12.039742379468745, "kl": 3.265625, "learning_rate": 2.2728291950934368e-07, "loss": 0.1772, "reward": 0.2253224328160286, "reward_std": 0.011254892684519291, "rewards/accuracy_reward": 0.2253224328160286, "step": 999 }, { "clip_ratio": 0.0, "completion_length": 7.187500238418579, "epoch": 0.7797270955165692, "grad_norm": 7.4709825879130625, "kl": 5.15625, "learning_rate": 2.2642961098718883e-07, "loss": 0.3011, "reward": 0.20339345186948776, "reward_std": 0.01704784482717514, "rewards/accuracy_reward": 0.20339345186948776, "step": 1000 }, { "clip_ratio": 0.0, "completion_length": 7.125, "epoch": 0.7805068226120858, "grad_norm": 5.1863406122968065, "kl": 3.3359375, "learning_rate": 2.2557870466763653e-07, "loss": 0.1604, "reward": 0.21778274327516556, "reward_std": 0.006443641148507595, "rewards/accuracy_reward": 0.21778274327516556, "step": 1001 }, { "clip_ratio": 0.0, "completion_length": 8.375, "epoch": 0.7812865497076024, "grad_norm": 9.812251880904517, "kl": 2.73046875, "learning_rate": 2.2473020686785755e-07, "loss": 0.1048, "reward": 0.2005208432674408, "reward_std": 0.011694144457578659, "rewards/accuracy_reward": 0.2005208432674408, "step": 1002 }, { "clip_ratio": 0.0, "completion_length": 8.041666984558105, "epoch": 0.7820662768031189, "grad_norm": 70.98439771418607, "kl": 8.515625, "learning_rate": 2.2388412388714154e-07, "loss": 0.5227, "reward": 0.18989139795303345, "reward_std": 0.030605142936110497, "rewards/accuracy_reward": 0.18989139795303345, "step": 1003 }, { "clip_ratio": 0.0, "completion_length": 10.437500476837158, "epoch": 0.7828460038986355, "grad_norm": 51.15610748122753, "kl": 8.578125, "learning_rate": 2.2304046200685054e-07, "loss": 0.4686, "reward": 0.1909569576382637, "reward_std": 0.01652967883273959, "rewards/accuracy_reward": 0.1909569576382637, "step": 1004 }, { "clip_ratio": 0.0, "completion_length": 7.770833492279053, "epoch": 0.783625730994152, "grad_norm": 189.8733122812511, "kl": 9.15625, "learning_rate": 2.2219922749037206e-07, "loss": 0.5671, "reward": 0.20651844143867493, "reward_std": 0.024225687608122826, "rewards/accuracy_reward": 0.20651844143867493, "step": 1005 }, { "clip_ratio": 0.0, "completion_length": 6.8541669845581055, "epoch": 0.7844054580896687, "grad_norm": 15.698955863909807, "kl": 5.09375, "learning_rate": 2.2136042658307302e-07, "loss": 0.2326, "reward": 0.2251984179019928, "reward_std": 0.007877668365836143, "rewards/accuracy_reward": 0.2251984179019928, "step": 1006 }, { "clip_ratio": 0.0, "completion_length": 9.520833730697632, "epoch": 0.7851851851851852, "grad_norm": 8.027174839109064, "kl": 3.203125, "learning_rate": 2.2052406551225272e-07, "loss": 0.1836, "reward": 0.18282855302095413, "reward_std": 0.00655776634812355, "rewards/accuracy_reward": 0.18282855302095413, "step": 1007 }, { "clip_ratio": 0.0, "completion_length": 7.187500238418579, "epoch": 0.7859649122807018, "grad_norm": 21.33752135003128, "kl": 6.546875, "learning_rate": 2.1969015048709733e-07, "loss": 0.2838, "reward": 0.22746394574642181, "reward_std": 0.022172683267854154, "rewards/accuracy_reward": 0.22746394574642181, "step": 1008 }, { "clip_ratio": 0.0, "completion_length": 7.1041669845581055, "epoch": 0.7867446393762183, "grad_norm": 9.143039487813896, "kl": 3.703125, "learning_rate": 2.1885868769863314e-07, "loss": 0.1767, "reward": 0.2083333283662796, "reward_std": 0.014433758333325386, "rewards/accuracy_reward": 0.2083333283662796, "step": 1009 }, { "clip_ratio": 0.0, "completion_length": 8.3125, "epoch": 0.7875243664717348, "grad_norm": 13.356369077202201, "kl": 3.9140625, "learning_rate": 2.1802968331968137e-07, "loss": 0.3821, "reward": 0.20141476392745972, "reward_std": 0.016035879030823708, "rewards/accuracy_reward": 0.20141476392745972, "step": 1010 }, { "clip_ratio": 0.0, "completion_length": 7.2916669845581055, "epoch": 0.7883040935672515, "grad_norm": 506.3640246843003, "kl": 21.3125, "learning_rate": 2.1720314350481142e-07, "loss": 1.2658, "reward": 0.17922842502593994, "reward_std": 0.04507869854569435, "rewards/accuracy_reward": 0.17922842502593994, "step": 1011 }, { "clip_ratio": 0.0, "completion_length": 7.6041669845581055, "epoch": 0.789083820662768, "grad_norm": 12.47273968731663, "kl": 4.4921875, "learning_rate": 2.163790743902959e-07, "loss": 0.3566, "reward": 0.23570772260427475, "reward_std": 0.006208638660609722, "rewards/accuracy_reward": 0.23570772260427475, "step": 1012 }, { "clip_ratio": 0.0, "completion_length": 6.958333492279053, "epoch": 0.7898635477582846, "grad_norm": 203.22345676605886, "kl": 10.546875, "learning_rate": 2.1555748209406476e-07, "loss": 0.7871, "reward": 0.20383402705192566, "reward_std": 0.016734477481804788, "rewards/accuracy_reward": 0.20383402705192566, "step": 1013 }, { "clip_ratio": 0.0, "completion_length": 7.937500476837158, "epoch": 0.7906432748538011, "grad_norm": 20.280019297378633, "kl": 6.421875, "learning_rate": 2.1473837271565986e-07, "loss": 0.3951, "reward": 0.21068129688501358, "reward_std": 0.018166671507060528, "rewards/accuracy_reward": 0.21068129688501358, "step": 1014 }, { "clip_ratio": 0.0, "completion_length": 7.7916669845581055, "epoch": 0.7914230019493177, "grad_norm": 9.917389491397316, "kl": 4.4453125, "learning_rate": 2.1392175233618992e-07, "loss": 0.24, "reward": 0.1961805671453476, "reward_std": 0.007370158564299345, "rewards/accuracy_reward": 0.1961805671453476, "step": 1015 }, { "clip_ratio": 0.0, "completion_length": 9.645833730697632, "epoch": 0.7922027290448344, "grad_norm": 2348.0626510959946, "kl": 53.6328125, "learning_rate": 2.1310762701828498e-07, "loss": 5.0785, "reward": 0.1970486119389534, "reward_std": 0.03187454678118229, "rewards/accuracy_reward": 0.1970486119389534, "step": 1016 }, { "clip_ratio": 0.0, "completion_length": 8.083333730697632, "epoch": 0.7929824561403509, "grad_norm": 150.73356857932657, "kl": 9.6875, "learning_rate": 2.1229600280605187e-07, "loss": 0.8747, "reward": 0.18567539006471634, "reward_std": 0.031576406210660934, "rewards/accuracy_reward": 0.18567539006471634, "step": 1017 }, { "clip_ratio": 0.0, "completion_length": 8.6875, "epoch": 0.7937621832358674, "grad_norm": 1142.1620880944467, "kl": 41.515625, "learning_rate": 2.114868857250288e-07, "loss": 2.3744, "reward": 0.16367706656455994, "reward_std": 0.007582499412819743, "rewards/accuracy_reward": 0.16367706656455994, "step": 1018 }, { "clip_ratio": 0.0, "completion_length": 7.9791669845581055, "epoch": 0.794541910331384, "grad_norm": 9.853267263384136, "kl": 4.59375, "learning_rate": 2.106802817821413e-07, "loss": 0.1671, "reward": 0.2074694186449051, "reward_std": 0.031790743581950665, "rewards/accuracy_reward": 0.2074694186449051, "step": 1019 }, { "clip_ratio": 0.0, "completion_length": 7.187500238418579, "epoch": 0.7953216374269005, "grad_norm": 11.391390259422268, "kl": 6.015625, "learning_rate": 2.0987619696565678e-07, "loss": 0.2342, "reward": 0.2012605145573616, "reward_std": 0.020377070643007755, "rewards/accuracy_reward": 0.2012605145573616, "step": 1020 }, { "clip_ratio": 0.0, "completion_length": 7.375000238418579, "epoch": 0.7961013645224172, "grad_norm": 3.7377393142006117, "kl": 4.421875, "learning_rate": 2.0907463724514096e-07, "loss": 0.1816, "reward": 0.2247685194015503, "reward_std": 0.007151786237955093, "rewards/accuracy_reward": 0.2247685194015503, "step": 1021 }, { "clip_ratio": 0.0, "completion_length": 7.145833492279053, "epoch": 0.7968810916179337, "grad_norm": 11.297806677714458, "kl": 5.453125, "learning_rate": 2.0827560857141275e-07, "loss": 0.2572, "reward": 0.22240155190229416, "reward_std": 0.021373063325881958, "rewards/accuracy_reward": 0.22240155190229416, "step": 1022 }, { "clip_ratio": 0.0, "completion_length": 8.020833730697632, "epoch": 0.7976608187134503, "grad_norm": 33.05835196916885, "kl": 6.96875, "learning_rate": 2.0747911687650077e-07, "loss": 0.2696, "reward": 0.20787452161312103, "reward_std": 0.026464111171662807, "rewards/accuracy_reward": 0.20787452161312103, "step": 1023 }, { "clip_ratio": 0.0, "completion_length": 7.1041669845581055, "epoch": 0.7984405458089668, "grad_norm": 15.751242644265972, "kl": 4.4375, "learning_rate": 2.066851680735986e-07, "loss": 0.2591, "reward": 0.21856820583343506, "reward_std": 0.015401717275381088, "rewards/accuracy_reward": 0.21856820583343506, "step": 1024 }, { "clip_ratio": 0.0, "completion_length": 8.208333492279053, "epoch": 0.7992202729044834, "grad_norm": 12.789644900557542, "kl": 3.203125, "learning_rate": 2.058937680570218e-07, "loss": 0.1328, "reward": 0.19362745434045792, "reward_std": 0.022075158543884754, "rewards/accuracy_reward": 0.19362745434045792, "step": 1025 }, { "clip_ratio": 0.0, "completion_length": 7.5416669845581055, "epoch": 0.8, "grad_norm": 16.823301316471195, "kl": 4.515625, "learning_rate": 2.0510492270216317e-07, "loss": 0.3153, "reward": 0.20570098608732224, "reward_std": 0.024778381921350956, "rewards/accuracy_reward": 0.20570098608732224, "step": 1026 }, { "clip_ratio": 0.0, "completion_length": 6.9791669845581055, "epoch": 0.8007797270955166, "grad_norm": 7.0176385628468765, "kl": 5.078125, "learning_rate": 2.0431863786544977e-07, "loss": 0.2485, "reward": 0.23282968252897263, "reward_std": 0.018245520535856485, "rewards/accuracy_reward": 0.23282968252897263, "step": 1027 }, { "clip_ratio": 0.0, "completion_length": 7.9166669845581055, "epoch": 0.8015594541910331, "grad_norm": 11.85168331368778, "kl": 4.7890625, "learning_rate": 2.035349193842991e-07, "loss": 0.2387, "reward": 0.20412935316562653, "reward_std": 0.019685628358274698, "rewards/accuracy_reward": 0.20412935316562653, "step": 1028 }, { "clip_ratio": 0.0, "completion_length": 7.854166746139526, "epoch": 0.8023391812865497, "grad_norm": 39.04736862176875, "kl": 7.5625, "learning_rate": 2.0275377307707604e-07, "loss": 0.6151, "reward": 0.20278416574001312, "reward_std": 0.02558747213333845, "rewards/accuracy_reward": 0.20278416574001312, "step": 1029 }, { "clip_ratio": 0.0, "completion_length": 8.958333492279053, "epoch": 0.8031189083820662, "grad_norm": 54.012928361862016, "kl": 5.921875, "learning_rate": 2.0197520474304957e-07, "loss": 0.6474, "reward": 0.21085228025913239, "reward_std": 0.022825470194220543, "rewards/accuracy_reward": 0.21085228025913239, "step": 1030 }, { "clip_ratio": 0.0, "completion_length": 7.6041669845581055, "epoch": 0.8038986354775829, "grad_norm": 17.831836069334148, "kl": 5.859375, "learning_rate": 2.011992201623494e-07, "loss": 0.3242, "reward": 0.17871718108654022, "reward_std": 0.025753642432391644, "rewards/accuracy_reward": 0.17871718108654022, "step": 1031 }, { "clip_ratio": 0.0, "completion_length": 7.083333492279053, "epoch": 0.8046783625730994, "grad_norm": 16.98702764958395, "kl": 4.71875, "learning_rate": 2.004258250959237e-07, "loss": 0.2511, "reward": 0.2085336670279503, "reward_std": 0.01236457098275423, "rewards/accuracy_reward": 0.2085336670279503, "step": 1032 }, { "clip_ratio": 0.0, "completion_length": 7.270833492279053, "epoch": 0.805458089668616, "grad_norm": 1.8858344069108528, "kl": 3.5546875, "learning_rate": 1.9965502528549564e-07, "loss": 0.1398, "reward": 0.22857142984867096, "reward_std": 0.0, "rewards/accuracy_reward": 0.22857142984867096, "step": 1033 }, { "clip_ratio": 0.0, "completion_length": 8.916666984558105, "epoch": 0.8062378167641325, "grad_norm": 38.35429856673092, "kl": 5.265625, "learning_rate": 1.9888682645352124e-07, "loss": 0.4413, "reward": 0.20756207406520844, "reward_std": 0.012499698903411627, "rewards/accuracy_reward": 0.20756207406520844, "step": 1034 }, { "clip_ratio": 0.0, "completion_length": 9.708333492279053, "epoch": 0.8070175438596491, "grad_norm": 14.693012741694409, "kl": 5.828125, "learning_rate": 1.9812123430314675e-07, "loss": 0.3358, "reward": 0.17690972983837128, "reward_std": 0.03676615282893181, "rewards/accuracy_reward": 0.17690972983837128, "step": 1035 }, { "clip_ratio": 0.0, "completion_length": 7.500000476837158, "epoch": 0.8077972709551657, "grad_norm": 52.126792421583936, "kl": 6.109375, "learning_rate": 1.9735825451816625e-07, "loss": 0.499, "reward": 0.2145041972398758, "reward_std": 0.005429095006547868, "rewards/accuracy_reward": 0.2145041972398758, "step": 1036 }, { "clip_ratio": 0.0, "completion_length": 7.520833730697632, "epoch": 0.8085769980506823, "grad_norm": 28.50515379450771, "kl": 7.671875, "learning_rate": 1.9659789276297933e-07, "loss": 0.3041, "reward": 0.20659500360488892, "reward_std": 0.028170164674520493, "rewards/accuracy_reward": 0.20659500360488892, "step": 1037 }, { "clip_ratio": 0.0, "completion_length": 7.958333730697632, "epoch": 0.8093567251461988, "grad_norm": 126.35070528069173, "kl": 10.21875, "learning_rate": 1.9584015468254943e-07, "loss": 0.6263, "reward": 0.2020823061466217, "reward_std": 0.01635452453047037, "rewards/accuracy_reward": 0.2020823061466217, "step": 1038 }, { "clip_ratio": 0.0, "completion_length": 7.229166746139526, "epoch": 0.8101364522417154, "grad_norm": 100.24633028112018, "kl": 8.1875, "learning_rate": 1.9508504590236152e-07, "loss": 0.454, "reward": 0.21619106084108353, "reward_std": 0.010137997101992369, "rewards/accuracy_reward": 0.21619106084108353, "step": 1039 }, { "clip_ratio": 0.0, "completion_length": 9.645833492279053, "epoch": 0.8109161793372319, "grad_norm": 1.7762657686368508, "kl": 2.6953125, "learning_rate": 1.943325720283808e-07, "loss": 0.1047, "reward": 0.18125000596046448, "reward_std": 0.0, "rewards/accuracy_reward": 0.18125000596046448, "step": 1040 }, { "clip_ratio": 0.0, "completion_length": 8.687500476837158, "epoch": 0.8116959064327486, "grad_norm": 43.31100210880956, "kl": 6.578125, "learning_rate": 1.9358273864701053e-07, "loss": 0.3818, "reward": 0.20244773477315903, "reward_std": 0.014119701460003853, "rewards/accuracy_reward": 0.20244773477315903, "step": 1041 }, { "clip_ratio": 0.0, "completion_length": 9.541666984558105, "epoch": 0.8124756335282651, "grad_norm": 12.221328146155246, "kl": 3.48046875, "learning_rate": 1.9283555132505078e-07, "loss": 0.1876, "reward": 0.19107519835233688, "reward_std": 0.009265800181310624, "rewards/accuracy_reward": 0.19107519835233688, "step": 1042 }, { "clip_ratio": 0.0, "completion_length": 8.895833730697632, "epoch": 0.8132553606237817, "grad_norm": 625.3281368873542, "kl": 21.3515625, "learning_rate": 1.9209101560965733e-07, "loss": 1.2169, "reward": 0.17777778208255768, "reward_std": 0.04005598463118076, "rewards/accuracy_reward": 0.17777778208255768, "step": 1043 }, { "clip_ratio": 0.0, "completion_length": 7.208333492279053, "epoch": 0.8140350877192982, "grad_norm": 3.7742028504680545, "kl": 4.28125, "learning_rate": 1.913491370283003e-07, "loss": 0.1812, "reward": 0.23549679666757584, "reward_std": 0.006939306389540434, "rewards/accuracy_reward": 0.23549679666757584, "step": 1044 }, { "clip_ratio": 0.0, "completion_length": 8.770833730697632, "epoch": 0.8148148148148148, "grad_norm": 34.93806399167577, "kl": 4.40625, "learning_rate": 1.9060992108872293e-07, "loss": 0.282, "reward": 0.20257117599248886, "reward_std": 0.00905408151447773, "rewards/accuracy_reward": 0.20257117599248886, "step": 1045 }, { "clip_ratio": 0.0, "completion_length": 7.3125, "epoch": 0.8155945419103314, "grad_norm": 17.054482394533192, "kl": 3.8203125, "learning_rate": 1.89873373278901e-07, "loss": 0.2557, "reward": 0.21153846383094788, "reward_std": 0.0033308672718703747, "rewards/accuracy_reward": 0.21153846383094788, "step": 1046 }, { "clip_ratio": 0.0, "completion_length": 7.937500238418579, "epoch": 0.816374269005848, "grad_norm": 25.633578744677195, "kl": 4.5078125, "learning_rate": 1.89139499067002e-07, "loss": 0.4133, "reward": 0.1970544531941414, "reward_std": 0.02082281280308962, "rewards/accuracy_reward": 0.1970544531941414, "step": 1047 }, { "clip_ratio": 0.0, "completion_length": 6.708333492279053, "epoch": 0.8171539961013645, "grad_norm": 4.896591068565919, "kl": 4.5, "learning_rate": 1.8840830390134442e-07, "loss": 0.2521, "reward": 0.20726492255926132, "reward_std": 0.010129572823643684, "rewards/accuracy_reward": 0.20726492255926132, "step": 1048 }, { "clip_ratio": 0.0, "completion_length": 7.187500238418579, "epoch": 0.8179337231968811, "grad_norm": 127.22399983848484, "kl": 10.921875, "learning_rate": 1.8767979321035742e-07, "loss": 0.6176, "reward": 0.19040203094482422, "reward_std": 0.03167390776798129, "rewards/accuracy_reward": 0.19040203094482422, "step": 1049 }, { "clip_ratio": 0.0, "completion_length": 11.916666984558105, "epoch": 0.8187134502923976, "grad_norm": 171.92806685223522, "kl": 12.9609375, "learning_rate": 1.8695397240254042e-07, "loss": 0.9516, "reward": 0.20736607909202576, "reward_std": 0.018282258417457342, "rewards/accuracy_reward": 0.20736607909202576, "step": 1050 }, { "clip_ratio": 0.0, "completion_length": 7.687500238418579, "epoch": 0.8194931773879143, "grad_norm": 45.74204857070197, "kl": 10.28125, "learning_rate": 1.8623084686642309e-07, "loss": 0.5615, "reward": 0.20191581547260284, "reward_std": 0.06121378019452095, "rewards/accuracy_reward": 0.20191581547260284, "step": 1051 }, { "clip_ratio": 0.0, "completion_length": 9.104166984558105, "epoch": 0.8202729044834308, "grad_norm": 5.828633045565711, "kl": 3.171875, "learning_rate": 1.8551042197052523e-07, "loss": 0.1471, "reward": 0.18169643729925156, "reward_std": 0.007298428099602461, "rewards/accuracy_reward": 0.18169643729925156, "step": 1052 }, { "clip_ratio": 0.0, "completion_length": 7.833333730697632, "epoch": 0.8210526315789474, "grad_norm": 17.68023406395046, "kl": 5.875, "learning_rate": 1.8479270306331694e-07, "loss": 0.2516, "reward": 0.19993094354867935, "reward_std": 0.012093579396605492, "rewards/accuracy_reward": 0.19993094354867935, "step": 1053 }, { "clip_ratio": 0.0, "completion_length": 6.979166746139526, "epoch": 0.8218323586744639, "grad_norm": 3.6126801027387647, "kl": 3.9609375, "learning_rate": 1.8407769547317915e-07, "loss": 0.2158, "reward": 0.18511030822992325, "reward_std": 0.007298428099602461, "rewards/accuracy_reward": 0.18511030822992325, "step": 1054 }, { "clip_ratio": 0.0, "completion_length": 9.958333492279053, "epoch": 0.8226120857699805, "grad_norm": 12.042133738623981, "kl": 4.0078125, "learning_rate": 1.8336540450836343e-07, "loss": 0.2628, "reward": 0.19201961904764175, "reward_std": 0.014172513037919998, "rewards/accuracy_reward": 0.19201961904764175, "step": 1055 }, { "clip_ratio": 0.0, "completion_length": 6.666666746139526, "epoch": 0.8233918128654971, "grad_norm": 16.12702371214127, "kl": 4.4921875, "learning_rate": 1.8265583545695329e-07, "loss": 0.2757, "reward": 0.2366071492433548, "reward_std": 0.015464740805327892, "rewards/accuracy_reward": 0.2366071492433548, "step": 1056 }, { "clip_ratio": 0.0, "completion_length": 8.125000476837158, "epoch": 0.8241715399610137, "grad_norm": 32.75921238737116, "kl": 6.453125, "learning_rate": 1.8194899358682457e-07, "loss": 0.4349, "reward": 0.2033722996711731, "reward_std": 0.03450218332000077, "rewards/accuracy_reward": 0.2033722996711731, "step": 1057 }, { "clip_ratio": 0.0, "completion_length": 8.500000238418579, "epoch": 0.8249512670565302, "grad_norm": 16.961396326812746, "kl": 2.515625, "learning_rate": 1.812448841456062e-07, "loss": 0.2371, "reward": 0.19489890336990356, "reward_std": 0.013980986550450325, "rewards/accuracy_reward": 0.19489890336990356, "step": 1058 }, { "clip_ratio": 0.0, "completion_length": 6.895833492279053, "epoch": 0.8257309941520468, "grad_norm": 103.72792059975536, "kl": 11.03125, "learning_rate": 1.8054351236064162e-07, "loss": 0.769, "reward": 0.21145452558994293, "reward_std": 0.030768138356506824, "rewards/accuracy_reward": 0.21145452558994293, "step": 1059 }, { "clip_ratio": 0.0, "completion_length": 7.8541669845581055, "epoch": 0.8265107212475633, "grad_norm": 10.812579986161575, "kl": 4.59375, "learning_rate": 1.7984488343894982e-07, "loss": 0.1807, "reward": 0.2256365865468979, "reward_std": 0.011488264426589012, "rewards/accuracy_reward": 0.2256365865468979, "step": 1060 }, { "clip_ratio": 0.0, "completion_length": 6.895833492279053, "epoch": 0.82729044834308, "grad_norm": 18.84286940153334, "kl": 5.15625, "learning_rate": 1.7914900256718646e-07, "loss": 0.2495, "reward": 0.20378590375185013, "reward_std": 0.009470305405557156, "rewards/accuracy_reward": 0.20378590375185013, "step": 1061 }, { "clip_ratio": 0.0, "completion_length": 8.437500476837158, "epoch": 0.8280701754385965, "grad_norm": 14.078637171258807, "kl": 4.421875, "learning_rate": 1.7845587491160554e-07, "loss": 0.1668, "reward": 0.20159144699573517, "reward_std": 0.018634121399372816, "rewards/accuracy_reward": 0.20159144699573517, "step": 1062 }, { "clip_ratio": 0.0, "completion_length": 7.750000476837158, "epoch": 0.8288499025341131, "grad_norm": 77.07146085200857, "kl": 9.71875, "learning_rate": 1.777655056180213e-07, "loss": 0.8015, "reward": 0.22262488305568695, "reward_std": 0.028617288917303085, "rewards/accuracy_reward": 0.22262488305568695, "step": 1063 }, { "clip_ratio": 0.0, "completion_length": 7.729166746139526, "epoch": 0.8296296296296296, "grad_norm": 11.256085752051334, "kl": 5.25, "learning_rate": 1.770778998117694e-07, "loss": 0.2411, "reward": 0.2345610186457634, "reward_std": 0.013148694764822721, "rewards/accuracy_reward": 0.2345610186457634, "step": 1064 }, { "clip_ratio": 0.0, "completion_length": 7.687500476837158, "epoch": 0.8304093567251462, "grad_norm": 14.300891039222625, "kl": 4.578125, "learning_rate": 1.7639306259766951e-07, "loss": 0.2684, "reward": 0.204239122569561, "reward_std": 0.016638413071632385, "rewards/accuracy_reward": 0.204239122569561, "step": 1065 }, { "clip_ratio": 0.0, "completion_length": 8.750000476837158, "epoch": 0.8311890838206628, "grad_norm": 1993.4019754339108, "kl": 119.390625, "learning_rate": 1.7571099905998709e-07, "loss": 5.1469, "reward": 0.18536876887083054, "reward_std": 0.024212586577050388, "rewards/accuracy_reward": 0.18536876887083054, "step": 1066 }, { "clip_ratio": 0.0, "completion_length": 8.458333492279053, "epoch": 0.8319688109161794, "grad_norm": 131.53961724061853, "kl": 13.90625, "learning_rate": 1.7503171426239556e-07, "loss": 0.9368, "reward": 0.19176756590604782, "reward_std": 0.046957389917224646, "rewards/accuracy_reward": 0.19176756590604782, "step": 1067 }, { "clip_ratio": 0.0, "completion_length": 7.020833492279053, "epoch": 0.8327485380116959, "grad_norm": 16.594409607851688, "kl": 4.234375, "learning_rate": 1.7435521324793886e-07, "loss": 0.2707, "reward": 0.2057005614042282, "reward_std": 0.02216450870037079, "rewards/accuracy_reward": 0.2057005614042282, "step": 1068 }, { "clip_ratio": 0.0, "completion_length": 7.270833492279053, "epoch": 0.8335282651072125, "grad_norm": 5.820446340316934, "kl": 3.0390625, "learning_rate": 1.7368150103899424e-07, "loss": 0.1297, "reward": 0.21212121844291687, "reward_std": 0.0013121606316417456, "rewards/accuracy_reward": 0.21212121844291687, "step": 1069 }, { "clip_ratio": 0.0, "completion_length": 7.1875, "epoch": 0.834307992202729, "grad_norm": 19.98879790427782, "kl": 5.1875, "learning_rate": 1.7301058263723438e-07, "loss": 0.3566, "reward": 0.21365749835968018, "reward_std": 0.008362155756913126, "rewards/accuracy_reward": 0.21365749835968018, "step": 1070 }, { "clip_ratio": 0.0, "completion_length": 7.791666746139526, "epoch": 0.8350877192982457, "grad_norm": 10.243919312850661, "kl": 4.03125, "learning_rate": 1.7234246302359088e-07, "loss": 0.2648, "reward": 0.20929622650146484, "reward_std": 0.01460318360477686, "rewards/accuracy_reward": 0.20929622650146484, "step": 1071 }, { "clip_ratio": 0.0, "completion_length": 7.4791669845581055, "epoch": 0.8358674463937622, "grad_norm": 16.12075424043002, "kl": 4.09375, "learning_rate": 1.7167714715821671e-07, "loss": 0.1907, "reward": 0.20540675520896912, "reward_std": 0.022147890646010637, "rewards/accuracy_reward": 0.20540675520896912, "step": 1072 }, { "clip_ratio": 0.0, "completion_length": 8.416666984558105, "epoch": 0.8366471734892788, "grad_norm": 12.333060693824349, "kl": 4.140625, "learning_rate": 1.7101463998044991e-07, "loss": 0.2911, "reward": 0.19160164892673492, "reward_std": 0.00890958949457854, "rewards/accuracy_reward": 0.19160164892673492, "step": 1073 }, { "clip_ratio": 0.0, "completion_length": 7.937500476837158, "epoch": 0.8374269005847953, "grad_norm": 15.598845701690792, "kl": 4.875, "learning_rate": 1.7035494640877647e-07, "loss": 0.3631, "reward": 0.20432112365961075, "reward_std": 0.009817011887207627, "rewards/accuracy_reward": 0.20432112365961075, "step": 1074 }, { "clip_ratio": 0.0, "completion_length": 8.229166984558105, "epoch": 0.8382066276803118, "grad_norm": 14.73306902829391, "kl": 3.8125, "learning_rate": 1.6969807134079434e-07, "loss": 0.1718, "reward": 0.17998628318309784, "reward_std": 0.008431690745055676, "rewards/accuracy_reward": 0.17998628318309784, "step": 1075 }, { "clip_ratio": 0.0, "completion_length": 6.75, "epoch": 0.8389863547758285, "grad_norm": 15.513617412255428, "kl": 5.84375, "learning_rate": 1.6904401965317646e-07, "loss": 0.3963, "reward": 0.21415407955646515, "reward_std": 0.01877352735027671, "rewards/accuracy_reward": 0.21415407955646515, "step": 1076 }, { "clip_ratio": 0.0, "completion_length": 7.208333492279053, "epoch": 0.839766081871345, "grad_norm": 16.032939544231144, "kl": 5.328125, "learning_rate": 1.6839279620163505e-07, "loss": 0.2211, "reward": 0.21927083283662796, "reward_std": 0.019846416427753866, "rewards/accuracy_reward": 0.21927083283662796, "step": 1077 }, { "clip_ratio": 0.0, "completion_length": 7.750000238418579, "epoch": 0.8405458089668616, "grad_norm": 8.330629812285995, "kl": 5.703125, "learning_rate": 1.6774440582088524e-07, "loss": 0.1835, "reward": 0.21867559850215912, "reward_std": 0.02545811329036951, "rewards/accuracy_reward": 0.21867559850215912, "step": 1078 }, { "clip_ratio": 0.0, "completion_length": 8.5, "epoch": 0.8413255360623781, "grad_norm": 19.131966075715294, "kl": 5.96875, "learning_rate": 1.6709885332460926e-07, "loss": 0.4623, "reward": 0.17179232835769653, "reward_std": 0.02054938394576311, "rewards/accuracy_reward": 0.17179232835769653, "step": 1079 }, { "clip_ratio": 0.0, "completion_length": 8.979166984558105, "epoch": 0.8421052631578947, "grad_norm": 10.63528188445893, "kl": 4.34375, "learning_rate": 1.6645614350542097e-07, "loss": 0.2807, "reward": 0.19352678954601288, "reward_std": 0.013145028613507748, "rewards/accuracy_reward": 0.19352678954601288, "step": 1080 }, { "clip_ratio": 0.0, "completion_length": 9.25, "epoch": 0.8428849902534113, "grad_norm": 9.784262780475066, "kl": 3.6875, "learning_rate": 1.658162811348298e-07, "loss": 0.1786, "reward": 0.18511030077934265, "reward_std": 0.026056154631078243, "rewards/accuracy_reward": 0.18511030077934265, "step": 1081 }, { "clip_ratio": 0.0, "completion_length": 8.875, "epoch": 0.8436647173489279, "grad_norm": 173.97832607036148, "kl": 16.3125, "learning_rate": 1.6517927096320582e-07, "loss": 1.0345, "reward": 0.19607388228178024, "reward_std": 0.03261665580794215, "rewards/accuracy_reward": 0.19607388228178024, "step": 1082 }, { "clip_ratio": 0.0, "completion_length": 7.437500238418579, "epoch": 0.8444444444444444, "grad_norm": 16.53077320724463, "kl": 5.2421875, "learning_rate": 1.6454511771974406e-07, "loss": 0.3544, "reward": 0.20962875336408615, "reward_std": 0.020014699548482895, "rewards/accuracy_reward": 0.20962875336408615, "step": 1083 }, { "clip_ratio": 0.0, "completion_length": 8.541666984558105, "epoch": 0.845224171539961, "grad_norm": 42.62182815112753, "kl": 7.53125, "learning_rate": 1.639138261124296e-07, "loss": 0.4622, "reward": 0.16087768226861954, "reward_std": 0.037379974499344826, "rewards/accuracy_reward": 0.16087768226861954, "step": 1084 }, { "clip_ratio": 0.0, "completion_length": 7.4791669845581055, "epoch": 0.8460038986354775, "grad_norm": 30.856084535171473, "kl": 8.53125, "learning_rate": 1.6328540082800268e-07, "loss": 0.497, "reward": 0.21097032725811005, "reward_std": 0.027320554479956627, "rewards/accuracy_reward": 0.21097032725811005, "step": 1085 }, { "clip_ratio": 0.0, "completion_length": 6.958333492279053, "epoch": 0.8467836257309942, "grad_norm": 24.395286835136066, "kl": 5.359375, "learning_rate": 1.6265984653192367e-07, "loss": 0.3167, "reward": 0.21588580310344696, "reward_std": 0.024897389113903046, "rewards/accuracy_reward": 0.21588580310344696, "step": 1086 }, { "clip_ratio": 0.0, "completion_length": 7.479166746139526, "epoch": 0.8475633528265107, "grad_norm": 8.89274010583597, "kl": 3.109375, "learning_rate": 1.6203716786833872e-07, "loss": 0.1553, "reward": 0.20885418355464935, "reward_std": 0.006443641148507595, "rewards/accuracy_reward": 0.20885418355464935, "step": 1087 }, { "clip_ratio": 0.0, "completion_length": 7.666666746139526, "epoch": 0.8483430799220273, "grad_norm": 15.980591382919561, "kl": 5.640625, "learning_rate": 1.6141736946004507e-07, "loss": 0.36, "reward": 0.20034722983837128, "reward_std": 0.01898013800382614, "rewards/accuracy_reward": 0.20034722983837128, "step": 1088 }, { "clip_ratio": 0.0, "completion_length": 7.937500238418579, "epoch": 0.8491228070175438, "grad_norm": 20.49717041289614, "kl": 7.234375, "learning_rate": 1.608004559084568e-07, "loss": 0.4776, "reward": 0.21609757840633392, "reward_std": 0.03613634407520294, "rewards/accuracy_reward": 0.21609757840633392, "step": 1089 }, { "clip_ratio": 0.0, "completion_length": 7.645833492279053, "epoch": 0.8499025341130604, "grad_norm": 11.02603716971991, "kl": 4.4140625, "learning_rate": 1.6018643179357067e-07, "loss": 0.2682, "reward": 0.21140385419130325, "reward_std": 0.016452504321932793, "rewards/accuracy_reward": 0.21140385419130325, "step": 1090 }, { "clip_ratio": 0.0, "completion_length": 6.895833492279053, "epoch": 0.850682261208577, "grad_norm": 73.09758019250357, "kl": 9.265625, "learning_rate": 1.5957530167393224e-07, "loss": 0.2924, "reward": 0.1901041865348816, "reward_std": 0.02291376329958439, "rewards/accuracy_reward": 0.1901041865348816, "step": 1091 }, { "clip_ratio": 0.0, "completion_length": 8.020833492279053, "epoch": 0.8514619883040936, "grad_norm": 23.47442812407075, "kl": 5.59375, "learning_rate": 1.5896707008660167e-07, "loss": 0.3556, "reward": 0.2113695815205574, "reward_std": 0.016287715174257755, "rewards/accuracy_reward": 0.2113695815205574, "step": 1092 }, { "clip_ratio": 0.0, "completion_length": 7.270833492279053, "epoch": 0.8522417153996101, "grad_norm": 9.563165608181944, "kl": 3.5703125, "learning_rate": 1.5836174154712056e-07, "loss": 0.1525, "reward": 0.20505953580141068, "reward_std": 0.005154912360012531, "rewards/accuracy_reward": 0.20505953580141068, "step": 1093 }, { "clip_ratio": 0.0, "completion_length": 8.229166984558105, "epoch": 0.8530214424951267, "grad_norm": 17.736260052621304, "kl": 6.015625, "learning_rate": 1.577593205494778e-07, "loss": 0.3716, "reward": 0.1960604414343834, "reward_std": 0.031415728852152824, "rewards/accuracy_reward": 0.1960604414343834, "step": 1094 }, { "clip_ratio": 0.0, "completion_length": 7.020833492279053, "epoch": 0.8538011695906432, "grad_norm": 22.876046476277946, "kl": 5.6875, "learning_rate": 1.571598115660769e-07, "loss": 0.4149, "reward": 0.20456326007843018, "reward_std": 0.021579432766884565, "rewards/accuracy_reward": 0.20456326007843018, "step": 1095 }, { "clip_ratio": 0.0, "completion_length": 7.333333492279053, "epoch": 0.8545808966861599, "grad_norm": 12.806809681129913, "kl": 4.296875, "learning_rate": 1.5656321904770214e-07, "loss": 0.2756, "reward": 0.2205662503838539, "reward_std": 0.015358997508883476, "rewards/accuracy_reward": 0.2205662503838539, "step": 1096 }, { "clip_ratio": 0.0, "completion_length": 12.895833730697632, "epoch": 0.8553606237816764, "grad_norm": 38.40337729864022, "kl": 5.65625, "learning_rate": 1.55969547423486e-07, "loss": 0.7367, "reward": 0.21845005452632904, "reward_std": 0.03194329887628555, "rewards/accuracy_reward": 0.21845005452632904, "step": 1097 }, { "clip_ratio": 0.0, "completion_length": 7.895833730697632, "epoch": 0.856140350877193, "grad_norm": 15.999375182880474, "kl": 4.515625, "learning_rate": 1.5537880110087604e-07, "loss": 0.2166, "reward": 0.2007308229804039, "reward_std": 0.012282190844416618, "rewards/accuracy_reward": 0.2007308229804039, "step": 1098 }, { "clip_ratio": 0.0, "completion_length": 8.000000238418579, "epoch": 0.8569200779727095, "grad_norm": 13.29523527642342, "kl": 3.6640625, "learning_rate": 1.547909844656022e-07, "loss": 0.3176, "reward": 0.21111111342906952, "reward_std": 0.016218727454543114, "rewards/accuracy_reward": 0.21111111342906952, "step": 1099 }, { "clip_ratio": 0.0, "completion_length": 7.8541669845581055, "epoch": 0.8576998050682261, "grad_norm": 22.085077058631693, "kl": 7.21875, "learning_rate": 1.5420610188164424e-07, "loss": 0.5826, "reward": 0.20067840069532394, "reward_std": 0.013114954112097621, "rewards/accuracy_reward": 0.20067840069532394, "step": 1100 }, { "clip_ratio": 0.0, "completion_length": 6.791666746139526, "epoch": 0.8584795321637427, "grad_norm": 422.52783751527136, "kl": 29.296875, "learning_rate": 1.5362415769119962e-07, "loss": 1.3244, "reward": 0.2271214872598648, "reward_std": 0.026086248457431793, "rewards/accuracy_reward": 0.2271214872598648, "step": 1101 }, { "clip_ratio": 0.0, "completion_length": 7.270833492279053, "epoch": 0.8592592592592593, "grad_norm": 1.7704852847080452, "kl": 3.21875, "learning_rate": 1.5304515621465076e-07, "loss": 0.1288, "reward": 0.20000001788139343, "reward_std": 0.0, "rewards/accuracy_reward": 0.20000001788139343, "step": 1102 }, { "clip_ratio": 0.0, "completion_length": 8.645833492279053, "epoch": 0.8600389863547758, "grad_norm": 25.778829730750065, "kl": 5.6015625, "learning_rate": 1.5246910175053333e-07, "loss": 0.3308, "reward": 0.20887897163629532, "reward_std": 0.019954406656324863, "rewards/accuracy_reward": 0.20887897163629532, "step": 1103 }, { "clip_ratio": 0.0, "completion_length": 8.604166746139526, "epoch": 0.8608187134502924, "grad_norm": 19.133346968288162, "kl": 5.546875, "learning_rate": 1.5189599857550417e-07, "loss": 0.2343, "reward": 0.18292327225208282, "reward_std": 0.029843223746865988, "rewards/accuracy_reward": 0.18292327225208282, "step": 1104 }, { "clip_ratio": 0.0, "completion_length": 7.333333492279053, "epoch": 0.8615984405458089, "grad_norm": 15.622047407391348, "kl": 4.1015625, "learning_rate": 1.5132585094430973e-07, "loss": 0.2418, "reward": 0.2036207765340805, "reward_std": 0.01471833884716034, "rewards/accuracy_reward": 0.2036207765340805, "step": 1105 }, { "clip_ratio": 0.0, "completion_length": 7.5416669845581055, "epoch": 0.8623781676413256, "grad_norm": 69.2852733075212, "kl": 7.796875, "learning_rate": 1.507586630897542e-07, "loss": 0.5039, "reward": 0.21492642164230347, "reward_std": 0.03321198932826519, "rewards/accuracy_reward": 0.21492642164230347, "step": 1106 }, { "clip_ratio": 0.0, "completion_length": 7.583333492279053, "epoch": 0.8631578947368421, "grad_norm": 10.048310485483272, "kl": 4.8125, "learning_rate": 1.501944392226684e-07, "loss": 0.2632, "reward": 0.20047122985124588, "reward_std": 0.019584303721785545, "rewards/accuracy_reward": 0.20047122985124588, "step": 1107 }, { "clip_ratio": 0.0, "completion_length": 7.916666746139526, "epoch": 0.8639376218323587, "grad_norm": 8.334636581804242, "kl": 4.421875, "learning_rate": 1.4963318353187814e-07, "loss": 0.1982, "reward": 0.23336397111415863, "reward_std": 0.011155277490615845, "rewards/accuracy_reward": 0.23336397111415863, "step": 1108 }, { "clip_ratio": 0.0, "completion_length": 7.020833492279053, "epoch": 0.8647173489278752, "grad_norm": 14.618675105592677, "kl": 5.40625, "learning_rate": 1.4907490018417364e-07, "loss": 0.3255, "reward": 0.20911459624767303, "reward_std": 0.024099217727780342, "rewards/accuracy_reward": 0.20911459624767303, "step": 1109 }, { "clip_ratio": 0.0, "completion_length": 9.875000238418579, "epoch": 0.8654970760233918, "grad_norm": 12.747303584440514, "kl": 4.6796875, "learning_rate": 1.4851959332427808e-07, "loss": 0.2918, "reward": 0.1488143801689148, "reward_std": 0.019283058121800423, "rewards/accuracy_reward": 0.1488143801689148, "step": 1110 }, { "clip_ratio": 0.0, "completion_length": 7.500000238418579, "epoch": 0.8662768031189084, "grad_norm": 17.54463188488618, "kl": 5.21875, "learning_rate": 1.4796726707481705e-07, "loss": 0.2586, "reward": 0.21874331682920456, "reward_std": 0.02411574963480234, "rewards/accuracy_reward": 0.21874331682920456, "step": 1111 }, { "clip_ratio": 0.0, "completion_length": 7.500000238418579, "epoch": 0.867056530214425, "grad_norm": 23.123309948564504, "kl": 7.09375, "learning_rate": 1.474179255362881e-07, "loss": 0.3821, "reward": 0.1921483427286148, "reward_std": 0.04064559563994408, "rewards/accuracy_reward": 0.1921483427286148, "step": 1112 }, { "clip_ratio": 0.0, "completion_length": 7.437500238418579, "epoch": 0.8678362573099415, "grad_norm": 12.95162905582068, "kl": 4.5625, "learning_rate": 1.4687157278702985e-07, "loss": 0.2659, "reward": 0.22105217725038528, "reward_std": 0.013675682246685028, "rewards/accuracy_reward": 0.22105217725038528, "step": 1113 }, { "clip_ratio": 0.0, "completion_length": 8.645833969116211, "epoch": 0.8686159844054581, "grad_norm": 16.9559773382592, "kl": 6.484375, "learning_rate": 1.4632821288319243e-07, "loss": 0.3578, "reward": 0.20171932876110077, "reward_std": 0.018030264880508184, "rewards/accuracy_reward": 0.20171932876110077, "step": 1114 }, { "clip_ratio": 0.0, "completion_length": 8.395833492279053, "epoch": 0.8693957115009746, "grad_norm": 24.630848662618373, "kl": 8.3125, "learning_rate": 1.4578784985870656e-07, "loss": 0.4284, "reward": 0.20220322161912918, "reward_std": 0.03783389087766409, "rewards/accuracy_reward": 0.20220322161912918, "step": 1115 }, { "clip_ratio": 0.0, "completion_length": 7.583333492279053, "epoch": 0.8701754385964913, "grad_norm": 10.76136783138401, "kl": 4.8125, "learning_rate": 1.4525048772525417e-07, "loss": 0.2265, "reward": 0.22199518978595734, "reward_std": 0.007469884469173849, "rewards/accuracy_reward": 0.22199518978595734, "step": 1116 }, { "clip_ratio": 0.0, "completion_length": 9.791666746139526, "epoch": 0.8709551656920078, "grad_norm": 462.75876021620957, "kl": 75.1875, "learning_rate": 1.447161304722383e-07, "loss": 1.5123, "reward": 0.16037777066230774, "reward_std": 0.03151809982955456, "rewards/accuracy_reward": 0.16037777066230774, "step": 1117 }, { "clip_ratio": 0.0, "completion_length": 7.541666746139526, "epoch": 0.8717348927875244, "grad_norm": 343.9876563620424, "kl": 14.2421875, "learning_rate": 1.4418478206675365e-07, "loss": 0.8938, "reward": 0.20913627743721008, "reward_std": 0.011012413073331118, "rewards/accuracy_reward": 0.20913627743721008, "step": 1118 }, { "clip_ratio": 0.0, "completion_length": 8.645833492279053, "epoch": 0.8725146198830409, "grad_norm": 1925.6196485974585, "kl": 30.875, "learning_rate": 1.4365644645355727e-07, "loss": 3.2486, "reward": 0.18649181723594666, "reward_std": 0.046888433396816254, "rewards/accuracy_reward": 0.18649181723594666, "step": 1119 }, { "clip_ratio": 0.0, "completion_length": 8.666666984558105, "epoch": 0.8732943469785575, "grad_norm": 140.65767714461947, "kl": 13.5625, "learning_rate": 1.4313112755503874e-07, "loss": 1.049, "reward": 0.2146035134792328, "reward_std": 0.02879335079342127, "rewards/accuracy_reward": 0.2146035134792328, "step": 1120 }, { "clip_ratio": 0.0, "completion_length": 8.500000476837158, "epoch": 0.8740740740740741, "grad_norm": 59.719873758277735, "kl": 8.0, "learning_rate": 1.4260882927119172e-07, "loss": 0.5343, "reward": 0.18853729963302612, "reward_std": 0.026253076270222664, "rewards/accuracy_reward": 0.18853729963302612, "step": 1121 }, { "clip_ratio": 0.0, "completion_length": 9.895833969116211, "epoch": 0.8748538011695907, "grad_norm": 8.89181466582034, "kl": 3.4140625, "learning_rate": 1.4208955547958455e-07, "loss": 0.1375, "reward": 0.19229472428560257, "reward_std": 0.005041201948188245, "rewards/accuracy_reward": 0.19229472428560257, "step": 1122 }, { "clip_ratio": 0.0, "completion_length": 8.062500476837158, "epoch": 0.8756335282651072, "grad_norm": 11.445772771655074, "kl": 3.7578125, "learning_rate": 1.4157331003533152e-07, "loss": 0.2011, "reward": 0.21140196174383163, "reward_std": 0.013236417435109615, "rewards/accuracy_reward": 0.21140196174383163, "step": 1123 }, { "clip_ratio": 0.0, "completion_length": 7.5, "epoch": 0.8764132553606238, "grad_norm": 10.780250433067875, "kl": 3.5546875, "learning_rate": 1.410600967710644e-07, "loss": 0.3108, "reward": 0.2260010838508606, "reward_std": 0.008903942070901394, "rewards/accuracy_reward": 0.2260010838508606, "step": 1124 }, { "clip_ratio": 0.0, "completion_length": 7.833333492279053, "epoch": 0.8771929824561403, "grad_norm": 97.82024019220468, "kl": 12.125, "learning_rate": 1.4054991949690399e-07, "loss": 0.6938, "reward": 0.1811150163412094, "reward_std": 0.030483581125736237, "rewards/accuracy_reward": 0.1811150163412094, "step": 1125 }, { "clip_ratio": 0.0, "completion_length": 7.7916669845581055, "epoch": 0.877972709551657, "grad_norm": 424.8255646396545, "kl": 15.640625, "learning_rate": 1.400427820004316e-07, "loss": 1.2221, "reward": 0.1847241073846817, "reward_std": 0.017680260352790356, "rewards/accuracy_reward": 0.1847241073846817, "step": 1126 }, { "clip_ratio": 0.0, "completion_length": 7.2291669845581055, "epoch": 0.8787524366471735, "grad_norm": 13.177723315748436, "kl": 3.65625, "learning_rate": 1.3953868804666098e-07, "loss": 0.1647, "reward": 0.21326472610235214, "reward_std": 0.008053896948695183, "rewards/accuracy_reward": 0.21326472610235214, "step": 1127 }, { "clip_ratio": 0.0, "completion_length": 7.312500238418579, "epoch": 0.87953216374269, "grad_norm": 15.35623347914603, "kl": 3.9921875, "learning_rate": 1.3903764137801087e-07, "loss": 0.228, "reward": 0.2282852679491043, "reward_std": 0.02157183224335313, "rewards/accuracy_reward": 0.2282852679491043, "step": 1128 }, { "clip_ratio": 0.0, "completion_length": 7.895833492279053, "epoch": 0.8803118908382066, "grad_norm": 716.5300574091425, "kl": 29.03125, "learning_rate": 1.3853964571427647e-07, "loss": 2.0881, "reward": 0.20357157289981842, "reward_std": 0.036397796124219894, "rewards/accuracy_reward": 0.20357157289981842, "step": 1129 }, { "clip_ratio": 0.0, "completion_length": 8.437500476837158, "epoch": 0.8810916179337231, "grad_norm": 12.857147794185964, "kl": 5.3671875, "learning_rate": 1.3804470475260233e-07, "loss": 0.2394, "reward": 0.20706189423799515, "reward_std": 0.04022690188139677, "rewards/accuracy_reward": 0.20706189423799515, "step": 1130 }, { "clip_ratio": 0.0, "completion_length": 7.750000238418579, "epoch": 0.8818713450292398, "grad_norm": 9.32015323076926, "kl": 5.09375, "learning_rate": 1.3755282216745453e-07, "loss": 0.2462, "reward": 0.21319444477558136, "reward_std": 0.02095174416899681, "rewards/accuracy_reward": 0.21319444477558136, "step": 1131 }, { "clip_ratio": 0.0, "completion_length": 8.270833969116211, "epoch": 0.8826510721247564, "grad_norm": 5.787760223589242, "kl": 4.1328125, "learning_rate": 1.3706400161059386e-07, "loss": 0.1816, "reward": 0.20977095514535904, "reward_std": 0.007338681723922491, "rewards/accuracy_reward": 0.20977095514535904, "step": 1132 }, { "clip_ratio": 0.0, "completion_length": 7.250000238418579, "epoch": 0.8834307992202729, "grad_norm": 5.163187287093115, "kl": 4.125, "learning_rate": 1.3657824671104836e-07, "loss": 0.2036, "reward": 0.22553323954343796, "reward_std": 0.007182053290307522, "rewards/accuracy_reward": 0.22553323954343796, "step": 1133 }, { "clip_ratio": 0.0, "completion_length": 7.979166746139526, "epoch": 0.8842105263157894, "grad_norm": 247.3670389819012, "kl": 18.0625, "learning_rate": 1.360955610750864e-07, "loss": 0.9605, "reward": 0.19943124800920486, "reward_std": 0.01915727835148573, "rewards/accuracy_reward": 0.19943124800920486, "step": 1134 }, { "clip_ratio": 0.0, "completion_length": 7.0, "epoch": 0.884990253411306, "grad_norm": 30.307461728398913, "kl": 6.265625, "learning_rate": 1.3561594828619013e-07, "loss": 0.4557, "reward": 0.2269803136587143, "reward_std": 0.01649410603567958, "rewards/accuracy_reward": 0.2269803136587143, "step": 1135 }, { "clip_ratio": 0.0, "completion_length": 9.791666746139526, "epoch": 0.8857699805068227, "grad_norm": 74.67816621501852, "kl": 11.40625, "learning_rate": 1.3513941190502867e-07, "loss": 0.9551, "reward": 0.1939118579030037, "reward_std": 0.04064749926328659, "rewards/accuracy_reward": 0.1939118579030037, "step": 1136 }, { "clip_ratio": 0.0, "completion_length": 8.354166984558105, "epoch": 0.8865497076023392, "grad_norm": 29.435556153930253, "kl": 7.5, "learning_rate": 1.346659554694317e-07, "loss": 0.4278, "reward": 0.1762477159500122, "reward_std": 0.022466310299932957, "rewards/accuracy_reward": 0.1762477159500122, "step": 1137 }, { "clip_ratio": 0.0, "completion_length": 7.8125, "epoch": 0.8873294346978557, "grad_norm": 8.002436086170635, "kl": 3.28125, "learning_rate": 1.3419558249436325e-07, "loss": 0.1409, "reward": 0.20936384797096252, "reward_std": 0.01086391182616353, "rewards/accuracy_reward": 0.20936384797096252, "step": 1138 }, { "clip_ratio": 0.0, "completion_length": 7.458333492279053, "epoch": 0.8881091617933723, "grad_norm": 10.483695744828937, "kl": 4.890625, "learning_rate": 1.3372829647189556e-07, "loss": 0.3631, "reward": 0.1949465423822403, "reward_std": 0.022146458737552166, "rewards/accuracy_reward": 0.1949465423822403, "step": 1139 }, { "clip_ratio": 0.0, "completion_length": 6.9791669845581055, "epoch": 0.8888888888888888, "grad_norm": 71.98654901799392, "kl": 8.09375, "learning_rate": 1.3326410087118333e-07, "loss": 0.5113, "reward": 0.22195132076740265, "reward_std": 0.020509536378085613, "rewards/accuracy_reward": 0.22195132076740265, "step": 1140 }, { "clip_ratio": 0.0, "completion_length": 7.270833492279053, "epoch": 0.8896686159844055, "grad_norm": 6.163650790043578, "kl": 3.8125, "learning_rate": 1.328029991384376e-07, "loss": 0.1678, "reward": 0.23054317384958267, "reward_std": 0.005541531601920724, "rewards/accuracy_reward": 0.23054317384958267, "step": 1141 }, { "clip_ratio": 0.0, "completion_length": 10.250000238418579, "epoch": 0.890448343079922, "grad_norm": 20.036961689839035, "kl": 5.78125, "learning_rate": 1.3234499469690053e-07, "loss": 0.3493, "reward": 0.18308180570602417, "reward_std": 0.026062607765197754, "rewards/accuracy_reward": 0.18308180570602417, "step": 1142 }, { "clip_ratio": 0.0, "completion_length": 7.3541669845581055, "epoch": 0.8912280701754386, "grad_norm": 972.6753125383146, "kl": 16.234375, "learning_rate": 1.3189009094681976e-07, "loss": 1.9322, "reward": 0.22068586945533752, "reward_std": 0.02204093337059021, "rewards/accuracy_reward": 0.22068586945533752, "step": 1143 }, { "clip_ratio": 0.0, "completion_length": 7.875000476837158, "epoch": 0.8920077972709551, "grad_norm": 10684.773364023755, "kl": 399.078125, "learning_rate": 1.3143829126542334e-07, "loss": 29.1504, "reward": 0.20947421342134476, "reward_std": 0.015208574943244457, "rewards/accuracy_reward": 0.20947421342134476, "step": 1144 }, { "clip_ratio": 0.0, "completion_length": 8.041666984558105, "epoch": 0.8927875243664717, "grad_norm": 26.873300554003084, "kl": 6.109375, "learning_rate": 1.309895990068944e-07, "loss": 0.364, "reward": 0.21848131716251373, "reward_std": 0.02330506592988968, "rewards/accuracy_reward": 0.21848131716251373, "step": 1145 }, { "clip_ratio": 0.0, "completion_length": 7.270833492279053, "epoch": 0.8935672514619883, "grad_norm": 7.956893906005812, "kl": 4.734375, "learning_rate": 1.3054401750234664e-07, "loss": 0.2887, "reward": 0.1941876858472824, "reward_std": 0.01563416514545679, "rewards/accuracy_reward": 0.1941876858472824, "step": 1146 }, { "clip_ratio": 0.0, "completion_length": 7.437500476837158, "epoch": 0.8943469785575049, "grad_norm": 65.8008291993919, "kl": 8.171875, "learning_rate": 1.3010155005979906e-07, "loss": 0.6336, "reward": 0.22762896120548248, "reward_std": 0.026936329435557127, "rewards/accuracy_reward": 0.22762896120548248, "step": 1147 }, { "clip_ratio": 0.0, "completion_length": 9.854166984558105, "epoch": 0.8951267056530214, "grad_norm": 112.0515083490587, "kl": 9.40625, "learning_rate": 1.29662199964152e-07, "loss": 0.4971, "reward": 0.16739416122436523, "reward_std": 0.03169836103916168, "rewards/accuracy_reward": 0.16739416122436523, "step": 1148 }, { "clip_ratio": 0.0, "completion_length": 7.7291669845581055, "epoch": 0.895906432748538, "grad_norm": 517.3697121252072, "kl": 17.640625, "learning_rate": 1.2922597047716234e-07, "loss": 1.4871, "reward": 0.19385723769664764, "reward_std": 0.015208422672003508, "rewards/accuracy_reward": 0.19385723769664764, "step": 1149 }, { "clip_ratio": 0.0, "completion_length": 7.4375, "epoch": 0.8966861598440545, "grad_norm": 11.284650736010855, "kl": 4.859375, "learning_rate": 1.2879286483741935e-07, "loss": 0.322, "reward": 0.19349761307239532, "reward_std": 0.02272272203117609, "rewards/accuracy_reward": 0.19349761307239532, "step": 1150 }, { "clip_ratio": 0.0, "completion_length": 7.625, "epoch": 0.8974658869395712, "grad_norm": 8.082345545362394, "kl": 3.7109375, "learning_rate": 1.2836288626032073e-07, "loss": 0.2926, "reward": 0.2061813324689865, "reward_std": 0.0033308672718703747, "rewards/accuracy_reward": 0.2061813324689865, "step": 1151 }, { "clip_ratio": 0.0, "completion_length": 8.041666984558105, "epoch": 0.8982456140350877, "grad_norm": 31.012880045472915, "kl": 5.625, "learning_rate": 1.2793603793804885e-07, "loss": 0.451, "reward": 0.17605771869421005, "reward_std": 0.033883729949593544, "rewards/accuracy_reward": 0.17605771869421005, "step": 1152 }, { "clip_ratio": 0.0, "completion_length": 7.645833730697632, "epoch": 0.8990253411306043, "grad_norm": 20.032442402711478, "kl": 5.375, "learning_rate": 1.275123230395468e-07, "loss": 0.2227, "reward": 0.21672210842370987, "reward_std": 0.012559764087200165, "rewards/accuracy_reward": 0.21672210842370987, "step": 1153 }, { "clip_ratio": 0.0, "completion_length": 7.583333492279053, "epoch": 0.8998050682261208, "grad_norm": 9.47799606863283, "kl": 5.546875, "learning_rate": 1.270917447104949e-07, "loss": 0.2963, "reward": 0.21750182658433914, "reward_std": 0.015390753746032715, "rewards/accuracy_reward": 0.21750182658433914, "step": 1154 }, { "clip_ratio": 0.0, "completion_length": 9.187500238418579, "epoch": 0.9005847953216374, "grad_norm": 4.135377287109526, "kl": 3.0625, "learning_rate": 1.266743060732875e-07, "loss": 0.1235, "reward": 0.19702380895614624, "reward_std": 0.0010309829376637936, "rewards/accuracy_reward": 0.19702380895614624, "step": 1155 }, { "clip_ratio": 0.0, "completion_length": 8.000000238418579, "epoch": 0.901364522417154, "grad_norm": 12.315522826394604, "kl": 5.390625, "learning_rate": 1.262600102270098e-07, "loss": 0.3911, "reward": 0.20960648357868195, "reward_std": 0.019279979169368744, "rewards/accuracy_reward": 0.20960648357868195, "step": 1156 }, { "clip_ratio": 0.0, "completion_length": 7.729166746139526, "epoch": 0.9021442495126706, "grad_norm": 75.26008963979227, "kl": 8.3046875, "learning_rate": 1.2584886024741467e-07, "loss": 0.3112, "reward": 0.21346726268529892, "reward_std": 0.006082022562623024, "rewards/accuracy_reward": 0.21346726268529892, "step": 1157 }, { "clip_ratio": 0.0, "completion_length": 9.1875, "epoch": 0.9029239766081871, "grad_norm": 13.550818569612096, "kl": 3.8125, "learning_rate": 1.2544085918689992e-07, "loss": 0.158, "reward": 0.18850447237491608, "reward_std": 0.010908886790275574, "rewards/accuracy_reward": 0.18850447237491608, "step": 1158 }, { "clip_ratio": 0.0, "completion_length": 7.250000238418579, "epoch": 0.9037037037037037, "grad_norm": 4.852081209370184, "kl": 3.59375, "learning_rate": 1.250360100744857e-07, "loss": 0.1377, "reward": 0.2282407358288765, "reward_std": 0.005406242795288563, "rewards/accuracy_reward": 0.2282407358288765, "step": 1159 }, { "clip_ratio": 0.0, "completion_length": 31.58333396911621, "epoch": 0.9044834307992202, "grad_norm": 18.573257939301595, "kl": 7.171875, "learning_rate": 1.2463431591579183e-07, "loss": 0.5888, "reward": 0.21907051652669907, "reward_std": 0.030235057114623487, "rewards/accuracy_reward": 0.21907051652669907, "step": 1160 }, { "clip_ratio": 0.0, "completion_length": 7.541666746139526, "epoch": 0.9052631578947369, "grad_norm": 33.315793730608064, "kl": 6.8125, "learning_rate": 1.2423577969301564e-07, "loss": 0.4517, "reward": 0.20457153022289276, "reward_std": 0.02641383372247219, "rewards/accuracy_reward": 0.20457153022289276, "step": 1161 }, { "clip_ratio": 0.0, "completion_length": 8.520833969116211, "epoch": 0.9060428849902534, "grad_norm": 115.1980286593509, "kl": 6.796875, "learning_rate": 1.238404043649098e-07, "loss": 0.448, "reward": 0.2078033685684204, "reward_std": 0.022235992830246687, "rewards/accuracy_reward": 0.2078033685684204, "step": 1162 }, { "clip_ratio": 0.0, "completion_length": 10.166666984558105, "epoch": 0.90682261208577, "grad_norm": 9.745138487856355, "kl": 3.4765625, "learning_rate": 1.234481928667604e-07, "loss": 0.145, "reward": 0.17665661126375198, "reward_std": 0.009434967301785946, "rewards/accuracy_reward": 0.17665661126375198, "step": 1163 }, { "clip_ratio": 0.0, "completion_length": 7.1041669845581055, "epoch": 0.9076023391812865, "grad_norm": 40.65086542764804, "kl": 5.765625, "learning_rate": 1.2305914811036495e-07, "loss": 0.2547, "reward": 0.20849844068288803, "reward_std": 0.019543007481843233, "rewards/accuracy_reward": 0.20849844068288803, "step": 1164 }, { "clip_ratio": 0.0, "completion_length": 7.895833492279053, "epoch": 0.9083820662768031, "grad_norm": 11.226648393999609, "kl": 4.25, "learning_rate": 1.226732729840111e-07, "loss": 0.2436, "reward": 0.208762988448143, "reward_std": 0.01885221805423498, "rewards/accuracy_reward": 0.208762988448143, "step": 1165 }, { "clip_ratio": 0.0, "completion_length": 7.625000238418579, "epoch": 0.9091617933723197, "grad_norm": 1.8076732133449984, "kl": 2.5625, "learning_rate": 1.222905703524548e-07, "loss": 0.0966, "reward": 0.21250000596046448, "reward_std": 0.0, "rewards/accuracy_reward": 0.21250000596046448, "step": 1166 }, { "clip_ratio": 0.0, "completion_length": 7.187500238418579, "epoch": 0.9099415204678363, "grad_norm": 27.695314065525512, "kl": 7.84375, "learning_rate": 1.2191104305689945e-07, "loss": 0.4291, "reward": 0.19272439181804657, "reward_std": 0.027663799934089184, "rewards/accuracy_reward": 0.19272439181804657, "step": 1167 }, { "clip_ratio": 0.0, "completion_length": 6.770833492279053, "epoch": 0.9107212475633528, "grad_norm": 5.968095678215035, "kl": 3.3046875, "learning_rate": 1.2153469391497442e-07, "loss": 0.1415, "reward": 0.20635723322629929, "reward_std": 0.0009021099540404975, "rewards/accuracy_reward": 0.20635723322629929, "step": 1168 }, { "clip_ratio": 0.0, "completion_length": 7.1666669845581055, "epoch": 0.9115009746588694, "grad_norm": 11.761933997975017, "kl": 4.1328125, "learning_rate": 1.2116152572071445e-07, "loss": 0.2332, "reward": 0.22108261287212372, "reward_std": 0.008909666910767555, "rewards/accuracy_reward": 0.22108261287212372, "step": 1169 }, { "clip_ratio": 0.0, "completion_length": 6.937500238418579, "epoch": 0.9122807017543859, "grad_norm": 52.160939766803395, "kl": 6.078125, "learning_rate": 1.2079154124453864e-07, "loss": 0.4229, "reward": 0.2112390249967575, "reward_std": 0.019674470648169518, "rewards/accuracy_reward": 0.2112390249967575, "step": 1170 }, { "clip_ratio": 0.0, "completion_length": 7.416666746139526, "epoch": 0.9130604288499026, "grad_norm": 16.42590522237208, "kl": 5.5625, "learning_rate": 1.2042474323323011e-07, "loss": 0.3279, "reward": 0.21971727162599564, "reward_std": 0.013451378792524338, "rewards/accuracy_reward": 0.21971727162599564, "step": 1171 }, { "clip_ratio": 0.0, "completion_length": 7.208333492279053, "epoch": 0.9138401559454191, "grad_norm": 83.65911752529338, "kl": 9.15625, "learning_rate": 1.2006113440991547e-07, "loss": 0.5471, "reward": 0.20092131197452545, "reward_std": 0.025659140199422836, "rewards/accuracy_reward": 0.20092131197452545, "step": 1172 }, { "clip_ratio": 0.0, "completion_length": 8.0625, "epoch": 0.9146198830409357, "grad_norm": 14.190439321257367, "kl": 5.59375, "learning_rate": 1.197007174740447e-07, "loss": 0.3338, "reward": 0.21384231001138687, "reward_std": 0.011795071884989738, "rewards/accuracy_reward": 0.21384231001138687, "step": 1173 }, { "clip_ratio": 0.0, "completion_length": 7.4166669845581055, "epoch": 0.9153996101364522, "grad_norm": 34609.638390187094, "kl": 1020.625, "learning_rate": 1.1934349510137093e-07, "loss": 73.4034, "reward": 0.18986543267965317, "reward_std": 0.02204020880162716, "rewards/accuracy_reward": 0.18986543267965317, "step": 1174 }, { "clip_ratio": 0.0, "completion_length": 7.916666746139526, "epoch": 0.9161793372319688, "grad_norm": 10.355660028530062, "kl": 3.0, "learning_rate": 1.1898946994393078e-07, "loss": 0.1932, "reward": 0.1994047835469246, "reward_std": 0.014433756470680237, "rewards/accuracy_reward": 0.1994047835469246, "step": 1175 }, { "clip_ratio": 0.0, "completion_length": 9.3125, "epoch": 0.9169590643274854, "grad_norm": 29.439951587205623, "kl": 6.6484375, "learning_rate": 1.1863864463002456e-07, "loss": 0.3758, "reward": 0.18477682024240494, "reward_std": 0.01697483379393816, "rewards/accuracy_reward": 0.18477682024240494, "step": 1176 }, { "clip_ratio": 0.0, "completion_length": 7.520833492279053, "epoch": 0.917738791423002, "grad_norm": 18.86348570769248, "kl": 6.609375, "learning_rate": 1.1829102176419677e-07, "loss": 0.3719, "reward": 0.21543899178504944, "reward_std": 0.030653451569378376, "rewards/accuracy_reward": 0.21543899178504944, "step": 1177 }, { "clip_ratio": 0.0, "completion_length": 8.312500238418579, "epoch": 0.9185185185185185, "grad_norm": 12.709368015183454, "kl": 5.03125, "learning_rate": 1.179466039272168e-07, "loss": 0.2233, "reward": 0.21021388471126556, "reward_std": 0.02950027398765087, "rewards/accuracy_reward": 0.21021388471126556, "step": 1178 }, { "clip_ratio": 0.0, "completion_length": 9.145833492279053, "epoch": 0.9192982456140351, "grad_norm": 8.751122325128433, "kl": 3.12890625, "learning_rate": 1.1760539367605973e-07, "loss": 0.1993, "reward": 0.19363171607255936, "reward_std": 0.012781540863215923, "rewards/accuracy_reward": 0.19363171607255936, "step": 1179 }, { "clip_ratio": 0.0, "completion_length": 7.375000238418579, "epoch": 0.9200779727095516, "grad_norm": 18.81870982814433, "kl": 4.7421875, "learning_rate": 1.1726739354388738e-07, "loss": 0.3269, "reward": 0.21153492480516434, "reward_std": 0.01997565571218729, "rewards/accuracy_reward": 0.21153492480516434, "step": 1180 }, { "clip_ratio": 0.0, "completion_length": 7.500000238418579, "epoch": 0.9208576998050683, "grad_norm": 18.514301584667013, "kl": 6.4375, "learning_rate": 1.1693260604002953e-07, "loss": 0.3957, "reward": 0.20284295082092285, "reward_std": 0.01831751875579357, "rewards/accuracy_reward": 0.20284295082092285, "step": 1181 }, { "clip_ratio": 0.0, "completion_length": 8.041666984558105, "epoch": 0.9216374269005848, "grad_norm": 16.032920708448362, "kl": 5.28125, "learning_rate": 1.1660103364996511e-07, "loss": 0.4031, "reward": 0.20760168880224228, "reward_std": 0.027552330866456032, "rewards/accuracy_reward": 0.20760168880224228, "step": 1182 }, { "clip_ratio": 0.0, "completion_length": 6.958333492279053, "epoch": 0.9224171539961014, "grad_norm": 16.156417531867408, "kl": 8.46875, "learning_rate": 1.1627267883530406e-07, "loss": 0.4259, "reward": 0.20990155637264252, "reward_std": 0.0494983084499836, "rewards/accuracy_reward": 0.20990155637264252, "step": 1183 }, { "clip_ratio": 0.0, "completion_length": 8.291666746139526, "epoch": 0.9231968810916179, "grad_norm": 6.488863215628915, "kl": 3.359375, "learning_rate": 1.1594754403376874e-07, "loss": 0.1805, "reward": 0.1920343115925789, "reward_std": 0.005943312309682369, "rewards/accuracy_reward": 0.1920343115925789, "step": 1184 }, { "clip_ratio": 0.0, "completion_length": 7.125, "epoch": 0.9239766081871345, "grad_norm": 14.413775158356152, "kl": 5.875, "learning_rate": 1.1562563165917601e-07, "loss": 0.2764, "reward": 0.19793479144573212, "reward_std": 0.042699236422777176, "rewards/accuracy_reward": 0.19793479144573212, "step": 1185 }, { "clip_ratio": 0.0, "completion_length": 10.104166984558105, "epoch": 0.9247563352826511, "grad_norm": 3.5160482172980037, "kl": 3.65625, "learning_rate": 1.1530694410141929e-07, "loss": 0.1318, "reward": 0.19533730298280716, "reward_std": 0.00583947729319334, "rewards/accuracy_reward": 0.19533730298280716, "step": 1186 }, { "clip_ratio": 0.0, "completion_length": 7.1875, "epoch": 0.9255360623781677, "grad_norm": 15.227634775719885, "kl": 4.984375, "learning_rate": 1.1499148372645093e-07, "loss": 0.2668, "reward": 0.20154379308223724, "reward_std": 0.02526291087269783, "rewards/accuracy_reward": 0.20154379308223724, "step": 1187 }, { "clip_ratio": 0.0, "completion_length": 8.166666746139526, "epoch": 0.9263157894736842, "grad_norm": 12.846189521542879, "kl": 3.1171875, "learning_rate": 1.1467925287626427e-07, "loss": 0.1805, "reward": 0.19768907874822617, "reward_std": 0.013208545744419098, "rewards/accuracy_reward": 0.19768907874822617, "step": 1188 }, { "clip_ratio": 0.0, "completion_length": 7.4375, "epoch": 0.9270955165692008, "grad_norm": 5.698184317439598, "kl": 4.09375, "learning_rate": 1.1437025386887663e-07, "loss": 0.2474, "reward": 0.21061508357524872, "reward_std": 0.007805786095559597, "rewards/accuracy_reward": 0.21061508357524872, "step": 1189 }, { "clip_ratio": 0.0, "completion_length": 7.583333492279053, "epoch": 0.9278752436647173, "grad_norm": 8.350626761169142, "kl": 3.8359375, "learning_rate": 1.1406448899831195e-07, "loss": 0.1786, "reward": 0.2227737009525299, "reward_std": 0.00771215558052063, "rewards/accuracy_reward": 0.2227737009525299, "step": 1190 }, { "clip_ratio": 0.0, "completion_length": 9.916666984558105, "epoch": 0.928654970760234, "grad_norm": 22.456403327207482, "kl": 3.9375, "learning_rate": 1.137619605345838e-07, "loss": 0.2947, "reward": 0.18116123229265213, "reward_std": 0.01267927698791027, "rewards/accuracy_reward": 0.18116123229265213, "step": 1191 }, { "clip_ratio": 0.0, "completion_length": 7.520833492279053, "epoch": 0.9294346978557505, "grad_norm": 258.7369868877813, "kl": 15.25, "learning_rate": 1.1346267072367852e-07, "loss": 1.0728, "reward": 0.20177220553159714, "reward_std": 0.025494796689599752, "rewards/accuracy_reward": 0.20177220553159714, "step": 1192 }, { "clip_ratio": 0.0, "completion_length": 8.5625, "epoch": 0.930214424951267, "grad_norm": 31.576097396456294, "kl": 4.8671875, "learning_rate": 1.1316662178753838e-07, "loss": 0.3667, "reward": 0.19582483172416687, "reward_std": 0.028432806953787804, "rewards/accuracy_reward": 0.19582483172416687, "step": 1193 }, { "clip_ratio": 0.0, "completion_length": 7.854166746139526, "epoch": 0.9309941520467836, "grad_norm": 7.601727572238798, "kl": 2.609375, "learning_rate": 1.1287381592404547e-07, "loss": 0.0806, "reward": 0.1994047686457634, "reward_std": 0.014433756470680237, "rewards/accuracy_reward": 0.1994047686457634, "step": 1194 }, { "clip_ratio": 0.0, "completion_length": 10.708333492279053, "epoch": 0.9317738791423001, "grad_norm": 12.930760197596525, "kl": 2.3359375, "learning_rate": 1.12584255307005e-07, "loss": 0.1788, "reward": 0.17796655744314194, "reward_std": 0.006866272073239088, "rewards/accuracy_reward": 0.17796655744314194, "step": 1195 }, { "clip_ratio": 0.0, "completion_length": 6.9166669845581055, "epoch": 0.9325536062378168, "grad_norm": 12.495804845981928, "kl": 4.28125, "learning_rate": 1.1229794208612933e-07, "loss": 0.2171, "reward": 0.22080034762620926, "reward_std": 0.006542775314301252, "rewards/accuracy_reward": 0.22080034762620926, "step": 1196 }, { "clip_ratio": 0.0, "completion_length": 7.9375, "epoch": 0.9333333333333333, "grad_norm": 8.945474450309012, "kl": 4.078125, "learning_rate": 1.1201487838702199e-07, "loss": 0.1545, "reward": 0.22458858788013458, "reward_std": 0.013796973042190075, "rewards/accuracy_reward": 0.22458858788013458, "step": 1197 }, { "clip_ratio": 0.0, "completion_length": 7.458333492279053, "epoch": 0.9341130604288499, "grad_norm": 16.513812977339807, "kl": 5.703125, "learning_rate": 1.1173506631116191e-07, "loss": 0.3528, "reward": 0.19142332673072815, "reward_std": 0.016939722350798547, "rewards/accuracy_reward": 0.19142332673072815, "step": 1198 }, { "clip_ratio": 0.0, "completion_length": 8.000000476837158, "epoch": 0.9348927875243664, "grad_norm": 20.319445965948496, "kl": 5.53125, "learning_rate": 1.1145850793588782e-07, "loss": 0.4997, "reward": 0.23345467448234558, "reward_std": 0.011634243186563253, "rewards/accuracy_reward": 0.23345467448234558, "step": 1199 }, { "clip_ratio": 0.0, "completion_length": 9.500000476837158, "epoch": 0.935672514619883, "grad_norm": 25.01932496061275, "kl": 5.921875, "learning_rate": 1.1118520531438278e-07, "loss": 0.3437, "reward": 0.1736689805984497, "reward_std": 0.02426753845065832, "rewards/accuracy_reward": 0.1736689805984497, "step": 1200 }, { "clip_ratio": 0.0, "completion_length": 7.833333730697632, "epoch": 0.9364522417153996, "grad_norm": 3.786562131627424, "kl": 3.3671875, "learning_rate": 1.1091516047565902e-07, "loss": 0.1425, "reward": 0.22369793057441711, "reward_std": 0.004510548897087574, "rewards/accuracy_reward": 0.22369793057441711, "step": 1201 }, { "clip_ratio": 0.0, "completion_length": 7.770833492279053, "epoch": 0.9372319688109162, "grad_norm": 481.3115022134051, "kl": 22.78125, "learning_rate": 1.1064837542454282e-07, "loss": 1.4153, "reward": 0.18739942461252213, "reward_std": 0.03562930226325989, "rewards/accuracy_reward": 0.18739942461252213, "step": 1202 }, { "clip_ratio": 0.0, "completion_length": 7.916666746139526, "epoch": 0.9380116959064327, "grad_norm": 11.11658018073384, "kl": 4.8125, "learning_rate": 1.1038485214165961e-07, "loss": 0.2915, "reward": 0.2019917070865631, "reward_std": 0.03573805186897516, "rewards/accuracy_reward": 0.2019917070865631, "step": 1203 }, { "clip_ratio": 0.0, "completion_length": 8.645833492279053, "epoch": 0.9387914230019493, "grad_norm": 603.6540781376382, "kl": 20.65625, "learning_rate": 1.1012459258341926e-07, "loss": 2.0731, "reward": 0.18863649666309357, "reward_std": 0.0340578593313694, "rewards/accuracy_reward": 0.18863649666309357, "step": 1204 }, { "clip_ratio": 0.0, "completion_length": 7.458333492279053, "epoch": 0.9395711500974658, "grad_norm": 615.0637870484035, "kl": 23.28125, "learning_rate": 1.0986759868200167e-07, "loss": 1.3454, "reward": 0.20687806606292725, "reward_std": 0.02090786350890994, "rewards/accuracy_reward": 0.20687806606292725, "step": 1205 }, { "clip_ratio": 0.0, "completion_length": 7.625000238418579, "epoch": 0.9403508771929825, "grad_norm": 36.43824596412844, "kl": 9.6875, "learning_rate": 1.096138723453423e-07, "loss": 0.5498, "reward": 0.18897167593240738, "reward_std": 0.025384733453392982, "rewards/accuracy_reward": 0.18897167593240738, "step": 1206 }, { "clip_ratio": 0.0, "completion_length": 7.4375, "epoch": 0.941130604288499, "grad_norm": 33.525121208377314, "kl": 8.546875, "learning_rate": 1.0936341545711805e-07, "loss": 0.4616, "reward": 0.20922619104385376, "reward_std": 0.03994132578372955, "rewards/accuracy_reward": 0.20922619104385376, "step": 1207 }, { "clip_ratio": 0.0, "completion_length": 8.354166984558105, "epoch": 0.9419103313840156, "grad_norm": 902.3143949112615, "kl": 29.734375, "learning_rate": 1.0911622987673322e-07, "loss": 2.8071, "reward": 0.21465390920639038, "reward_std": 0.02480224333703518, "rewards/accuracy_reward": 0.21465390920639038, "step": 1208 }, { "clip_ratio": 0.0, "completion_length": 8.166666984558105, "epoch": 0.9426900584795321, "grad_norm": 85.01216407765074, "kl": 6.296875, "learning_rate": 1.0887231743930586e-07, "loss": 0.3739, "reward": 0.1962299346923828, "reward_std": 0.019409867003560066, "rewards/accuracy_reward": 0.1962299346923828, "step": 1209 }, { "clip_ratio": 0.0, "completion_length": 7.5416669845581055, "epoch": 0.9434697855750487, "grad_norm": 26.91519146651911, "kl": 3.140625, "learning_rate": 1.0863167995565398e-07, "loss": 0.1239, "reward": 0.2066294327378273, "reward_std": 0.0058517600409686565, "rewards/accuracy_reward": 0.2066294327378273, "step": 1210 }, { "clip_ratio": 0.0, "completion_length": 10.666666984558105, "epoch": 0.9442495126705653, "grad_norm": 28.40528646739898, "kl": 4.09375, "learning_rate": 1.083943192122822e-07, "loss": 0.4532, "reward": 0.1771974265575409, "reward_std": 0.026410337537527084, "rewards/accuracy_reward": 0.1771974265575409, "step": 1211 }, { "clip_ratio": 0.0, "completion_length": 12.145833492279053, "epoch": 0.9450292397660819, "grad_norm": 95.29599006350061, "kl": 9.0390625, "learning_rate": 1.0816023697136834e-07, "loss": 0.2776, "reward": 0.15802696347236633, "reward_std": 0.014931716956198215, "rewards/accuracy_reward": 0.15802696347236633, "step": 1212 }, { "clip_ratio": 0.0, "completion_length": 9.270833969116211, "epoch": 0.9458089668615984, "grad_norm": 1438.3356344164663, "kl": 43.328125, "learning_rate": 1.079294349707507e-07, "loss": 2.8694, "reward": 0.18272939324378967, "reward_std": 0.026362543925642967, "rewards/accuracy_reward": 0.18272939324378967, "step": 1213 }, { "clip_ratio": 0.0, "completion_length": 7.562500238418579, "epoch": 0.946588693957115, "grad_norm": 16.099709958046574, "kl": 7.765625, "learning_rate": 1.0770191492391467e-07, "loss": 0.5786, "reward": 0.21423230320215225, "reward_std": 0.038808904588222504, "rewards/accuracy_reward": 0.21423230320215225, "step": 1214 }, { "clip_ratio": 0.0, "completion_length": 8.0, "epoch": 0.9473684210526315, "grad_norm": 81.7876986375748, "kl": 8.5, "learning_rate": 1.0747767851998042e-07, "loss": 0.6992, "reward": 0.22134177386760712, "reward_std": 0.019505458883941174, "rewards/accuracy_reward": 0.22134177386760712, "step": 1215 }, { "clip_ratio": 0.0, "completion_length": 7.583333492279053, "epoch": 0.9481481481481482, "grad_norm": 9.954648623766223, "kl": 3.3984375, "learning_rate": 1.072567274236901e-07, "loss": 0.1362, "reward": 0.21450893580913544, "reward_std": 0.005412658676505089, "rewards/accuracy_reward": 0.21450893580913544, "step": 1216 }, { "clip_ratio": 0.0, "completion_length": 10.145833492279053, "epoch": 0.9489278752436647, "grad_norm": 93.48502279620097, "kl": 14.296875, "learning_rate": 1.0703906327539558e-07, "loss": 0.7091, "reward": 0.21654657274484634, "reward_std": 0.033344101160764694, "rewards/accuracy_reward": 0.21654657274484634, "step": 1217 }, { "clip_ratio": 0.0, "completion_length": 7.416666746139526, "epoch": 0.9497076023391813, "grad_norm": 20.659318990927677, "kl": 5.046875, "learning_rate": 1.068246876910463e-07, "loss": 0.28, "reward": 0.2123677358031273, "reward_std": 0.016037508845329285, "rewards/accuracy_reward": 0.2123677358031273, "step": 1218 }, { "clip_ratio": 0.0, "completion_length": 8.083333492279053, "epoch": 0.9504873294346978, "grad_norm": 164.37267681489453, "kl": 8.796875, "learning_rate": 1.0661360226217719e-07, "loss": 0.6351, "reward": 0.2137548327445984, "reward_std": 0.011381349992007017, "rewards/accuracy_reward": 0.2137548327445984, "step": 1219 }, { "clip_ratio": 0.0, "completion_length": 8.375000238418579, "epoch": 0.9512670565302144, "grad_norm": 19.27620049583751, "kl": 3.4765625, "learning_rate": 1.0640580855589704e-07, "loss": 0.4161, "reward": 0.2089814841747284, "reward_std": 0.012188505847007036, "rewards/accuracy_reward": 0.2089814841747284, "step": 1220 }, { "clip_ratio": 0.0, "completion_length": 15.479167461395264, "epoch": 0.952046783625731, "grad_norm": 7.730954997286787, "kl": 1.640625, "learning_rate": 1.0620130811487648e-07, "loss": 0.106, "reward": 0.14662089943885803, "reward_std": 0.0024267282569780946, "rewards/accuracy_reward": 0.14662089943885803, "step": 1221 }, { "clip_ratio": 0.0, "completion_length": 7.333333730697632, "epoch": 0.9528265107212476, "grad_norm": 32.25352365358629, "kl": 4.9375, "learning_rate": 1.0600010245733696e-07, "loss": 0.3226, "reward": 0.2036026269197464, "reward_std": 0.014297558926045895, "rewards/accuracy_reward": 0.2036026269197464, "step": 1222 }, { "clip_ratio": 0.0, "completion_length": 8.604166984558105, "epoch": 0.9536062378167641, "grad_norm": 13.334690298168347, "kl": 3.6796875, "learning_rate": 1.0580219307703931e-07, "loss": 0.2758, "reward": 0.21032074093818665, "reward_std": 0.0075492357718758285, "rewards/accuracy_reward": 0.21032074093818665, "step": 1223 }, { "clip_ratio": 0.0, "completion_length": 8.458333492279053, "epoch": 0.9543859649122807, "grad_norm": 1280.042720248613, "kl": 42.4375, "learning_rate": 1.0560758144327249e-07, "loss": 2.3375, "reward": 0.1932620033621788, "reward_std": 0.019000491127371788, "rewards/accuracy_reward": 0.1932620033621788, "step": 1224 }, { "clip_ratio": 0.0, "completion_length": 7.125, "epoch": 0.9551656920077972, "grad_norm": 3.092564535732343, "kl": 4.1640625, "learning_rate": 1.05416269000843e-07, "loss": 0.1704, "reward": 0.22726935893297195, "reward_std": 0.004510548897087574, "rewards/accuracy_reward": 0.22726935893297195, "step": 1225 }, { "clip_ratio": 0.0, "completion_length": 7.229166746139526, "epoch": 0.9559454191033139, "grad_norm": 18.288901438928676, "kl": 4.625, "learning_rate": 1.0522825717006393e-07, "loss": 0.3177, "reward": 0.2161763682961464, "reward_std": 0.0243800962343812, "rewards/accuracy_reward": 0.2161763682961464, "step": 1226 }, { "clip_ratio": 0.0, "completion_length": 9.416666984558105, "epoch": 0.9567251461988304, "grad_norm": 68.73787223708729, "kl": 8.75, "learning_rate": 1.0504354734674423e-07, "loss": 1.0187, "reward": 0.19006869941949844, "reward_std": 0.0508610624819994, "rewards/accuracy_reward": 0.19006869941949844, "step": 1227 }, { "clip_ratio": 0.0, "completion_length": 7.645833492279053, "epoch": 0.957504873294347, "grad_norm": 17.623840564827777, "kl": 5.3203125, "learning_rate": 1.04862140902179e-07, "loss": 0.3469, "reward": 0.19324291497468948, "reward_std": 0.015531366690993309, "rewards/accuracy_reward": 0.19324291497468948, "step": 1228 }, { "clip_ratio": 0.0, "completion_length": 7.0, "epoch": 0.9582846003898635, "grad_norm": 11.588559130133975, "kl": 3.65625, "learning_rate": 1.0468403918313855e-07, "loss": 0.1767, "reward": 0.21558122336864471, "reward_std": 0.0016980890650302172, "rewards/accuracy_reward": 0.21558122336864471, "step": 1229 }, { "clip_ratio": 0.0, "completion_length": 6.75, "epoch": 0.9590643274853801, "grad_norm": 12.342223267124131, "kl": 3.9921875, "learning_rate": 1.0450924351185888e-07, "loss": 0.1872, "reward": 0.22793931514024734, "reward_std": 0.008539648144505918, "rewards/accuracy_reward": 0.22793931514024734, "step": 1230 }, { "clip_ratio": 0.0, "completion_length": 10.145833492279053, "epoch": 0.9598440545808967, "grad_norm": 61.24098797262069, "kl": 4.6875, "learning_rate": 1.0433775518603174e-07, "loss": 0.2307, "reward": 0.2035786584019661, "reward_std": 0.009253811556845903, "rewards/accuracy_reward": 0.2035786584019661, "step": 1231 }, { "clip_ratio": 0.0, "completion_length": 7.583333492279053, "epoch": 0.9606237816764133, "grad_norm": 13.356084363838436, "kl": 4.5625, "learning_rate": 1.04169575478795e-07, "loss": 0.3373, "reward": 0.20350747555494308, "reward_std": 0.020974313840270042, "rewards/accuracy_reward": 0.20350747555494308, "step": 1232 }, { "clip_ratio": 0.0, "completion_length": 8.5, "epoch": 0.9614035087719298, "grad_norm": 8.445724184250526, "kl": 4.03125, "learning_rate": 1.0400470563872306e-07, "loss": 0.1708, "reward": 0.17264822870492935, "reward_std": 0.0009540432947687805, "rewards/accuracy_reward": 0.17264822870492935, "step": 1233 }, { "clip_ratio": 0.0, "completion_length": 6.979166746139526, "epoch": 0.9621832358674464, "grad_norm": 18.51814365364438, "kl": 4.15625, "learning_rate": 1.0384314688981785e-07, "loss": 0.2084, "reward": 0.21725810319185257, "reward_std": 0.007227339199744165, "rewards/accuracy_reward": 0.21725810319185257, "step": 1234 }, { "clip_ratio": 0.0, "completion_length": 8.083333730697632, "epoch": 0.9629629629629629, "grad_norm": 8.989129470598803, "kl": 5.375, "learning_rate": 1.0368490043149947e-07, "loss": 0.341, "reward": 0.20741870999336243, "reward_std": 0.01566131366416812, "rewards/accuracy_reward": 0.20741870999336243, "step": 1235 }, { "clip_ratio": 0.0, "completion_length": 7.6041669845581055, "epoch": 0.9637426900584796, "grad_norm": 19.988993760877054, "kl": 5.453125, "learning_rate": 1.0352996743859747e-07, "loss": 0.3337, "reward": 0.2078678384423256, "reward_std": 0.02216923190280795, "rewards/accuracy_reward": 0.2078678384423256, "step": 1236 }, { "clip_ratio": 0.0, "completion_length": 7.291666746139526, "epoch": 0.9645224171539961, "grad_norm": 1.6801326524284173, "kl": 3.53125, "learning_rate": 1.0337834906134214e-07, "loss": 0.141, "reward": 0.22857142984867096, "reward_std": 0.0, "rewards/accuracy_reward": 0.22857142984867096, "step": 1237 }, { "clip_ratio": 0.0, "completion_length": 7.270833492279053, "epoch": 0.9653021442495127, "grad_norm": 13.702818498569883, "kl": 3.5390625, "learning_rate": 1.0323004642535574e-07, "loss": 0.1354, "reward": 0.22127975523471832, "reward_std": 0.012887283228337765, "rewards/accuracy_reward": 0.22127975523471832, "step": 1238 }, { "clip_ratio": 0.0, "completion_length": 7.1041669845581055, "epoch": 0.9660818713450292, "grad_norm": 15.791125628059532, "kl": 4.765625, "learning_rate": 1.0308506063164444e-07, "loss": 0.2684, "reward": 0.2126072272658348, "reward_std": 0.018195314332842827, "rewards/accuracy_reward": 0.2126072272658348, "step": 1239 }, { "clip_ratio": 0.0, "completion_length": 7.979166746139526, "epoch": 0.9668615984405458, "grad_norm": 62443.014210519956, "kl": 1961.703125, "learning_rate": 1.0294339275658995e-07, "loss": 140.2023, "reward": 0.1819750964641571, "reward_std": 0.024145943112671375, "rewards/accuracy_reward": 0.1819750964641571, "step": 1240 }, { "clip_ratio": 0.0, "completion_length": 8.062500476837158, "epoch": 0.9676413255360624, "grad_norm": 18.503225726011376, "kl": 6.875, "learning_rate": 1.0280504385194162e-07, "loss": 0.2852, "reward": 0.20905672013759613, "reward_std": 0.03467312082648277, "rewards/accuracy_reward": 0.20905672013759613, "step": 1241 }, { "clip_ratio": 0.0, "completion_length": 7.3541669845581055, "epoch": 0.968421052631579, "grad_norm": 74.83209982632016, "kl": 10.09375, "learning_rate": 1.0267001494480849e-07, "loss": 0.5172, "reward": 0.2077132984995842, "reward_std": 0.03274921886622906, "rewards/accuracy_reward": 0.2077132984995842, "step": 1242 }, { "clip_ratio": 0.0, "completion_length": 7.750000238418579, "epoch": 0.9692007797270955, "grad_norm": 6954.26387137655, "kl": 121.25, "learning_rate": 1.0253830703765194e-07, "loss": 11.2978, "reward": 0.18995098769664764, "reward_std": 0.029535386711359024, "rewards/accuracy_reward": 0.18995098769664764, "step": 1243 }, { "clip_ratio": 0.0, "completion_length": 8.125, "epoch": 0.969980506822612, "grad_norm": 6.886992709405694, "kl": 3.265625, "learning_rate": 1.0240992110827795e-07, "loss": 0.1332, "reward": 0.21111111342906952, "reward_std": 0.0048112524673342705, "rewards/accuracy_reward": 0.21111111342906952, "step": 1244 }, { "clip_ratio": 0.0, "completion_length": 8.520833492279053, "epoch": 0.9707602339181286, "grad_norm": 26.88384821686633, "kl": 6.109375, "learning_rate": 1.0228485810983002e-07, "loss": 0.3363, "reward": 0.1592872068285942, "reward_std": 0.034972164779901505, "rewards/accuracy_reward": 0.1592872068285942, "step": 1245 }, { "clip_ratio": 0.0, "completion_length": 7.750000238418579, "epoch": 0.9715399610136453, "grad_norm": 6.611852730437946, "kl": 3.71875, "learning_rate": 1.0216311897078211e-07, "loss": 0.2004, "reward": 0.21078432351350784, "reward_std": 0.005943312309682369, "rewards/accuracy_reward": 0.21078432351350784, "step": 1246 }, { "clip_ratio": 0.0, "completion_length": 7.562500238418579, "epoch": 0.9723196881091618, "grad_norm": 287.6797822502787, "kl": 18.25, "learning_rate": 1.0204470459493156e-07, "loss": 1.0408, "reward": 0.21247275918722153, "reward_std": 0.037593014538288116, "rewards/accuracy_reward": 0.21247275918722153, "step": 1247 }, { "clip_ratio": 0.0, "completion_length": 7.458333492279053, "epoch": 0.9730994152046784, "grad_norm": 12.864392783785085, "kl": 4.953125, "learning_rate": 1.0192961586139258e-07, "loss": 0.2233, "reward": 0.19879601150751114, "reward_std": 0.023160387645475566, "rewards/accuracy_reward": 0.19879601150751114, "step": 1248 }, { "clip_ratio": 0.0, "completion_length": 7.354166746139526, "epoch": 0.9738791423001949, "grad_norm": 32.808840714369744, "kl": 7.171875, "learning_rate": 1.0181785362458968e-07, "loss": 0.4135, "reward": 0.22201142460107803, "reward_std": 0.02556948084384203, "rewards/accuracy_reward": 0.22201142460107803, "step": 1249 }, { "clip_ratio": 0.0, "completion_length": 6.708333492279053, "epoch": 0.9746588693957114, "grad_norm": 39.84092094840707, "kl": 6.65625, "learning_rate": 1.0170941871425117e-07, "loss": 0.3967, "reward": 0.21897321939468384, "reward_std": 0.01583898440003395, "rewards/accuracy_reward": 0.21897321939468384, "step": 1250 }, { "clip_ratio": 0.0, "completion_length": 7.916666746139526, "epoch": 0.9754385964912281, "grad_norm": 10.262711165620495, "kl": 2.3125, "learning_rate": 1.0160431193540322e-07, "loss": 0.0968, "reward": 0.203311026096344, "reward_std": 0.0009021099540404975, "rewards/accuracy_reward": 0.203311026096344, "step": 1251 }, { "clip_ratio": 0.0, "completion_length": 10.145833492279053, "epoch": 0.9762183235867447, "grad_norm": 8.736925306283432, "kl": 3.1171875, "learning_rate": 1.0150253406836379e-07, "loss": 0.1789, "reward": 0.1967427283525467, "reward_std": 0.002004689071327448, "rewards/accuracy_reward": 0.1967427283525467, "step": 1252 }, { "clip_ratio": 0.0, "completion_length": 8.083333492279053, "epoch": 0.9769980506822612, "grad_norm": 23.63281770606272, "kl": 8.15625, "learning_rate": 1.0140408586873678e-07, "loss": 0.511, "reward": 0.19002976268529892, "reward_std": 0.03935716859996319, "rewards/accuracy_reward": 0.19002976268529892, "step": 1253 }, { "clip_ratio": 0.0, "completion_length": 7.6041669845581055, "epoch": 0.9777777777777777, "grad_norm": 15.597506038748369, "kl": 5.7734375, "learning_rate": 1.0130896806740653e-07, "loss": 0.1966, "reward": 0.2057291641831398, "reward_std": 0.023454854264855385, "rewards/accuracy_reward": 0.2057291641831398, "step": 1254 }, { "clip_ratio": 0.0, "completion_length": 9.562500238418579, "epoch": 0.9785575048732943, "grad_norm": 139.57860657873223, "kl": 12.015625, "learning_rate": 1.0121718137053227e-07, "loss": 0.5991, "reward": 0.1736689880490303, "reward_std": 0.02626141719520092, "rewards/accuracy_reward": 0.1736689880490303, "step": 1255 }, { "clip_ratio": 0.0, "completion_length": 7.5625, "epoch": 0.979337231968811, "grad_norm": 4.295624653767669, "kl": 3.796875, "learning_rate": 1.0112872645954301e-07, "loss": 0.1564, "reward": 0.199768528342247, "reward_std": 0.007151786237955093, "rewards/accuracy_reward": 0.199768528342247, "step": 1256 }, { "clip_ratio": 0.0, "completion_length": 8.020833492279053, "epoch": 0.9801169590643275, "grad_norm": 12.883196542470925, "kl": 4.0625, "learning_rate": 1.0104360399113245e-07, "loss": 0.2487, "reward": 0.22572863847017288, "reward_std": 0.013378364033997059, "rewards/accuracy_reward": 0.22572863847017288, "step": 1257 }, { "clip_ratio": 0.0, "completion_length": 7.958333492279053, "epoch": 0.980896686159844, "grad_norm": 8.154645974639108, "kl": 4.765625, "learning_rate": 1.009618145972539e-07, "loss": 0.2565, "reward": 0.20494791120290756, "reward_std": 0.013742412440478802, "rewards/accuracy_reward": 0.20494791120290756, "step": 1258 }, { "clip_ratio": 0.0, "completion_length": 8.229166746139526, "epoch": 0.9816764132553606, "grad_norm": 161.61828964264342, "kl": 9.03125, "learning_rate": 1.0088335888511596e-07, "loss": 0.4625, "reward": 0.18164489418268204, "reward_std": 0.028004980646073818, "rewards/accuracy_reward": 0.18164489418268204, "step": 1259 }, { "clip_ratio": 0.0, "completion_length": 7.270833492279053, "epoch": 0.9824561403508771, "grad_norm": 231.5601376793861, "kl": 13.078125, "learning_rate": 1.0080823743717767e-07, "loss": 0.6933, "reward": 0.23341548442840576, "reward_std": 0.021758271381258965, "rewards/accuracy_reward": 0.23341548442840576, "step": 1260 }, { "clip_ratio": 0.0, "completion_length": 7.833333492279053, "epoch": 0.9832358674463938, "grad_norm": 21.823511268124925, "kl": 8.234375, "learning_rate": 1.0073645081114439e-07, "loss": 0.5074, "reward": 0.1992010995745659, "reward_std": 0.03368851449340582, "rewards/accuracy_reward": 0.1992010995745659, "step": 1261 }, { "clip_ratio": 0.0, "completion_length": 8.062500238418579, "epoch": 0.9840155945419103, "grad_norm": 11.9517005931049, "kl": 5.703125, "learning_rate": 1.0066799953996367e-07, "loss": 0.3365, "reward": 0.20743610709905624, "reward_std": 0.016178769059479237, "rewards/accuracy_reward": 0.20743610709905624, "step": 1262 }, { "clip_ratio": 0.0, "completion_length": 7.1666669845581055, "epoch": 0.9847953216374269, "grad_norm": 8.902749826657654, "kl": 4.953125, "learning_rate": 1.0060288413182106e-07, "loss": 0.2441, "reward": 0.22991492599248886, "reward_std": 0.005898410454392433, "rewards/accuracy_reward": 0.22991492599248886, "step": 1263 }, { "clip_ratio": 0.0, "completion_length": 8.895833492279053, "epoch": 0.9855750487329434, "grad_norm": 69.05943044232757, "kl": 6.59375, "learning_rate": 1.0054110507013654e-07, "loss": 0.8897, "reward": 0.21852556616067886, "reward_std": 0.014509346336126328, "rewards/accuracy_reward": 0.21852556616067886, "step": 1264 }, { "clip_ratio": 0.0, "completion_length": 7.458333492279053, "epoch": 0.98635477582846, "grad_norm": 12.546071757736438, "kl": 5.0, "learning_rate": 1.0048266281356104e-07, "loss": 0.3133, "reward": 0.19441090524196625, "reward_std": 0.023479340597987175, "rewards/accuracy_reward": 0.19441090524196625, "step": 1265 }, { "clip_ratio": 0.0, "completion_length": 7.895833730697632, "epoch": 0.9871345029239766, "grad_norm": 28.319472096410852, "kl": 7.28125, "learning_rate": 1.0042755779597266e-07, "loss": 0.6178, "reward": 0.20146438479423523, "reward_std": 0.0390668660402298, "rewards/accuracy_reward": 0.20146438479423523, "step": 1266 }, { "clip_ratio": 0.0, "completion_length": 12.645833492279053, "epoch": 0.9879142300194932, "grad_norm": 12.912836876293984, "kl": 3.0, "learning_rate": 1.0037579042647388e-07, "loss": 0.1384, "reward": 0.16597173362970352, "reward_std": 0.006705150939524174, "rewards/accuracy_reward": 0.16597173362970352, "step": 1267 }, { "clip_ratio": 0.0, "completion_length": 7.375000238418579, "epoch": 0.9886939571150097, "grad_norm": 11.633395365255938, "kl": 5.1953125, "learning_rate": 1.0032736108938814e-07, "loss": 0.3637, "reward": 0.2040233239531517, "reward_std": 0.01249195309355855, "rewards/accuracy_reward": 0.2040233239531517, "step": 1268 }, { "clip_ratio": 0.0, "completion_length": 8.437500476837158, "epoch": 0.9894736842105263, "grad_norm": 45.452321688826196, "kl": 7.8125, "learning_rate": 1.0028227014425727e-07, "loss": 0.615, "reward": 0.19626323133707047, "reward_std": 0.032283578999340534, "rewards/accuracy_reward": 0.19626323133707047, "step": 1269 }, { "clip_ratio": 0.0, "completion_length": 10.270833492279053, "epoch": 0.9902534113060428, "grad_norm": 37.94943283136805, "kl": 7.46875, "learning_rate": 1.0024051792583864e-07, "loss": 0.3525, "reward": 0.1793772429227829, "reward_std": 0.022501050494611263, "rewards/accuracy_reward": 0.1793772429227829, "step": 1270 }, { "clip_ratio": 0.0, "completion_length": 7.729166746139526, "epoch": 0.9910331384015595, "grad_norm": 7.035641756101844, "kl": 2.9921875, "learning_rate": 1.0020210474410287e-07, "loss": 0.1701, "reward": 0.2020089402794838, "reward_std": 0.005412658676505089, "rewards/accuracy_reward": 0.2020089402794838, "step": 1271 }, { "clip_ratio": 0.0, "completion_length": 8.208333492279053, "epoch": 0.991812865497076, "grad_norm": 56.37035675158799, "kl": 6.875, "learning_rate": 1.0016703088423122e-07, "loss": 0.3265, "reward": 0.18746256083250046, "reward_std": 0.02559092827141285, "rewards/accuracy_reward": 0.18746256083250046, "step": 1272 }, { "clip_ratio": 0.0, "completion_length": 8.0, "epoch": 0.9925925925925926, "grad_norm": 8.637001339240673, "kl": 3.4296875, "learning_rate": 1.0013529660661385e-07, "loss": 0.1913, "reward": 0.19130875170230865, "reward_std": 0.01228992361575365, "rewards/accuracy_reward": 0.19130875170230865, "step": 1273 }, { "clip_ratio": 0.0, "completion_length": 7.8541669845581055, "epoch": 0.9933723196881091, "grad_norm": 39.20174231712636, "kl": 4.8046875, "learning_rate": 1.0010690214684763e-07, "loss": 0.2005, "reward": 0.20710954070091248, "reward_std": 0.006324126850813627, "rewards/accuracy_reward": 0.20710954070091248, "step": 1274 }, { "clip_ratio": 0.0, "completion_length": 7.083333492279053, "epoch": 0.9941520467836257, "grad_norm": 19.651620932828706, "kl": 5.140625, "learning_rate": 1.0008184771573434e-07, "loss": 0.2307, "reward": 0.20983947068452835, "reward_std": 0.02158815204165876, "rewards/accuracy_reward": 0.20983947068452835, "step": 1275 }, { "clip_ratio": 0.0, "completion_length": 7.3541669845581055, "epoch": 0.9949317738791423, "grad_norm": 31.11847289386265, "kl": 4.875, "learning_rate": 1.0006013349927938e-07, "loss": 0.2856, "reward": 0.20519594103097916, "reward_std": 0.0067443439620547, "rewards/accuracy_reward": 0.20519594103097916, "step": 1276 }, { "clip_ratio": 0.0, "completion_length": 14.479166984558105, "epoch": 0.9957115009746589, "grad_norm": 157.28736066483953, "kl": 12.125, "learning_rate": 1.0004175965869012e-07, "loss": 0.7557, "reward": 0.1874784380197525, "reward_std": 0.014421529602259398, "rewards/accuracy_reward": 0.1874784380197525, "step": 1277 }, { "clip_ratio": 0.0, "completion_length": 7.208333492279053, "epoch": 0.9964912280701754, "grad_norm": 21.41478823472146, "kl": 7.296875, "learning_rate": 1.0002672633037497e-07, "loss": 0.4059, "reward": 0.22284622490406036, "reward_std": 0.015839688014239073, "rewards/accuracy_reward": 0.22284622490406036, "step": 1278 }, { "clip_ratio": 0.0, "completion_length": 8.104166984558105, "epoch": 0.997270955165692, "grad_norm": 6.684971065452833, "kl": 4.1484375, "learning_rate": 1.0001503362594198e-07, "loss": 0.1739, "reward": 0.20092594623565674, "reward_std": 0.006838416215032339, "rewards/accuracy_reward": 0.20092594623565674, "step": 1279 }, { "clip_ratio": 0.0, "completion_length": 6.958333492279053, "epoch": 0.9980506822612085, "grad_norm": 9.472871495229432, "kl": 3.5546875, "learning_rate": 1.0000668163219843e-07, "loss": 0.1427, "reward": 0.2108631134033203, "reward_std": 0.018042195588350296, "rewards/accuracy_reward": 0.2108631134033203, "step": 1280 }, { "clip_ratio": 0.0, "completion_length": 7.750000238418579, "epoch": 0.9988304093567252, "grad_norm": 11.175917789351356, "kl": 4.171875, "learning_rate": 1.000016704111499e-07, "loss": 0.2831, "reward": 0.19787301868200302, "reward_std": 0.023838849738240242, "rewards/accuracy_reward": 0.19787301868200302, "step": 1281 }, { "clip_ratio": 0.0, "completion_length": 6.833333253860474, "epoch": 0.9996101364522417, "grad_norm": 13.260794823295758, "kl": 3.9375, "learning_rate": 1e-07, "loss": 0.2462, "reward": 0.2180059626698494, "reward_std": 0.018042195588350296, "rewards/accuracy_reward": 0.2180059626698494, "step": 1282 }, { "epoch": 0.9996101364522417, "step": 1282, "total_flos": 0.0, "train_loss": 0.6022085310079814, "train_runtime": 10443.6197, "train_samples_per_second": 0.491, "train_steps_per_second": 0.123 } ], "logging_steps": 1, "max_steps": 1282, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 200, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 8, "trial_name": null, "trial_params": null }