diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,16708 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.9996101364522417, + "eval_steps": 500, + "global_step": 1282, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_ratio": 0.0, + "completion_length": 72.97917175292969, + "epoch": 0.0007797270955165692, + "grad_norm": 2.0360797534837407, + "kl": 0.0, + "learning_rate": 7.751937984496123e-09, + "loss": 0.1572, + "reward": 0.034821840934455395, + "reward_std": 0.006848668563179672, + "rewards/accuracy_reward": 0.034821840934455395, + "step": 1 + }, + { + "clip_ratio": 0.0, + "completion_length": 74.54166793823242, + "epoch": 0.0015594541910331384, + "grad_norm": 1.9195591077510774, + "kl": 0.0, + "learning_rate": 1.5503875968992246e-08, + "loss": 0.1859, + "reward": 0.035166485235095024, + "reward_std": 0.007111588958650827, + "rewards/accuracy_reward": 0.035166485235095024, + "step": 2 + }, + { + "clip_ratio": 0.0, + "completion_length": 85.10416793823242, + "epoch": 0.0023391812865497076, + "grad_norm": 2.077424101765198, + "kl": 2.440810203552246e-05, + "learning_rate": 2.3255813953488372e-08, + "loss": 0.3084, + "reward": 0.030526814982295036, + "reward_std": 0.005166596965864301, + "rewards/accuracy_reward": 0.030526814982295036, + "step": 3 + }, + { + "clip_ratio": 0.0, + "completion_length": 78.72916793823242, + "epoch": 0.0031189083820662767, + "grad_norm": 2.0593942152096165, + "kl": 4.89652156829834e-05, + "learning_rate": 3.100775193798449e-08, + "loss": 0.2624, + "reward": 0.031168147921562195, + "reward_std": 0.004581201123073697, + "rewards/accuracy_reward": 0.031168147921562195, + "step": 4 + }, + { + "clip_ratio": 0.0, + "completion_length": 69.10416984558105, + "epoch": 0.003898635477582846, + "grad_norm": 2.4255141009351355, + "kl": 5.245208740234375e-06, + "learning_rate": 3.8759689922480615e-08, + "loss": 0.3318, + "reward": 0.037738995626568794, + "reward_std": 0.00937926722690463, + "rewards/accuracy_reward": 0.037738995626568794, + "step": 5 + }, + { + "clip_ratio": 0.0, + "completion_length": 77.20833587646484, + "epoch": 0.004678362573099415, + "grad_norm": 2.277346720017906, + "kl": 4.00543212890625e-05, + "learning_rate": 4.6511627906976744e-08, + "loss": 0.1688, + "reward": 0.039112428203225136, + "reward_std": 0.011630565393716097, + "rewards/accuracy_reward": 0.039112428203225136, + "step": 6 + }, + { + "clip_ratio": 0.0, + "completion_length": 63.812503814697266, + "epoch": 0.005458089668615985, + "grad_norm": 2.49297928409578, + "kl": -2.3618340492248535e-06, + "learning_rate": 5.426356589147287e-08, + "loss": 0.1724, + "reward": 0.04318097606301308, + "reward_std": 0.009315338917076588, + "rewards/accuracy_reward": 0.04318097606301308, + "step": 7 + }, + { + "clip_ratio": 0.0, + "completion_length": 73.39583587646484, + "epoch": 0.006237816764132553, + "grad_norm": 2.452193481562994, + "kl": 2.4437904357910156e-05, + "learning_rate": 6.201550387596898e-08, + "loss": 0.3233, + "reward": 0.03827681764960289, + "reward_std": 0.006922724889591336, + "rewards/accuracy_reward": 0.03827681764960289, + "step": 8 + }, + { + "clip_ratio": 0.0, + "completion_length": 84.62500381469727, + "epoch": 0.007017543859649123, + "grad_norm": 1.9369529915399666, + "kl": -2.384185791015625e-07, + "learning_rate": 6.976744186046511e-08, + "loss": 0.1277, + "reward": 0.033217170275747776, + "reward_std": 0.006850299425423145, + "rewards/accuracy_reward": 0.033217170275747776, + "step": 9 + }, + { + "clip_ratio": 0.0, + "completion_length": 72.45833587646484, + "epoch": 0.007797270955165692, + "grad_norm": 2.165737547300236, + "kl": 1.2695789337158203e-05, + "learning_rate": 7.751937984496123e-08, + "loss": 0.3101, + "reward": 0.033010080456733704, + "reward_std": 0.004679354839026928, + "rewards/accuracy_reward": 0.033010080456733704, + "step": 10 + }, + { + "clip_ratio": 0.0, + "completion_length": 42.43750190734863, + "epoch": 0.008576998050682261, + "grad_norm": 3.9467304654500217, + "kl": 8.845329284667969e-05, + "learning_rate": 8.527131782945737e-08, + "loss": 0.2326, + "reward": 0.05289607308804989, + "reward_std": 0.011021225713193417, + "rewards/accuracy_reward": 0.05289607308804989, + "step": 11 + }, + { + "clip_ratio": 0.0, + "completion_length": 98.04166793823242, + "epoch": 0.00935672514619883, + "grad_norm": 3.435577459737531, + "kl": 5.418062210083008e-05, + "learning_rate": 9.302325581395349e-08, + "loss": 0.4666, + "reward": 0.04114745929837227, + "reward_std": 0.010439948178827763, + "rewards/accuracy_reward": 0.04114745929837227, + "step": 12 + }, + { + "clip_ratio": 0.0, + "completion_length": 74.72916793823242, + "epoch": 0.0101364522417154, + "grad_norm": 1.8829822654527015, + "kl": -7.525086402893066e-06, + "learning_rate": 1.0077519379844961e-07, + "loss": 0.1082, + "reward": 0.0373533945530653, + "reward_std": 0.010985687375068665, + "rewards/accuracy_reward": 0.0373533945530653, + "step": 13 + }, + { + "clip_ratio": 0.0, + "completion_length": 46.5, + "epoch": 0.01091617933723197, + "grad_norm": 3.9183025177944577, + "kl": 4.0650367736816406e-05, + "learning_rate": 1.0852713178294573e-07, + "loss": 0.2163, + "reward": 0.048170048743486404, + "reward_std": 0.009852790739387274, + "rewards/accuracy_reward": 0.048170048743486404, + "step": 14 + }, + { + "clip_ratio": 0.0, + "completion_length": 79.58333587646484, + "epoch": 0.011695906432748537, + "grad_norm": 2.052544767124884, + "kl": 6.449222564697266e-05, + "learning_rate": 1.1627906976744186e-07, + "loss": 0.2347, + "reward": 0.036619529128074646, + "reward_std": 0.008851302787661552, + "rewards/accuracy_reward": 0.036619529128074646, + "step": 15 + }, + { + "clip_ratio": 0.0, + "completion_length": 59.89583396911621, + "epoch": 0.012475633528265107, + "grad_norm": 2.656217708142409, + "kl": -8.165836334228516e-06, + "learning_rate": 1.2403100775193797e-07, + "loss": 0.2568, + "reward": 0.03845728747546673, + "reward_std": 0.00811865832656622, + "rewards/accuracy_reward": 0.03845728747546673, + "step": 16 + }, + { + "clip_ratio": 0.0, + "completion_length": 80.22917175292969, + "epoch": 0.013255360623781676, + "grad_norm": 1.9427871138959392, + "kl": 3.3020973205566406e-05, + "learning_rate": 1.317829457364341e-07, + "loss": 0.2086, + "reward": 0.03203662671148777, + "reward_std": 0.003987034084275365, + "rewards/accuracy_reward": 0.03203662671148777, + "step": 17 + }, + { + "clip_ratio": 0.0, + "completion_length": 63.416669845581055, + "epoch": 0.014035087719298246, + "grad_norm": 3.33750190352222, + "kl": 3.084540367126465e-05, + "learning_rate": 1.3953488372093021e-07, + "loss": 0.1715, + "reward": 0.04148095287382603, + "reward_std": 0.014872618019580841, + "rewards/accuracy_reward": 0.04148095287382603, + "step": 18 + }, + { + "clip_ratio": 0.0, + "completion_length": 69.85417175292969, + "epoch": 0.014814814814814815, + "grad_norm": 2.0649953692424075, + "kl": 1.9073486328125e-05, + "learning_rate": 1.4728682170542635e-07, + "loss": 0.2053, + "reward": 0.04008662328124046, + "reward_std": 0.008591718040406704, + "rewards/accuracy_reward": 0.04008662328124046, + "step": 19 + }, + { + "clip_ratio": 0.0, + "completion_length": 78.79167175292969, + "epoch": 0.015594541910331383, + "grad_norm": 2.0297255573747295, + "kl": -1.74790620803833e-05, + "learning_rate": 1.5503875968992246e-07, + "loss": 0.322, + "reward": 0.03438205271959305, + "reward_std": 0.007812638068571687, + "rewards/accuracy_reward": 0.03438205271959305, + "step": 20 + }, + { + "clip_ratio": 0.0, + "completion_length": 65.52083587646484, + "epoch": 0.016374269005847954, + "grad_norm": 2.4261350923804264, + "kl": 1.4513731002807617e-05, + "learning_rate": 1.6279069767441862e-07, + "loss": 0.2003, + "reward": 0.03825371153652668, + "reward_std": 0.008155276766046882, + "rewards/accuracy_reward": 0.03825371153652668, + "step": 21 + }, + { + "clip_ratio": 0.0, + "completion_length": 71.625, + "epoch": 0.017153996101364522, + "grad_norm": 1.9250078658310887, + "kl": -2.5033950805664062e-06, + "learning_rate": 1.7054263565891473e-07, + "loss": 0.0647, + "reward": 0.03269452229142189, + "reward_std": 0.008812714833766222, + "rewards/accuracy_reward": 0.03269452229142189, + "step": 22 + }, + { + "clip_ratio": 0.0, + "completion_length": 79.29166793823242, + "epoch": 0.01793372319688109, + "grad_norm": 2.010657233810007, + "kl": 2.331542782485485e-05, + "learning_rate": 1.7829457364341087e-07, + "loss": 0.2563, + "reward": 0.03494575526565313, + "reward_std": 0.006035172613337636, + "rewards/accuracy_reward": 0.03494575526565313, + "step": 23 + }, + { + "clip_ratio": 0.0, + "completion_length": 76.20833587646484, + "epoch": 0.01871345029239766, + "grad_norm": 2.0027894508025343, + "kl": 8.577108383178711e-05, + "learning_rate": 1.8604651162790698e-07, + "loss": 0.1875, + "reward": 0.030015477910637856, + "reward_std": 0.0032516615465283394, + "rewards/accuracy_reward": 0.030015477910637856, + "step": 24 + }, + { + "clip_ratio": 0.0, + "completion_length": 68.52083587646484, + "epoch": 0.01949317738791423, + "grad_norm": 2.444697061646816, + "kl": 7.224082946777344e-05, + "learning_rate": 1.9379844961240311e-07, + "loss": 0.312, + "reward": 0.03344830125570297, + "reward_std": 0.004560418426990509, + "rewards/accuracy_reward": 0.03344830125570297, + "step": 25 + }, + { + "clip_ratio": 0.0, + "completion_length": 64.97916793823242, + "epoch": 0.0202729044834308, + "grad_norm": 2.532369903498374, + "kl": 8.785724639892578e-05, + "learning_rate": 2.0155038759689922e-07, + "loss": 0.0783, + "reward": 0.04169601574540138, + "reward_std": 0.015225842595100403, + "rewards/accuracy_reward": 0.04169601574540138, + "step": 26 + }, + { + "clip_ratio": 0.0, + "completion_length": 70.77083587646484, + "epoch": 0.021052631578947368, + "grad_norm": 2.3095745232275755, + "kl": 5.412101745605469e-05, + "learning_rate": 2.0930232558139536e-07, + "loss": 0.1652, + "reward": 0.03223975747823715, + "reward_std": 0.0025580759393051267, + "rewards/accuracy_reward": 0.03223975747823715, + "step": 27 + }, + { + "clip_ratio": 0.0, + "completion_length": 68.93750190734863, + "epoch": 0.02183235867446394, + "grad_norm": 2.788800470549753, + "kl": 0.0001437664031982422, + "learning_rate": 2.1705426356589147e-07, + "loss": 0.2741, + "reward": 0.036526650190353394, + "reward_std": 0.005832177121192217, + "rewards/accuracy_reward": 0.036526650190353394, + "step": 28 + }, + { + "clip_ratio": 0.0, + "completion_length": 74.87500190734863, + "epoch": 0.022612085769980507, + "grad_norm": 2.282049473570082, + "kl": 0.00024366378784179688, + "learning_rate": 2.248062015503876e-07, + "loss": 0.4466, + "reward": 0.03732151258736849, + "reward_std": 0.0073881205171346664, + "rewards/accuracy_reward": 0.03732151258736849, + "step": 29 + }, + { + "clip_ratio": 0.0, + "completion_length": 65.12500190734863, + "epoch": 0.023391812865497075, + "grad_norm": 2.8771857181708316, + "kl": 0.00025177001953125, + "learning_rate": 2.3255813953488372e-07, + "loss": 0.3322, + "reward": 0.0398801788687706, + "reward_std": 0.011091248597949743, + "rewards/accuracy_reward": 0.0398801788687706, + "step": 30 + }, + { + "clip_ratio": 0.0, + "completion_length": 68.22916793823242, + "epoch": 0.024171539961013646, + "grad_norm": 2.9801638513714446, + "kl": 0.0003724098205566406, + "learning_rate": 2.403100775193798e-07, + "loss": 0.1957, + "reward": 0.04014836438000202, + "reward_std": 0.010948444716632366, + "rewards/accuracy_reward": 0.04014836438000202, + "step": 31 + }, + { + "clip_ratio": 0.0, + "completion_length": 71.02083587646484, + "epoch": 0.024951267056530214, + "grad_norm": 2.7228841256045686, + "kl": 0.00022363662719726562, + "learning_rate": 2.4806201550387593e-07, + "loss": 0.1496, + "reward": 0.037018182687461376, + "reward_std": 0.00551977613940835, + "rewards/accuracy_reward": 0.037018182687461376, + "step": 32 + }, + { + "clip_ratio": 0.0, + "completion_length": 72.875, + "epoch": 0.025730994152046785, + "grad_norm": 2.1519240668442303, + "kl": 0.00014019012451171875, + "learning_rate": 2.558139534883721e-07, + "loss": 0.1798, + "reward": 0.031542438082396984, + "reward_std": 0.006069755996577442, + "rewards/accuracy_reward": 0.031542438082396984, + "step": 33 + }, + { + "clip_ratio": 0.0, + "completion_length": 122.20833969116211, + "epoch": 0.026510721247563353, + "grad_norm": 1.9690755699819855, + "kl": 0.00036907196044921875, + "learning_rate": 2.635658914728682e-07, + "loss": 0.4417, + "reward": 0.032262321561574936, + "reward_std": 0.008104364853352308, + "rewards/accuracy_reward": 0.032262321561574936, + "step": 34 + }, + { + "clip_ratio": 0.0, + "completion_length": 58.35416793823242, + "epoch": 0.02729044834307992, + "grad_norm": 2.3964490155178697, + "kl": 0.0004863739013671875, + "learning_rate": 2.713178294573643e-07, + "loss": 0.2393, + "reward": 0.04383615590631962, + "reward_std": 0.010008457116782665, + "rewards/accuracy_reward": 0.04383615590631962, + "step": 35 + }, + { + "clip_ratio": 0.0, + "completion_length": 60.625003814697266, + "epoch": 0.028070175438596492, + "grad_norm": 2.486430271357229, + "kl": 0.0010223388671875, + "learning_rate": 2.7906976744186043e-07, + "loss": 0.2161, + "reward": 0.0347980335354805, + "reward_std": 0.006134419701993465, + "rewards/accuracy_reward": 0.0347980335354805, + "step": 36 + }, + { + "clip_ratio": 0.0, + "completion_length": 49.39583396911621, + "epoch": 0.02884990253411306, + "grad_norm": 3.782970773046541, + "kl": 0.001277923583984375, + "learning_rate": 2.868217054263566e-07, + "loss": 0.3321, + "reward": 0.04106608219444752, + "reward_std": 0.006041216081939638, + "rewards/accuracy_reward": 0.04106608219444752, + "step": 37 + }, + { + "clip_ratio": 0.0, + "completion_length": 70.0625, + "epoch": 0.02962962962962963, + "grad_norm": 2.6091415664520716, + "kl": 0.001514434814453125, + "learning_rate": 2.945736434108527e-07, + "loss": 0.2164, + "reward": 0.04012400843203068, + "reward_std": 0.010114745702594519, + "rewards/accuracy_reward": 0.04012400843203068, + "step": 38 + }, + { + "clip_ratio": 0.0, + "completion_length": 62.72916793823242, + "epoch": 0.0304093567251462, + "grad_norm": 2.2160406971755457, + "kl": 0.0016937255859375, + "learning_rate": 3.023255813953488e-07, + "loss": 0.1534, + "reward": 0.03779347240924835, + "reward_std": 0.005149615230038762, + "rewards/accuracy_reward": 0.03779347240924835, + "step": 39 + }, + { + "clip_ratio": 0.0, + "completion_length": 75.89583587646484, + "epoch": 0.031189083820662766, + "grad_norm": 2.12251220530095, + "kl": 0.001483917236328125, + "learning_rate": 3.100775193798449e-07, + "loss": 0.0937, + "reward": 0.03355565946549177, + "reward_std": 0.005913240602239966, + "rewards/accuracy_reward": 0.03355565946549177, + "step": 40 + }, + { + "clip_ratio": 0.0, + "completion_length": 55.770835876464844, + "epoch": 0.031968810916179334, + "grad_norm": 2.628803783097872, + "kl": 0.001811981201171875, + "learning_rate": 3.178294573643411e-07, + "loss": 0.1487, + "reward": 0.04058343544602394, + "reward_std": 0.007635206216946244, + "rewards/accuracy_reward": 0.04058343544602394, + "step": 41 + }, + { + "clip_ratio": 0.0, + "completion_length": 69.70833396911621, + "epoch": 0.03274853801169591, + "grad_norm": 2.9487477608573878, + "kl": 0.001972198486328125, + "learning_rate": 3.2558139534883724e-07, + "loss": 0.1824, + "reward": 0.04240095615386963, + "reward_std": 0.013564284890890121, + "rewards/accuracy_reward": 0.04240095615386963, + "step": 42 + }, + { + "clip_ratio": 0.0, + "completion_length": 80.83333587646484, + "epoch": 0.03352826510721248, + "grad_norm": 2.2158750430327725, + "kl": 0.001964569091796875, + "learning_rate": 3.333333333333333e-07, + "loss": 0.4318, + "reward": 0.03499251697212458, + "reward_std": 0.009471837896853685, + "rewards/accuracy_reward": 0.03499251697212458, + "step": 43 + }, + { + "clip_ratio": 0.0, + "completion_length": 63.354169845581055, + "epoch": 0.034307992202729044, + "grad_norm": 2.3484723186586662, + "kl": 0.0038604736328125, + "learning_rate": 3.4108527131782946e-07, + "loss": 0.1758, + "reward": 0.04066397808492184, + "reward_std": 0.012765896040946245, + "rewards/accuracy_reward": 0.04066397808492184, + "step": 44 + }, + { + "clip_ratio": 0.0, + "completion_length": 54.06250190734863, + "epoch": 0.03508771929824561, + "grad_norm": 3.1431566341022905, + "kl": 0.006072998046875, + "learning_rate": 3.4883720930232557e-07, + "loss": 0.3093, + "reward": 0.04505058936774731, + "reward_std": 0.009412168525159359, + "rewards/accuracy_reward": 0.04505058936774731, + "step": 45 + }, + { + "clip_ratio": 0.0, + "completion_length": 68.25000190734863, + "epoch": 0.03586744639376218, + "grad_norm": 2.2081442575684522, + "kl": 0.0042266845703125, + "learning_rate": 3.5658914728682174e-07, + "loss": 0.0523, + "reward": 0.0339159881696105, + "reward_std": 0.010420492850244045, + "rewards/accuracy_reward": 0.0339159881696105, + "step": 46 + }, + { + "clip_ratio": 0.0, + "completion_length": 54.25000190734863, + "epoch": 0.036647173489278755, + "grad_norm": 2.8522356482704643, + "kl": 0.00418853759765625, + "learning_rate": 3.643410852713178e-07, + "loss": 0.1387, + "reward": 0.04024900496006012, + "reward_std": 0.0033548662904649973, + "rewards/accuracy_reward": 0.04024900496006012, + "step": 47 + }, + { + "clip_ratio": 0.0, + "completion_length": 50.70833396911621, + "epoch": 0.03742690058479532, + "grad_norm": 3.1034968757450483, + "kl": 0.00927734375, + "learning_rate": 3.7209302325581396e-07, + "loss": 0.2677, + "reward": 0.04862676560878754, + "reward_std": 0.010755512397736311, + "rewards/accuracy_reward": 0.04862676560878754, + "step": 48 + }, + { + "clip_ratio": 0.0, + "completion_length": 67.29166984558105, + "epoch": 0.03820662768031189, + "grad_norm": 2.3737010947513615, + "kl": 0.0064544677734375, + "learning_rate": 3.7984496124031006e-07, + "loss": 0.2127, + "reward": 0.03502965345978737, + "reward_std": 0.006760421558283269, + "rewards/accuracy_reward": 0.03502965345978737, + "step": 49 + }, + { + "clip_ratio": 0.0, + "completion_length": 58.625003814697266, + "epoch": 0.03898635477582846, + "grad_norm": 3.303783495475012, + "kl": 0.00738525390625, + "learning_rate": 3.8759689922480623e-07, + "loss": 0.1732, + "reward": 0.04058924503624439, + "reward_std": 0.005226747365668416, + "rewards/accuracy_reward": 0.04058924503624439, + "step": 50 + }, + { + "clip_ratio": 0.0, + "completion_length": 43.43750190734863, + "epoch": 0.03976608187134503, + "grad_norm": 3.113620942783793, + "kl": 0.0135498046875, + "learning_rate": 3.953488372093023e-07, + "loss": 0.0656, + "reward": 0.05482747219502926, + "reward_std": 0.01050096983090043, + "rewards/accuracy_reward": 0.05482747219502926, + "step": 51 + }, + { + "clip_ratio": 0.0, + "completion_length": 45.62500190734863, + "epoch": 0.0405458089668616, + "grad_norm": 2.8610826676549244, + "kl": 0.009246826171875, + "learning_rate": 4.0310077519379845e-07, + "loss": 0.0302, + "reward": 0.04160502180457115, + "reward_std": 0.006433177040889859, + "rewards/accuracy_reward": 0.04160502180457115, + "step": 52 + }, + { + "clip_ratio": 0.0, + "completion_length": 50.00000190734863, + "epoch": 0.04132553606237817, + "grad_norm": 2.7999246179813975, + "kl": 0.01318359375, + "learning_rate": 4.1085271317829456e-07, + "loss": 0.233, + "reward": 0.043028466403484344, + "reward_std": 0.003487872891128063, + "rewards/accuracy_reward": 0.043028466403484344, + "step": 53 + }, + { + "clip_ratio": 0.0, + "completion_length": 52.10416793823242, + "epoch": 0.042105263157894736, + "grad_norm": 2.568960242942063, + "kl": 0.01348876953125, + "learning_rate": 4.186046511627907e-07, + "loss": 0.1753, + "reward": 0.038340842351317406, + "reward_std": 0.007927641505375504, + "rewards/accuracy_reward": 0.038340842351317406, + "step": 54 + }, + { + "clip_ratio": 0.0, + "completion_length": 57.16666793823242, + "epoch": 0.042884990253411304, + "grad_norm": 2.4342441327480553, + "kl": 0.0233154296875, + "learning_rate": 4.263565891472868e-07, + "loss": 0.0992, + "reward": 0.047976596280932426, + "reward_std": 0.013746438082307577, + "rewards/accuracy_reward": 0.047976596280932426, + "step": 55 + }, + { + "clip_ratio": 0.0, + "completion_length": 60.520835876464844, + "epoch": 0.04366471734892788, + "grad_norm": 2.770172478808812, + "kl": 0.023834228515625, + "learning_rate": 4.3410852713178294e-07, + "loss": 0.2833, + "reward": 0.03680317848920822, + "reward_std": 0.005510715185664594, + "rewards/accuracy_reward": 0.03680317848920822, + "step": 56 + }, + { + "clip_ratio": 0.0, + "completion_length": 39.35416793823242, + "epoch": 0.044444444444444446, + "grad_norm": 3.68010060383285, + "kl": 0.03204345703125, + "learning_rate": 4.4186046511627905e-07, + "loss": 0.1525, + "reward": 0.04910232499241829, + "reward_std": 0.007614273577928543, + "rewards/accuracy_reward": 0.04910232499241829, + "step": 57 + }, + { + "clip_ratio": 0.0, + "completion_length": 53.250003814697266, + "epoch": 0.045224171539961014, + "grad_norm": 2.70642849083914, + "kl": 0.0291748046875, + "learning_rate": 4.496124031007752e-07, + "loss": 0.0844, + "reward": 0.04976135119795799, + "reward_std": 0.011796995531767607, + "rewards/accuracy_reward": 0.04976135119795799, + "step": 58 + }, + { + "clip_ratio": 0.0, + "completion_length": 50.895835876464844, + "epoch": 0.04600389863547758, + "grad_norm": 2.812435002261949, + "kl": 0.03460693359375, + "learning_rate": 4.5736434108527127e-07, + "loss": 0.0578, + "reward": 0.047287384048104286, + "reward_std": 0.011063814163208008, + "rewards/accuracy_reward": 0.047287384048104286, + "step": 59 + }, + { + "clip_ratio": 0.0, + "completion_length": 46.60416793823242, + "epoch": 0.04678362573099415, + "grad_norm": 3.1631616288164066, + "kl": 0.050537109375, + "learning_rate": 4.6511627906976743e-07, + "loss": 0.1621, + "reward": 0.04765086621046066, + "reward_std": 0.007884890539571643, + "rewards/accuracy_reward": 0.04765086621046066, + "step": 60 + }, + { + "clip_ratio": 0.0, + "completion_length": 52.125, + "epoch": 0.047563352826510724, + "grad_norm": 3.4169815314016287, + "kl": 0.040283203125, + "learning_rate": 4.7286821705426354e-07, + "loss": 0.1953, + "reward": 0.037693820893764496, + "reward_std": 0.0043515488505363464, + "rewards/accuracy_reward": 0.037693820893764496, + "step": 61 + }, + { + "clip_ratio": 0.0, + "completion_length": 44.89583396911621, + "epoch": 0.04834307992202729, + "grad_norm": 3.7525171815893428, + "kl": 0.0565185546875, + "learning_rate": 4.806201550387597e-07, + "loss": 0.2159, + "reward": 0.0389871671795845, + "reward_std": 0.004751588683575392, + "rewards/accuracy_reward": 0.0389871671795845, + "step": 62 + }, + { + "clip_ratio": 0.0, + "completion_length": 44.06250190734863, + "epoch": 0.04912280701754386, + "grad_norm": 3.1742788723870086, + "kl": 0.070068359375, + "learning_rate": 4.883720930232558e-07, + "loss": 0.0506, + "reward": 0.05800568871200085, + "reward_std": 0.008456971496343613, + "rewards/accuracy_reward": 0.05800568871200085, + "step": 63 + }, + { + "clip_ratio": 0.0, + "completion_length": 39.72916793823242, + "epoch": 0.04990253411306043, + "grad_norm": 3.3869515054521253, + "kl": 0.085205078125, + "learning_rate": 4.961240310077519e-07, + "loss": -0.0272, + "reward": 0.07096873968839645, + "reward_std": 0.01401633769273758, + "rewards/accuracy_reward": 0.07096873968839645, + "step": 64 + }, + { + "clip_ratio": 0.0, + "completion_length": 51.208335876464844, + "epoch": 0.050682261208576995, + "grad_norm": 2.2197188203978144, + "kl": 0.070068359375, + "learning_rate": 5.038759689922481e-07, + "loss": 0.1532, + "reward": 0.04562339745461941, + "reward_std": 0.007543514017015696, + "rewards/accuracy_reward": 0.04562339745461941, + "step": 65 + }, + { + "clip_ratio": 0.0, + "completion_length": 34.43750190734863, + "epoch": 0.05146198830409357, + "grad_norm": 3.6225373710567994, + "kl": 0.07861328125, + "learning_rate": 5.116279069767442e-07, + "loss": 0.1782, + "reward": 0.04998355731368065, + "reward_std": 0.004972400842234492, + "rewards/accuracy_reward": 0.04998355731368065, + "step": 66 + }, + { + "clip_ratio": 0.0, + "completion_length": 48.3125, + "epoch": 0.05224171539961014, + "grad_norm": 4.479964724095867, + "kl": 0.093017578125, + "learning_rate": 5.193798449612403e-07, + "loss": 0.1327, + "reward": 0.05973537266254425, + "reward_std": 0.01126884389668703, + "rewards/accuracy_reward": 0.05973537266254425, + "step": 67 + }, + { + "clip_ratio": 0.0, + "completion_length": 42.08333396911621, + "epoch": 0.053021442495126705, + "grad_norm": 3.47463936829341, + "kl": 0.111328125, + "learning_rate": 5.271317829457364e-07, + "loss": 0.0991, + "reward": 0.05231078714132309, + "reward_std": 0.004367304500192404, + "rewards/accuracy_reward": 0.05231078714132309, + "step": 68 + }, + { + "clip_ratio": 0.0, + "completion_length": 45.85416793823242, + "epoch": 0.05380116959064327, + "grad_norm": 2.984127507818342, + "kl": 0.0819091796875, + "learning_rate": 5.348837209302325e-07, + "loss": 0.0718, + "reward": 0.04398638755083084, + "reward_std": 0.0028009897796437144, + "rewards/accuracy_reward": 0.04398638755083084, + "step": 69 + }, + { + "clip_ratio": 0.0, + "completion_length": 53.29166793823242, + "epoch": 0.05458089668615984, + "grad_norm": 2.5160706221746945, + "kl": 0.130126953125, + "learning_rate": 5.426356589147286e-07, + "loss": 0.1763, + "reward": 0.054356709122657776, + "reward_std": 0.009095613844692707, + "rewards/accuracy_reward": 0.054356709122657776, + "step": 70 + }, + { + "clip_ratio": 0.0, + "completion_length": 31.187501907348633, + "epoch": 0.055360623781676416, + "grad_norm": 3.5612309454881075, + "kl": 0.18359375, + "learning_rate": 5.503875968992247e-07, + "loss": 0.064, + "reward": 0.06594585627317429, + "reward_std": 0.008575513027608395, + "rewards/accuracy_reward": 0.06594585627317429, + "step": 71 + }, + { + "clip_ratio": 0.0, + "completion_length": 36.5, + "epoch": 0.056140350877192984, + "grad_norm": 2.8448103966973943, + "kl": 0.176025390625, + "learning_rate": 5.581395348837209e-07, + "loss": 0.0491, + "reward": 0.06807346642017365, + "reward_std": 0.011498718522489071, + "rewards/accuracy_reward": 0.06807346642017365, + "step": 72 + }, + { + "clip_ratio": 0.0, + "completion_length": 34.95833396911621, + "epoch": 0.05692007797270955, + "grad_norm": 4.3184359855094945, + "kl": 0.2158203125, + "learning_rate": 5.658914728682171e-07, + "loss": 0.1619, + "reward": 0.04520327411592007, + "reward_std": 0.0040213235188275576, + "rewards/accuracy_reward": 0.04520327411592007, + "step": 73 + }, + { + "clip_ratio": 0.0, + "completion_length": 43.50000190734863, + "epoch": 0.05769980506822612, + "grad_norm": 3.1250028835345613, + "kl": 0.20654296875, + "learning_rate": 5.736434108527132e-07, + "loss": 0.0279, + "reward": 0.06468020007014275, + "reward_std": 0.014381224755197763, + "rewards/accuracy_reward": 0.06468020007014275, + "step": 74 + }, + { + "clip_ratio": 0.0, + "completion_length": 34.687500953674316, + "epoch": 0.05847953216374269, + "grad_norm": 3.779421591825818, + "kl": 0.208984375, + "learning_rate": 5.813953488372093e-07, + "loss": 0.2085, + "reward": 0.04720890522003174, + "reward_std": 0.005046627949923277, + "rewards/accuracy_reward": 0.04720890522003174, + "step": 75 + }, + { + "clip_ratio": 0.0, + "completion_length": 32.45833396911621, + "epoch": 0.05925925925925926, + "grad_norm": 3.967730635271566, + "kl": 0.21044921875, + "learning_rate": 5.891472868217054e-07, + "loss": 0.0697, + "reward": 0.07102777436375618, + "reward_std": 0.010164339561015368, + "rewards/accuracy_reward": 0.07102777436375618, + "step": 76 + }, + { + "clip_ratio": 0.0, + "completion_length": 47.458335876464844, + "epoch": 0.06003898635477583, + "grad_norm": 3.80599765313446, + "kl": 0.20361328125, + "learning_rate": 5.968992248062015e-07, + "loss": 0.2024, + "reward": 0.04373868927359581, + "reward_std": 0.0035682034213095903, + "rewards/accuracy_reward": 0.04373868927359581, + "step": 77 + }, + { + "clip_ratio": 0.0, + "completion_length": 36.95833396911621, + "epoch": 0.0608187134502924, + "grad_norm": 4.269746663131536, + "kl": 0.18994140625, + "learning_rate": 6.046511627906976e-07, + "loss": 0.1813, + "reward": 0.04619111493229866, + "reward_std": 0.004070800496265292, + "rewards/accuracy_reward": 0.04619111493229866, + "step": 78 + }, + { + "clip_ratio": 0.0, + "completion_length": 27.33333396911621, + "epoch": 0.061598440545808965, + "grad_norm": 4.4134349427597765, + "kl": 0.218994140625, + "learning_rate": 6.124031007751937e-07, + "loss": 0.1671, + "reward": 0.04927775636315346, + "reward_std": 0.0030711302533745766, + "rewards/accuracy_reward": 0.04927775636315346, + "step": 79 + }, + { + "clip_ratio": 0.0, + "completion_length": 42.35416793823242, + "epoch": 0.06237816764132553, + "grad_norm": 4.268446019202903, + "kl": 0.1572265625, + "learning_rate": 6.201550387596898e-07, + "loss": 0.2639, + "reward": 0.043139971792697906, + "reward_std": 0.005017469171434641, + "rewards/accuracy_reward": 0.043139971792697906, + "step": 80 + }, + { + "clip_ratio": 0.0, + "completion_length": 49.14583396911621, + "epoch": 0.06315789473684211, + "grad_norm": 2.4508441348623284, + "kl": 0.1826171875, + "learning_rate": 6.27906976744186e-07, + "loss": 0.0904, + "reward": 0.04530907794833183, + "reward_std": 0.0029870180878788233, + "rewards/accuracy_reward": 0.04530907794833183, + "step": 81 + }, + { + "clip_ratio": 0.0, + "completion_length": 39.875, + "epoch": 0.06393762183235867, + "grad_norm": 4.338489355975129, + "kl": 0.30126953125, + "learning_rate": 6.356589147286822e-07, + "loss": 0.0967, + "reward": 0.06821105815470219, + "reward_std": 0.008517026668414474, + "rewards/accuracy_reward": 0.06821105815470219, + "step": 82 + }, + { + "clip_ratio": 0.0, + "completion_length": 43.04166793823242, + "epoch": 0.06471734892787524, + "grad_norm": 2.908327544492862, + "kl": 0.1826171875, + "learning_rate": 6.434108527131783e-07, + "loss": 0.047, + "reward": 0.04304710030555725, + "reward_std": 0.00637028506025672, + "rewards/accuracy_reward": 0.04304710030555725, + "step": 83 + }, + { + "clip_ratio": 0.0, + "completion_length": 37.3125, + "epoch": 0.06549707602339182, + "grad_norm": 2.3979524790932225, + "kl": 0.106201171875, + "learning_rate": 6.511627906976745e-07, + "loss": 0.0158, + "reward": 0.054630450904369354, + "reward_std": 0.0019857838633470237, + "rewards/accuracy_reward": 0.054630450904369354, + "step": 84 + }, + { + "clip_ratio": 0.0, + "completion_length": 35.8125, + "epoch": 0.06627680311890838, + "grad_norm": 2.7520842907479537, + "kl": 0.18408203125, + "learning_rate": 6.589147286821705e-07, + "loss": 0.1055, + "reward": 0.06916381046175957, + "reward_std": 0.005404126946814358, + "rewards/accuracy_reward": 0.06916381046175957, + "step": 85 + }, + { + "clip_ratio": 0.0, + "completion_length": 46.47916793823242, + "epoch": 0.06705653021442495, + "grad_norm": 3.0163826480352762, + "kl": 0.2412109375, + "learning_rate": 6.666666666666666e-07, + "loss": 0.0081, + "reward": 0.04147185757756233, + "reward_std": 0.0031410371884703636, + "rewards/accuracy_reward": 0.04147185757756233, + "step": 86 + }, + { + "clip_ratio": 0.0, + "completion_length": 33.89583396911621, + "epoch": 0.06783625730994151, + "grad_norm": 3.040312568921289, + "kl": 0.1787109375, + "learning_rate": 6.744186046511627e-07, + "loss": 0.1352, + "reward": 0.04949432238936424, + "reward_std": 0.004746574675664306, + "rewards/accuracy_reward": 0.04949432238936424, + "step": 87 + }, + { + "clip_ratio": 0.0, + "completion_length": 32.47916793823242, + "epoch": 0.06861598440545809, + "grad_norm": 4.513511687058106, + "kl": 0.296875, + "learning_rate": 6.821705426356589e-07, + "loss": 0.0967, + "reward": 0.047069137915968895, + "reward_std": 0.00340858765412122, + "rewards/accuracy_reward": 0.047069137915968895, + "step": 88 + }, + { + "clip_ratio": 0.0, + "completion_length": 36.70833396911621, + "epoch": 0.06939571150097466, + "grad_norm": 3.2046936061466686, + "kl": 0.185546875, + "learning_rate": 6.89922480620155e-07, + "loss": 0.1379, + "reward": 0.045855697244405746, + "reward_std": 0.002462368458509445, + "rewards/accuracy_reward": 0.045855697244405746, + "step": 89 + }, + { + "clip_ratio": 0.0, + "completion_length": 42.83333396911621, + "epoch": 0.07017543859649122, + "grad_norm": 3.4631744406118687, + "kl": 0.28125, + "learning_rate": 6.976744186046511e-07, + "loss": 0.1755, + "reward": 0.07655094563961029, + "reward_std": 0.01264206226915121, + "rewards/accuracy_reward": 0.07655094563961029, + "step": 90 + }, + { + "clip_ratio": 0.0, + "completion_length": 35.10416793823242, + "epoch": 0.0709551656920078, + "grad_norm": 2.8068188607333937, + "kl": 0.30419921875, + "learning_rate": 7.054263565891473e-07, + "loss": 0.0932, + "reward": 0.05006732605397701, + "reward_std": 0.005144392838701606, + "rewards/accuracy_reward": 0.05006732605397701, + "step": 91 + }, + { + "clip_ratio": 0.0, + "completion_length": 31.416667938232422, + "epoch": 0.07173489278752436, + "grad_norm": 3.6618368049219336, + "kl": 0.43359375, + "learning_rate": 7.131782945736435e-07, + "loss": 0.0811, + "reward": 0.047380127012729645, + "reward_std": 0.005735602229833603, + "rewards/accuracy_reward": 0.047380127012729645, + "step": 92 + }, + { + "clip_ratio": 0.0, + "completion_length": 36.72916793823242, + "epoch": 0.07251461988304093, + "grad_norm": 2.808379222783262, + "kl": 0.165771484375, + "learning_rate": 7.209302325581395e-07, + "loss": 0.0626, + "reward": 0.040977967903018, + "reward_std": 0.0020597196416929364, + "rewards/accuracy_reward": 0.040977967903018, + "step": 93 + }, + { + "clip_ratio": 0.0, + "completion_length": 35.20833396911621, + "epoch": 0.07329434697855751, + "grad_norm": 2.641084827728231, + "kl": 0.2216796875, + "learning_rate": 7.286821705426356e-07, + "loss": -0.0525, + "reward": 0.06772063486278057, + "reward_std": 0.00525827007368207, + "rewards/accuracy_reward": 0.06772063486278057, + "step": 94 + }, + { + "clip_ratio": 0.0, + "completion_length": 34.479166984558105, + "epoch": 0.07407407407407407, + "grad_norm": 2.3420543141817687, + "kl": 0.3154296875, + "learning_rate": 7.364341085271317e-07, + "loss": 0.033, + "reward": 0.047532547265291214, + "reward_std": 0.0015460846479982138, + "rewards/accuracy_reward": 0.047532547265291214, + "step": 95 + }, + { + "clip_ratio": 0.0, + "completion_length": 37.25, + "epoch": 0.07485380116959064, + "grad_norm": 3.8272598250695147, + "kl": 0.3349609375, + "learning_rate": 7.441860465116279e-07, + "loss": 0.0745, + "reward": 0.0494490060955286, + "reward_std": 0.006539649562910199, + "rewards/accuracy_reward": 0.0494490060955286, + "step": 96 + }, + { + "clip_ratio": 0.0, + "completion_length": 37.66666793823242, + "epoch": 0.07563352826510722, + "grad_norm": 2.8283134363372766, + "kl": 0.2939453125, + "learning_rate": 7.51937984496124e-07, + "loss": 0.0619, + "reward": 0.06136970967054367, + "reward_std": 0.005943382391706109, + "rewards/accuracy_reward": 0.06136970967054367, + "step": 97 + }, + { + "clip_ratio": 0.0, + "completion_length": 38.95833396911621, + "epoch": 0.07641325536062378, + "grad_norm": 4.0926550610955825, + "kl": 0.26318359375, + "learning_rate": 7.596899224806201e-07, + "loss": 0.0133, + "reward": 0.05305527709424496, + "reward_std": 0.010987138841301203, + "rewards/accuracy_reward": 0.05305527709424496, + "step": 98 + }, + { + "clip_ratio": 0.0, + "completion_length": 27.08333396911621, + "epoch": 0.07719298245614035, + "grad_norm": 2.227907651484132, + "kl": 0.20556640625, + "learning_rate": 7.674418604651162e-07, + "loss": -0.0106, + "reward": 0.06309524551033974, + "reward_std": 0.0008820741786621511, + "rewards/accuracy_reward": 0.06309524551033974, + "step": 99 + }, + { + "clip_ratio": 0.0, + "completion_length": 37.35416793823242, + "epoch": 0.07797270955165692, + "grad_norm": 3.4502302886910363, + "kl": 0.26708984375, + "learning_rate": 7.751937984496125e-07, + "loss": 0.0379, + "reward": 0.04482625797390938, + "reward_std": 0.0022178655490279198, + "rewards/accuracy_reward": 0.04482625797390938, + "step": 100 + }, + { + "clip_ratio": 0.0, + "completion_length": 42.4375, + "epoch": 0.07875243664717349, + "grad_norm": 3.0138541752807932, + "kl": 0.25830078125, + "learning_rate": 7.829457364341085e-07, + "loss": 0.0461, + "reward": 0.04530594311654568, + "reward_std": 0.0014333150465972722, + "rewards/accuracy_reward": 0.04530594311654568, + "step": 101 + }, + { + "clip_ratio": 0.0, + "completion_length": 43.08333396911621, + "epoch": 0.07953216374269007, + "grad_norm": 2.9524828938696444, + "kl": 0.244140625, + "learning_rate": 7.906976744186046e-07, + "loss": 0.07, + "reward": 0.0649220421910286, + "reward_std": 0.01101213088259101, + "rewards/accuracy_reward": 0.0649220421910286, + "step": 102 + }, + { + "clip_ratio": 0.0, + "completion_length": 47.50000190734863, + "epoch": 0.08031189083820663, + "grad_norm": 2.6451002225368594, + "kl": 0.19970703125, + "learning_rate": 7.984496124031007e-07, + "loss": 0.1396, + "reward": 0.05022517405450344, + "reward_std": 0.002265424351207912, + "rewards/accuracy_reward": 0.05022517405450344, + "step": 103 + }, + { + "clip_ratio": 0.0, + "completion_length": 46.10416793823242, + "epoch": 0.0810916179337232, + "grad_norm": 2.1928991316645, + "kl": 0.26171875, + "learning_rate": 8.062015503875969e-07, + "loss": 0.0561, + "reward": 0.045890724286437035, + "reward_std": 0.006344977300614119, + "rewards/accuracy_reward": 0.045890724286437035, + "step": 104 + }, + { + "clip_ratio": 0.0, + "completion_length": 59.72916793823242, + "epoch": 0.08187134502923976, + "grad_norm": 2.0426574583498014, + "kl": 0.17578125, + "learning_rate": 8.13953488372093e-07, + "loss": 0.0054, + "reward": 0.06249103508889675, + "reward_std": 0.0027366310823708773, + "rewards/accuracy_reward": 0.06249103508889675, + "step": 105 + }, + { + "clip_ratio": 0.0, + "completion_length": 37.37500190734863, + "epoch": 0.08265107212475634, + "grad_norm": 3.2710869824871627, + "kl": 0.419921875, + "learning_rate": 8.217054263565891e-07, + "loss": 0.0478, + "reward": 0.09790483117103577, + "reward_std": 0.00470519601367414, + "rewards/accuracy_reward": 0.09790483117103577, + "step": 106 + }, + { + "clip_ratio": 0.0, + "completion_length": 45.10416793823242, + "epoch": 0.08343079922027291, + "grad_norm": 1.9761863371392694, + "kl": 0.2353515625, + "learning_rate": 8.294573643410852e-07, + "loss": 0.0536, + "reward": 0.04955454729497433, + "reward_std": 0.0018941238522529602, + "rewards/accuracy_reward": 0.04955454729497433, + "step": 107 + }, + { + "clip_ratio": 0.0, + "completion_length": 48.54166793823242, + "epoch": 0.08421052631578947, + "grad_norm": 2.2771383882088854, + "kl": 0.3203125, + "learning_rate": 8.372093023255814e-07, + "loss": 0.0618, + "reward": 0.06308815255761147, + "reward_std": 0.007483313325792551, + "rewards/accuracy_reward": 0.06308815255761147, + "step": 108 + }, + { + "clip_ratio": 0.0, + "completion_length": 38.72916793823242, + "epoch": 0.08499025341130605, + "grad_norm": 2.2159968094278586, + "kl": 0.23046875, + "learning_rate": 8.449612403100774e-07, + "loss": 0.0829, + "reward": 0.05840716511011124, + "reward_std": 0.004084525164216757, + "rewards/accuracy_reward": 0.05840716511011124, + "step": 109 + }, + { + "clip_ratio": 0.0, + "completion_length": 38.895835876464844, + "epoch": 0.08576998050682261, + "grad_norm": 3.590058268549366, + "kl": 0.4697265625, + "learning_rate": 8.527131782945736e-07, + "loss": 0.0697, + "reward": 0.062450679019093513, + "reward_std": 0.007699631154537201, + "rewards/accuracy_reward": 0.062450679019093513, + "step": 110 + }, + { + "clip_ratio": 0.0, + "completion_length": 37.60416793823242, + "epoch": 0.08654970760233918, + "grad_norm": 2.970550745807065, + "kl": 0.3603515625, + "learning_rate": 8.604651162790697e-07, + "loss": 0.1128, + "reward": 0.07873068004846573, + "reward_std": 0.0034102650824934244, + "rewards/accuracy_reward": 0.07873068004846573, + "step": 111 + }, + { + "clip_ratio": 0.0, + "completion_length": 37.33333396911621, + "epoch": 0.08732943469785576, + "grad_norm": 2.159898735456206, + "kl": 0.279296875, + "learning_rate": 8.682170542635659e-07, + "loss": 0.0268, + "reward": 0.08472547307610512, + "reward_std": 0.004890290321782231, + "rewards/accuracy_reward": 0.08472547307610512, + "step": 112 + }, + { + "clip_ratio": 0.0, + "completion_length": 41.93750190734863, + "epoch": 0.08810916179337232, + "grad_norm": 2.4048395200356865, + "kl": 0.373046875, + "learning_rate": 8.75968992248062e-07, + "loss": 0.0822, + "reward": 0.044123949483036995, + "reward_std": 0.002274714468512684, + "rewards/accuracy_reward": 0.044123949483036995, + "step": 113 + }, + { + "clip_ratio": 0.0, + "completion_length": 44.79166793823242, + "epoch": 0.08888888888888889, + "grad_norm": 2.110076357861851, + "kl": 0.349609375, + "learning_rate": 8.837209302325581e-07, + "loss": 0.0353, + "reward": 0.05389041267335415, + "reward_std": 0.004451837099622935, + "rewards/accuracy_reward": 0.05389041267335415, + "step": 114 + }, + { + "clip_ratio": 0.0, + "completion_length": 44.375000953674316, + "epoch": 0.08966861598440545, + "grad_norm": 2.388794050449426, + "kl": 0.32470703125, + "learning_rate": 8.914728682170542e-07, + "loss": 0.0636, + "reward": 0.05829278379678726, + "reward_std": 0.0021095441188663244, + "rewards/accuracy_reward": 0.05829278379678726, + "step": 115 + }, + { + "clip_ratio": 0.0, + "completion_length": 32.08333396911621, + "epoch": 0.09044834307992203, + "grad_norm": 2.2972470263546634, + "kl": 0.23583984375, + "learning_rate": 8.992248062015504e-07, + "loss": 0.0318, + "reward": 0.05514957569539547, + "reward_std": 0.0009780407708603889, + "rewards/accuracy_reward": 0.05514957569539547, + "step": 116 + }, + { + "clip_ratio": 0.0, + "completion_length": 42.43750190734863, + "epoch": 0.0912280701754386, + "grad_norm": 2.9001639996696222, + "kl": 0.2783203125, + "learning_rate": 9.069767441860464e-07, + "loss": 0.2305, + "reward": 0.06270255520939827, + "reward_std": 0.004146690247580409, + "rewards/accuracy_reward": 0.06270255520939827, + "step": 117 + }, + { + "clip_ratio": 0.0, + "completion_length": 39.14583396911621, + "epoch": 0.09200779727095516, + "grad_norm": 2.0766943646013654, + "kl": 0.3134765625, + "learning_rate": 9.147286821705425e-07, + "loss": 0.0964, + "reward": 0.07618878036737442, + "reward_std": 0.002100339625030756, + "rewards/accuracy_reward": 0.07618878036737442, + "step": 118 + }, + { + "clip_ratio": 0.0, + "completion_length": 39.29166793823242, + "epoch": 0.09278752436647174, + "grad_norm": 2.4811588444022363, + "kl": 0.443359375, + "learning_rate": 9.224806201550386e-07, + "loss": 0.0313, + "reward": 0.09027307853102684, + "reward_std": 0.006509718834422529, + "rewards/accuracy_reward": 0.09027307853102684, + "step": 119 + }, + { + "clip_ratio": 0.0, + "completion_length": 36.9375, + "epoch": 0.0935672514619883, + "grad_norm": 1.594512917972913, + "kl": 0.251953125, + "learning_rate": 9.302325581395349e-07, + "loss": 0.0489, + "reward": 0.07478075847029686, + "reward_std": 0.0012831513013225049, + "rewards/accuracy_reward": 0.07478075847029686, + "step": 120 + }, + { + "clip_ratio": 0.0, + "completion_length": 46.93750190734863, + "epoch": 0.09434697855750487, + "grad_norm": 2.360172429007107, + "kl": 0.4814453125, + "learning_rate": 9.37984496124031e-07, + "loss": 0.0438, + "reward": 0.052850114181637764, + "reward_std": 0.005657479981891811, + "rewards/accuracy_reward": 0.052850114181637764, + "step": 121 + }, + { + "clip_ratio": 0.0, + "completion_length": 44.66666793823242, + "epoch": 0.09512670565302145, + "grad_norm": 1.7782689608680646, + "kl": 0.29638671875, + "learning_rate": 9.457364341085271e-07, + "loss": 0.0316, + "reward": 0.04962804540991783, + "reward_std": 0.0010844895150512457, + "rewards/accuracy_reward": 0.04962804540991783, + "step": 122 + }, + { + "clip_ratio": 0.0, + "completion_length": 41.81250190734863, + "epoch": 0.09590643274853801, + "grad_norm": 2.064930953634142, + "kl": 0.279296875, + "learning_rate": 9.534883720930232e-07, + "loss": 0.0474, + "reward": 0.06100150756537914, + "reward_std": 0.00565731490496546, + "rewards/accuracy_reward": 0.06100150756537914, + "step": 123 + }, + { + "clip_ratio": 0.0, + "completion_length": 44.60416793823242, + "epoch": 0.09668615984405458, + "grad_norm": 2.7146972146100303, + "kl": 0.3837890625, + "learning_rate": 9.612403100775193e-07, + "loss": 0.2327, + "reward": 0.04944268800318241, + "reward_std": 0.004167388891801238, + "rewards/accuracy_reward": 0.04944268800318241, + "step": 124 + }, + { + "clip_ratio": 0.0, + "completion_length": 45.43750190734863, + "epoch": 0.09746588693957114, + "grad_norm": 1.315801786583085, + "kl": 0.2353515625, + "learning_rate": 9.689922480620153e-07, + "loss": -0.0062, + "reward": 0.06027457118034363, + "reward_std": 0.0011762951035052538, + "rewards/accuracy_reward": 0.06027457118034363, + "step": 125 + }, + { + "clip_ratio": 0.0, + "completion_length": 46.31250190734863, + "epoch": 0.09824561403508772, + "grad_norm": 1.7782273137093043, + "kl": 0.2802734375, + "learning_rate": 9.767441860465115e-07, + "loss": 0.0041, + "reward": 0.05181322619318962, + "reward_std": 0.004530643695034087, + "rewards/accuracy_reward": 0.05181322619318962, + "step": 126 + }, + { + "clip_ratio": 0.0, + "completion_length": 38.64583396911621, + "epoch": 0.0990253411306043, + "grad_norm": 1.7871033542884303, + "kl": 0.25048828125, + "learning_rate": 9.844961240310077e-07, + "loss": 0.0708, + "reward": 0.05114087648689747, + "reward_std": 0.0008208061335608363, + "rewards/accuracy_reward": 0.05114087648689747, + "step": 127 + }, + { + "clip_ratio": 0.0, + "completion_length": 37.333335876464844, + "epoch": 0.09980506822612085, + "grad_norm": 2.0417982330649433, + "kl": 0.314453125, + "learning_rate": 9.922480620155037e-07, + "loss": 0.0212, + "reward": 0.06282653287053108, + "reward_std": 0.0034878100268542767, + "rewards/accuracy_reward": 0.06282653287053108, + "step": 128 + }, + { + "clip_ratio": 0.0, + "completion_length": 39.27083492279053, + "epoch": 0.10058479532163743, + "grad_norm": 1.58961800509684, + "kl": 0.3310546875, + "learning_rate": 1e-06, + "loss": 0.0221, + "reward": 0.06363019719719887, + "reward_std": 0.003304417827166617, + "rewards/accuracy_reward": 0.06363019719719887, + "step": 129 + }, + { + "clip_ratio": 0.0, + "completion_length": 34.58333396911621, + "epoch": 0.10136452241715399, + "grad_norm": 1.0712022227545728, + "kl": 0.28125, + "learning_rate": 9.9999832958885e-07, + "loss": 0.0152, + "reward": 0.0661378987133503, + "reward_std": 0.0005932368221692741, + "rewards/accuracy_reward": 0.0661378987133503, + "step": 130 + }, + { + "clip_ratio": 0.0, + "completion_length": 50.77083396911621, + "epoch": 0.10214424951267057, + "grad_norm": 3.841618839193188, + "kl": 0.490234375, + "learning_rate": 9.999933183678015e-07, + "loss": 0.0975, + "reward": 0.09110459685325623, + "reward_std": 0.004217098816297948, + "rewards/accuracy_reward": 0.09110459685325623, + "step": 131 + }, + { + "clip_ratio": 0.0, + "completion_length": 44.08333396911621, + "epoch": 0.10292397660818714, + "grad_norm": 3.4945102917866286, + "kl": 0.4404296875, + "learning_rate": 9.99984966374058e-07, + "loss": 0.0711, + "reward": 0.05980168469250202, + "reward_std": 0.004065339686349034, + "rewards/accuracy_reward": 0.05980168469250202, + "step": 132 + }, + { + "clip_ratio": 0.0, + "completion_length": 39.750000953674316, + "epoch": 0.1037037037037037, + "grad_norm": 1.0296024150123595, + "kl": 0.3134765625, + "learning_rate": 9.99973273669625e-07, + "loss": 0.0378, + "reward": 0.0822172649204731, + "reward_std": 0.0013668336905539036, + "rewards/accuracy_reward": 0.0822172649204731, + "step": 133 + }, + { + "clip_ratio": 0.0, + "completion_length": 42.60416793823242, + "epoch": 0.10448343079922028, + "grad_norm": 1.5599395926534922, + "kl": 0.306640625, + "learning_rate": 9.9995824034131e-07, + "loss": 0.0433, + "reward": 0.07356821000576019, + "reward_std": 0.005668101832270622, + "rewards/accuracy_reward": 0.07356821000576019, + "step": 134 + }, + { + "clip_ratio": 0.0, + "completion_length": 37.6875, + "epoch": 0.10526315789473684, + "grad_norm": 1.4729301419390965, + "kl": 0.3740234375, + "learning_rate": 9.999398665007205e-07, + "loss": 0.0572, + "reward": 0.04963262751698494, + "reward_std": 0.0009213387966156006, + "rewards/accuracy_reward": 0.04963262751698494, + "step": 135 + }, + { + "clip_ratio": 0.0, + "completion_length": 45.43750190734863, + "epoch": 0.10604288499025341, + "grad_norm": 2.154918468516356, + "kl": 0.4912109375, + "learning_rate": 9.999181522842657e-07, + "loss": 0.0624, + "reward": 0.06199992448091507, + "reward_std": 0.004981459584087133, + "rewards/accuracy_reward": 0.06199992448091507, + "step": 136 + }, + { + "clip_ratio": 0.0, + "completion_length": 44.6875, + "epoch": 0.10682261208576999, + "grad_norm": 1.6178102316186667, + "kl": 0.2998046875, + "learning_rate": 9.998930978531523e-07, + "loss": -0.0059, + "reward": 0.07799461856484413, + "reward_std": 0.0033870727056637406, + "rewards/accuracy_reward": 0.07799461856484413, + "step": 137 + }, + { + "clip_ratio": 0.0, + "completion_length": 47.95833396911621, + "epoch": 0.10760233918128655, + "grad_norm": 2.011618952941971, + "kl": 0.22314453125, + "learning_rate": 9.998647033933862e-07, + "loss": 0.1619, + "reward": 0.07276952639222145, + "reward_std": 0.0057456461363472044, + "rewards/accuracy_reward": 0.07276952639222145, + "step": 138 + }, + { + "clip_ratio": 0.0, + "completion_length": 40.93750190734863, + "epoch": 0.10838206627680312, + "grad_norm": 1.1953395051316285, + "kl": 0.3056640625, + "learning_rate": 9.998329691157686e-07, + "loss": 0.0179, + "reward": 0.0824020504951477, + "reward_std": 0.0007267132750712335, + "rewards/accuracy_reward": 0.0824020504951477, + "step": 139 + }, + { + "clip_ratio": 0.0, + "completion_length": 45.77083396911621, + "epoch": 0.10916179337231968, + "grad_norm": 1.4863258665541164, + "kl": 0.26513671875, + "learning_rate": 9.997978952558972e-07, + "loss": 0.0707, + "reward": 0.06538177467882633, + "reward_std": 0.0016729276394471526, + "rewards/accuracy_reward": 0.06538177467882633, + "step": 140 + }, + { + "clip_ratio": 0.0, + "completion_length": 43.93750190734863, + "epoch": 0.10994152046783626, + "grad_norm": 1.2974485192868461, + "kl": 0.31298828125, + "learning_rate": 9.997594820741615e-07, + "loss": 0.045, + "reward": 0.0717059038579464, + "reward_std": 0.005708017386496067, + "rewards/accuracy_reward": 0.0717059038579464, + "step": 141 + }, + { + "clip_ratio": 0.0, + "completion_length": 39.47916793823242, + "epoch": 0.11072124756335283, + "grad_norm": 1.738706218029015, + "kl": 0.34765625, + "learning_rate": 9.997177298557428e-07, + "loss": 0.0916, + "reward": 0.06964311562478542, + "reward_std": 0.0017773142317309976, + "rewards/accuracy_reward": 0.06964311562478542, + "step": 142 + }, + { + "clip_ratio": 0.0, + "completion_length": 47.70833396911621, + "epoch": 0.11150097465886939, + "grad_norm": 2.617996250225219, + "kl": 0.4169921875, + "learning_rate": 9.996726389106119e-07, + "loss": 0.0562, + "reward": 0.05835900455713272, + "reward_std": 0.0016873643035069108, + "rewards/accuracy_reward": 0.05835900455713272, + "step": 143 + }, + { + "clip_ratio": 0.0, + "completion_length": 47.33333396911621, + "epoch": 0.11228070175438597, + "grad_norm": 2.099286006908965, + "kl": 0.431640625, + "learning_rate": 9.99624209573526e-07, + "loss": 0.0435, + "reward": 0.056647004559636116, + "reward_std": 0.005114392610266805, + "rewards/accuracy_reward": 0.056647004559636116, + "step": 144 + }, + { + "clip_ratio": 0.0, + "completion_length": 43.8125, + "epoch": 0.11306042884990253, + "grad_norm": 1.2752446361191032, + "kl": 0.3056640625, + "learning_rate": 9.995724422040273e-07, + "loss": 0.0446, + "reward": 0.04826555773615837, + "reward_std": 0.0007271187496371567, + "rewards/accuracy_reward": 0.04826555773615837, + "step": 145 + }, + { + "clip_ratio": 0.0, + "completion_length": 34.91666793823242, + "epoch": 0.1138401559454191, + "grad_norm": 0.8139602608611531, + "kl": 0.38671875, + "learning_rate": 9.99517337186439e-07, + "loss": 0.0212, + "reward": 0.06071879714727402, + "reward_std": 0.0009029456123244017, + "rewards/accuracy_reward": 0.06071879714727402, + "step": 146 + }, + { + "clip_ratio": 0.0, + "completion_length": 39.25000190734863, + "epoch": 0.11461988304093568, + "grad_norm": 2.3088936352215654, + "kl": 0.5126953125, + "learning_rate": 9.994588949298635e-07, + "loss": -0.0505, + "reward": 0.05469323694705963, + "reward_std": 0.00674125796649605, + "rewards/accuracy_reward": 0.05469323694705963, + "step": 147 + }, + { + "clip_ratio": 0.0, + "completion_length": 42.20833396911621, + "epoch": 0.11539961013645224, + "grad_norm": 2.152011093405869, + "kl": 0.3154296875, + "learning_rate": 9.993971158681788e-07, + "loss": 0.0024, + "reward": 0.0823461264371872, + "reward_std": 0.0011642208555713296, + "rewards/accuracy_reward": 0.0823461264371872, + "step": 148 + }, + { + "clip_ratio": 0.0, + "completion_length": 41.50000190734863, + "epoch": 0.11617933723196881, + "grad_norm": 1.5947713011820153, + "kl": 0.396484375, + "learning_rate": 9.993320004600362e-07, + "loss": 0.0808, + "reward": 0.06809847429394722, + "reward_std": 0.0017332308925688267, + "rewards/accuracy_reward": 0.06809847429394722, + "step": 149 + }, + { + "clip_ratio": 0.0, + "completion_length": 32.22916793823242, + "epoch": 0.11695906432748537, + "grad_norm": 1.432565738285189, + "kl": 0.255859375, + "learning_rate": 9.992635491888554e-07, + "loss": 0.0147, + "reward": 0.05747877433896065, + "reward_std": 0.00021101986931171268, + "rewards/accuracy_reward": 0.05747877433896065, + "step": 150 + }, + { + "clip_ratio": 0.0, + "completion_length": 42.29166793823242, + "epoch": 0.11773879142300195, + "grad_norm": 1.5598261656388719, + "kl": 0.3251953125, + "learning_rate": 9.991917625628223e-07, + "loss": 0.0533, + "reward": 0.05868055857717991, + "reward_std": 0.007278923876583576, + "rewards/accuracy_reward": 0.05868055857717991, + "step": 151 + }, + { + "clip_ratio": 0.0, + "completion_length": 31.625000953674316, + "epoch": 0.11851851851851852, + "grad_norm": 3.0936394994220238, + "kl": 0.27294921875, + "learning_rate": 9.99116641114884e-07, + "loss": 0.0408, + "reward": 0.07421398535370827, + "reward_std": 0.01793223712593317, + "rewards/accuracy_reward": 0.07421398535370827, + "step": 152 + }, + { + "clip_ratio": 0.0, + "completion_length": 36.875, + "epoch": 0.11929824561403508, + "grad_norm": 3.6524430060918296, + "kl": 0.458984375, + "learning_rate": 9.99038185402746e-07, + "loss": 0.0116, + "reward": 0.08958383277058601, + "reward_std": 0.01494149150676094, + "rewards/accuracy_reward": 0.08958383277058601, + "step": 153 + }, + { + "clip_ratio": 0.0, + "completion_length": 40.27083396911621, + "epoch": 0.12007797270955166, + "grad_norm": 2.2393578921718267, + "kl": 0.38671875, + "learning_rate": 9.989563960088675e-07, + "loss": -0.0241, + "reward": 0.08549312502145767, + "reward_std": 0.009909003507345915, + "rewards/accuracy_reward": 0.08549312502145767, + "step": 154 + }, + { + "clip_ratio": 0.0, + "completion_length": 41.14583396911621, + "epoch": 0.12085769980506822, + "grad_norm": 2.2804733788183547, + "kl": 0.3916015625, + "learning_rate": 9.98871273540457e-07, + "loss": 0.1656, + "reward": 0.04912008345127106, + "reward_std": 0.0015631647547706962, + "rewards/accuracy_reward": 0.04912008345127106, + "step": 155 + }, + { + "clip_ratio": 0.0, + "completion_length": 43.33333396911621, + "epoch": 0.1216374269005848, + "grad_norm": 2.014573223378803, + "kl": 0.333984375, + "learning_rate": 9.987828186294677e-07, + "loss": -0.0351, + "reward": 0.10474389418959618, + "reward_std": 0.0107078772271052, + "rewards/accuracy_reward": 0.10474389418959618, + "step": 156 + }, + { + "clip_ratio": 0.0, + "completion_length": 36.916666984558105, + "epoch": 0.12241715399610137, + "grad_norm": 1.327646280560192, + "kl": 0.3857421875, + "learning_rate": 9.986910319325935e-07, + "loss": 0.0309, + "reward": 0.07857292518019676, + "reward_std": 0.0010515710746403784, + "rewards/accuracy_reward": 0.07857292518019676, + "step": 157 + }, + { + "clip_ratio": 0.0, + "completion_length": 38.22916793823242, + "epoch": 0.12319688109161793, + "grad_norm": 1.9698099567550267, + "kl": 0.416015625, + "learning_rate": 9.985959141312633e-07, + "loss": 0.0639, + "reward": 0.05107174627482891, + "reward_std": 0.0014609357458539307, + "rewards/accuracy_reward": 0.05107174627482891, + "step": 158 + }, + { + "clip_ratio": 0.0, + "completion_length": 37.58333396911621, + "epoch": 0.1239766081871345, + "grad_norm": 2.55280935272325, + "kl": 0.3740234375, + "learning_rate": 9.984974659316361e-07, + "loss": 0.0851, + "reward": 0.08227085322141647, + "reward_std": 0.0012928638607263565, + "rewards/accuracy_reward": 0.08227085322141647, + "step": 159 + }, + { + "clip_ratio": 0.0, + "completion_length": 32.479166984558105, + "epoch": 0.12475633528265107, + "grad_norm": 2.4000975504020046, + "kl": 0.4560546875, + "learning_rate": 9.983956880645967e-07, + "loss": 0.0671, + "reward": 0.07537728920578957, + "reward_std": 0.0013081615325063467, + "rewards/accuracy_reward": 0.07537728920578957, + "step": 160 + }, + { + "clip_ratio": 0.0, + "completion_length": 35.10416793823242, + "epoch": 0.12553606237816764, + "grad_norm": 1.7086066571358172, + "kl": 0.341796875, + "learning_rate": 9.982905812857488e-07, + "loss": 0.0484, + "reward": 0.05994926765561104, + "reward_std": 0.0005158578278496861, + "rewards/accuracy_reward": 0.05994926765561104, + "step": 161 + }, + { + "clip_ratio": 0.0, + "completion_length": 33.0625, + "epoch": 0.12631578947368421, + "grad_norm": 2.7607994144458665, + "kl": 0.474609375, + "learning_rate": 9.981821463754101e-07, + "loss": 0.1264, + "reward": 0.07282404229044914, + "reward_std": 0.002143740246538073, + "rewards/accuracy_reward": 0.07282404229044914, + "step": 162 + }, + { + "clip_ratio": 0.0, + "completion_length": 37.83333396911621, + "epoch": 0.1270955165692008, + "grad_norm": 2.503567849299546, + "kl": 0.3544921875, + "learning_rate": 9.980703841386074e-07, + "loss": 0.005, + "reward": 0.0794006958603859, + "reward_std": 0.0015508798533119261, + "rewards/accuracy_reward": 0.0794006958603859, + "step": 163 + }, + { + "clip_ratio": 0.0, + "completion_length": 42.91666793823242, + "epoch": 0.12787524366471734, + "grad_norm": 2.90340702021995, + "kl": 0.4052734375, + "learning_rate": 9.979552954050685e-07, + "loss": 0.0641, + "reward": 0.0734306201338768, + "reward_std": 0.004304809612222016, + "rewards/accuracy_reward": 0.0734306201338768, + "step": 164 + }, + { + "clip_ratio": 0.0, + "completion_length": 38.83333396911621, + "epoch": 0.1286549707602339, + "grad_norm": 0.7965501666473153, + "kl": 0.4326171875, + "learning_rate": 9.978368810292178e-07, + "loss": 0.0183, + "reward": 0.06607143208384514, + "reward_std": 0.0036375211784616113, + "rewards/accuracy_reward": 0.06607143208384514, + "step": 165 + }, + { + "clip_ratio": 0.0, + "completion_length": 37.70833396911621, + "epoch": 0.12943469785575049, + "grad_norm": 1.4183492878836457, + "kl": 0.55078125, + "learning_rate": 9.9771514189017e-07, + "loss": 0.027, + "reward": 0.08144724369049072, + "reward_std": 0.0009976768342312425, + "rewards/accuracy_reward": 0.08144724369049072, + "step": 166 + }, + { + "clip_ratio": 0.0, + "completion_length": 40.79166793823242, + "epoch": 0.13021442495126706, + "grad_norm": 2.6856419623277525, + "kl": 0.48046875, + "learning_rate": 9.97590078891722e-07, + "loss": 0.0681, + "reward": 0.08064528182148933, + "reward_std": 0.002155478810891509, + "rewards/accuracy_reward": 0.08064528182148933, + "step": 167 + }, + { + "clip_ratio": 0.0, + "completion_length": 35.812500953674316, + "epoch": 0.13099415204678364, + "grad_norm": 2.191222031223032, + "kl": 0.4013671875, + "learning_rate": 9.974616929623482e-07, + "loss": 0.075, + "reward": 0.07386208698153496, + "reward_std": 0.0022978404304012656, + "rewards/accuracy_reward": 0.07386208698153496, + "step": 168 + }, + { + "clip_ratio": 0.0, + "completion_length": 43.72916793823242, + "epoch": 0.13177387914230018, + "grad_norm": 1.9036553054266525, + "kl": 0.501953125, + "learning_rate": 9.973299850551916e-07, + "loss": 0.102, + "reward": 0.08643149957060814, + "reward_std": 0.003348127822391689, + "rewards/accuracy_reward": 0.08643149957060814, + "step": 169 + }, + { + "clip_ratio": 0.0, + "completion_length": 37.083335876464844, + "epoch": 0.13255360623781676, + "grad_norm": 4.344854173590328, + "kl": 0.80078125, + "learning_rate": 9.971949561480585e-07, + "loss": 0.0647, + "reward": 0.10409487038850784, + "reward_std": 0.0019580154912546277, + "rewards/accuracy_reward": 0.10409487038850784, + "step": 170 + }, + { + "clip_ratio": 0.0, + "completion_length": 38.64583396911621, + "epoch": 0.13333333333333333, + "grad_norm": 2.376886804733297, + "kl": 0.4404296875, + "learning_rate": 9.9705660724341e-07, + "loss": 0.0679, + "reward": 0.08254116401076317, + "reward_std": 0.001101005997043103, + "rewards/accuracy_reward": 0.08254116401076317, + "step": 171 + }, + { + "clip_ratio": 0.0, + "completion_length": 37.77083492279053, + "epoch": 0.1341130604288499, + "grad_norm": 1.2300007972044853, + "kl": 0.412109375, + "learning_rate": 9.969149393683554e-07, + "loss": 0.0539, + "reward": 0.06586156785488129, + "reward_std": 0.0014088151510804892, + "rewards/accuracy_reward": 0.06586156785488129, + "step": 172 + }, + { + "clip_ratio": 0.0, + "completion_length": 31.354166984558105, + "epoch": 0.13489278752436648, + "grad_norm": 2.8013904638274596, + "kl": 0.4775390625, + "learning_rate": 9.967699535746442e-07, + "loss": 0.1545, + "reward": 0.08266054093837738, + "reward_std": 0.0015777365770190954, + "rewards/accuracy_reward": 0.08266054093837738, + "step": 173 + }, + { + "clip_ratio": 0.0, + "completion_length": 25.77083396911621, + "epoch": 0.13567251461988303, + "grad_norm": 3.8423465417405556, + "kl": 0.703125, + "learning_rate": 9.966216509386579e-07, + "loss": 0.1449, + "reward": 0.07501306384801865, + "reward_std": 0.002388908062130213, + "rewards/accuracy_reward": 0.07501306384801865, + "step": 174 + }, + { + "clip_ratio": 0.0, + "completion_length": 32.22916793823242, + "epoch": 0.1364522417153996, + "grad_norm": 3.0980122015647042, + "kl": 0.4794921875, + "learning_rate": 9.964700325614025e-07, + "loss": 0.0415, + "reward": 0.07346947304904461, + "reward_std": 0.0008450358000118285, + "rewards/accuracy_reward": 0.07346947304904461, + "step": 175 + }, + { + "clip_ratio": 0.0, + "completion_length": 25.541667938232422, + "epoch": 0.13723196881091618, + "grad_norm": 1.4105538008175367, + "kl": 0.46484375, + "learning_rate": 9.963150995685005e-07, + "loss": 0.023, + "reward": 0.06290727108716965, + "reward_std": 0.00048778130440041423, + "rewards/accuracy_reward": 0.06290727108716965, + "step": 176 + }, + { + "clip_ratio": 0.0, + "completion_length": 31.375001907348633, + "epoch": 0.13801169590643275, + "grad_norm": 3.31868796728071, + "kl": 0.4736328125, + "learning_rate": 9.96156853110182e-07, + "loss": 0.0028, + "reward": 0.06304830685257912, + "reward_std": 0.0020843480597250164, + "rewards/accuracy_reward": 0.06304830685257912, + "step": 177 + }, + { + "clip_ratio": 0.0, + "completion_length": 32.35416793823242, + "epoch": 0.13879142300194933, + "grad_norm": 2.5423838619415133, + "kl": 0.69921875, + "learning_rate": 9.959952943612769e-07, + "loss": 0.1203, + "reward": 0.07849757373332977, + "reward_std": 0.005623013945296407, + "rewards/accuracy_reward": 0.07849757373332977, + "step": 178 + }, + { + "clip_ratio": 0.0, + "completion_length": 28.27083396911621, + "epoch": 0.13957115009746587, + "grad_norm": 5.737666881075233, + "kl": 1.26171875, + "learning_rate": 9.95830424521205e-07, + "loss": 0.1103, + "reward": 0.09047527983784676, + "reward_std": 0.005242464481852949, + "rewards/accuracy_reward": 0.09047527983784676, + "step": 179 + }, + { + "clip_ratio": 0.0, + "completion_length": 26.354166984558105, + "epoch": 0.14035087719298245, + "grad_norm": 4.495514537490841, + "kl": 0.685546875, + "learning_rate": 9.956622448139683e-07, + "loss": 0.0692, + "reward": 0.07989903539419174, + "reward_std": 0.004324757028371096, + "rewards/accuracy_reward": 0.07989903539419174, + "step": 180 + }, + { + "clip_ratio": 0.0, + "completion_length": 24.89583396911621, + "epoch": 0.14113060428849902, + "grad_norm": 3.27186911727588, + "kl": 0.86328125, + "learning_rate": 9.95490756488141e-07, + "loss": 0.1334, + "reward": 0.06422952748835087, + "reward_std": 0.0031616410706192255, + "rewards/accuracy_reward": 0.06422952748835087, + "step": 181 + }, + { + "clip_ratio": 0.0, + "completion_length": 27.5, + "epoch": 0.1419103313840156, + "grad_norm": 3.4239945776742995, + "kl": 0.806640625, + "learning_rate": 9.953159608168613e-07, + "loss": 0.154, + "reward": 0.05587464012205601, + "reward_std": 0.0017114627989940345, + "rewards/accuracy_reward": 0.05587464012205601, + "step": 182 + }, + { + "clip_ratio": 0.0, + "completion_length": 22.39583396911621, + "epoch": 0.14269005847953217, + "grad_norm": 2.2353911526023045, + "kl": 1.109375, + "learning_rate": 9.95137859097821e-07, + "loss": 0.0881, + "reward": 0.05541001074016094, + "reward_std": 0.0014768486726097763, + "rewards/accuracy_reward": 0.05541001074016094, + "step": 183 + }, + { + "clip_ratio": 0.0, + "completion_length": 27.458334922790527, + "epoch": 0.14346978557504872, + "grad_norm": 2.562843886095173, + "kl": 1.09765625, + "learning_rate": 9.949564526532557e-07, + "loss": 0.0957, + "reward": 0.09237155690789223, + "reward_std": 0.0017134369118139148, + "rewards/accuracy_reward": 0.09237155690789223, + "step": 184 + }, + { + "clip_ratio": 0.0, + "completion_length": 22.479167938232422, + "epoch": 0.1442495126705653, + "grad_norm": 4.646655813493582, + "kl": 1.5703125, + "learning_rate": 9.94771742829936e-07, + "loss": 0.1236, + "reward": 0.06373582407832146, + "reward_std": 0.003131686127744615, + "rewards/accuracy_reward": 0.06373582407832146, + "step": 185 + }, + { + "clip_ratio": 0.0, + "completion_length": 24.70833396911621, + "epoch": 0.14502923976608187, + "grad_norm": 3.4445156317270915, + "kl": 1.2734375, + "learning_rate": 9.94583730999157e-07, + "loss": 0.1488, + "reward": 0.09162703156471252, + "reward_std": 0.0037818995770066977, + "rewards/accuracy_reward": 0.09162703156471252, + "step": 186 + }, + { + "clip_ratio": 0.0, + "completion_length": 20.604166984558105, + "epoch": 0.14580896686159844, + "grad_norm": 2.7243563627467364, + "kl": 1.40234375, + "learning_rate": 9.943924185567274e-07, + "loss": 0.149, + "reward": 0.11167534068226814, + "reward_std": 0.004113486967980862, + "rewards/accuracy_reward": 0.11167534068226814, + "step": 187 + }, + { + "clip_ratio": 0.0, + "completion_length": 27.52083396911621, + "epoch": 0.14658869395711502, + "grad_norm": 11.256552780383403, + "kl": 2.4375, + "learning_rate": 9.941978069229606e-07, + "loss": 0.308, + "reward": 0.11367808282375336, + "reward_std": 0.010818830225616693, + "rewards/accuracy_reward": 0.11367808282375336, + "step": 188 + }, + { + "clip_ratio": 0.0, + "completion_length": 21.1875, + "epoch": 0.14736842105263157, + "grad_norm": 3.388916708085196, + "kl": 1.17578125, + "learning_rate": 9.93999897542663e-07, + "loss": 0.1004, + "reward": 0.08362429961562157, + "reward_std": 0.0025397372664883733, + "rewards/accuracy_reward": 0.08362429961562157, + "step": 189 + }, + { + "clip_ratio": 0.0, + "completion_length": 25.687500953674316, + "epoch": 0.14814814814814814, + "grad_norm": 2.177095776211899, + "kl": 0.826171875, + "learning_rate": 9.937986918851235e-07, + "loss": 0.1237, + "reward": 0.05470520816743374, + "reward_std": 0.0025200056843459606, + "rewards/accuracy_reward": 0.05470520816743374, + "step": 190 + }, + { + "clip_ratio": 0.0, + "completion_length": 24.729166984558105, + "epoch": 0.14892787524366471, + "grad_norm": 3.8120407278570934, + "kl": 1.29296875, + "learning_rate": 9.935941914441029e-07, + "loss": 0.1284, + "reward": 0.09313761256635189, + "reward_std": 0.006328779039904475, + "rewards/accuracy_reward": 0.09313761256635189, + "step": 191 + }, + { + "clip_ratio": 0.0, + "completion_length": 24.14583396911621, + "epoch": 0.1497076023391813, + "grad_norm": 1.7178873909449761, + "kl": 0.921875, + "learning_rate": 9.933863977378226e-07, + "loss": 0.0558, + "reward": 0.08159026131033897, + "reward_std": 0.000797520304331556, + "rewards/accuracy_reward": 0.08159026131033897, + "step": 192 + }, + { + "clip_ratio": 0.0, + "completion_length": 23.77083396911621, + "epoch": 0.15048732943469786, + "grad_norm": 0.6976605560382856, + "kl": 0.94921875, + "learning_rate": 9.931753123089537e-07, + "loss": 0.0371, + "reward": 0.06940789334475994, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.06940789334475994, + "step": 193 + }, + { + "clip_ratio": 0.0, + "completion_length": 21.83333396911621, + "epoch": 0.15126705653021444, + "grad_norm": 2.1454273958779524, + "kl": 1.201171875, + "learning_rate": 9.929609367246043e-07, + "loss": 0.0425, + "reward": 0.08664475753903389, + "reward_std": 0.000501172267831862, + "rewards/accuracy_reward": 0.08664475753903389, + "step": 194 + }, + { + "clip_ratio": 0.0, + "completion_length": 20.979167938232422, + "epoch": 0.15204678362573099, + "grad_norm": 4.148054190150091, + "kl": 1.5, + "learning_rate": 9.927432725763099e-07, + "loss": 0.1693, + "reward": 0.09565166756510735, + "reward_std": 0.003208951326087117, + "rewards/accuracy_reward": 0.09565166756510735, + "step": 195 + }, + { + "clip_ratio": 0.0, + "completion_length": 22.14583396911621, + "epoch": 0.15282651072124756, + "grad_norm": 3.3162153701918204, + "kl": 1.337890625, + "learning_rate": 9.925223214800195e-07, + "loss": 0.095, + "reward": 0.08522066101431847, + "reward_std": 0.005328871659003198, + "rewards/accuracy_reward": 0.08522066101431847, + "step": 196 + }, + { + "clip_ratio": 0.0, + "completion_length": 23.229166984558105, + "epoch": 0.15360623781676414, + "grad_norm": 1.4675893353585912, + "kl": 0.958984375, + "learning_rate": 9.922980850760853e-07, + "loss": 0.0315, + "reward": 0.05325814709067345, + "reward_std": 0.000361748127033934, + "rewards/accuracy_reward": 0.05325814709067345, + "step": 197 + }, + { + "clip_ratio": 0.0, + "completion_length": 19.4375, + "epoch": 0.1543859649122807, + "grad_norm": 0.1993853126111708, + "kl": 1.35546875, + "learning_rate": 9.920705650292492e-07, + "loss": 0.0539, + "reward": 0.12671999633312225, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.12671999633312225, + "step": 198 + }, + { + "clip_ratio": 0.0, + "completion_length": 19.416666984558105, + "epoch": 0.15516569200779728, + "grad_norm": 3.0105132998155137, + "kl": 1.75, + "learning_rate": 9.918397630286315e-07, + "loss": 0.0923, + "reward": 0.10997090488672256, + "reward_std": 0.0017170071951113641, + "rewards/accuracy_reward": 0.10997090488672256, + "step": 199 + }, + { + "clip_ratio": 0.0, + "completion_length": 24.666666984558105, + "epoch": 0.15594541910331383, + "grad_norm": 3.2302112602786344, + "kl": 1.390625, + "learning_rate": 9.916056807877178e-07, + "loss": 0.1054, + "reward": 0.06637427024543285, + "reward_std": 0.0010128951398655772, + "rewards/accuracy_reward": 0.06637427024543285, + "step": 200 + }, + { + "clip_ratio": 0.0, + "completion_length": 24.062500953674316, + "epoch": 0.1567251461988304, + "grad_norm": 1.6619564613991185, + "kl": 0.8984375, + "learning_rate": 9.91368320044346e-07, + "loss": 0.08, + "reward": 0.05893051438033581, + "reward_std": 0.0006389423506334424, + "rewards/accuracy_reward": 0.05893051438033581, + "step": 201 + }, + { + "clip_ratio": 0.0, + "completion_length": 21.312500953674316, + "epoch": 0.15750487329434698, + "grad_norm": 0.23056134045996093, + "kl": 1.7890625, + "learning_rate": 9.91127682560694e-07, + "loss": 0.0707, + "reward": 0.12712419778108597, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.12712419778108597, + "step": 202 + }, + { + "clip_ratio": 0.0, + "completion_length": 22.937500953674316, + "epoch": 0.15828460038986356, + "grad_norm": 26.567204103242744, + "kl": 1.984375, + "learning_rate": 9.908837701232667e-07, + "loss": 0.1266, + "reward": 0.07912641391158104, + "reward_std": 0.0013177419314160943, + "rewards/accuracy_reward": 0.07912641391158104, + "step": 203 + }, + { + "clip_ratio": 0.0, + "completion_length": 24.666666984558105, + "epoch": 0.15906432748538013, + "grad_norm": 2.0500912368942474, + "kl": 1.40234375, + "learning_rate": 9.906365845428819e-07, + "loss": 0.1702, + "reward": 0.10993445664644241, + "reward_std": 0.005610934458673, + "rewards/accuracy_reward": 0.10993445664644241, + "step": 204 + }, + { + "clip_ratio": 0.0, + "completion_length": 23.14583396911621, + "epoch": 0.15984405458089668, + "grad_norm": 2.096245744417209, + "kl": 1.166015625, + "learning_rate": 9.903861276546576e-07, + "loss": 0.0771, + "reward": 0.11442387849092484, + "reward_std": 0.0017005138797685504, + "rewards/accuracy_reward": 0.11442387849092484, + "step": 205 + }, + { + "clip_ratio": 0.0, + "completion_length": 25.89583396911621, + "epoch": 0.16062378167641325, + "grad_norm": 2.9395943067284422, + "kl": 1.19140625, + "learning_rate": 9.901324013179982e-07, + "loss": 0.0954, + "reward": 0.061130259186029434, + "reward_std": 0.006737629068084061, + "rewards/accuracy_reward": 0.061130259186029434, + "step": 206 + }, + { + "clip_ratio": 0.0, + "completion_length": 24.70833396911621, + "epoch": 0.16140350877192983, + "grad_norm": 1.7048208623990584, + "kl": 0.86328125, + "learning_rate": 9.898754074165806e-07, + "loss": 0.0277, + "reward": 0.09889741241931915, + "reward_std": 0.00021101986931171268, + "rewards/accuracy_reward": 0.09889741241931915, + "step": 207 + }, + { + "clip_ratio": 0.0, + "completion_length": 23.77083396911621, + "epoch": 0.1621832358674464, + "grad_norm": 3.3598199170740166, + "kl": 1.087890625, + "learning_rate": 9.896151478583404e-07, + "loss": 0.099, + "reward": 0.053031887859106064, + "reward_std": 0.0011279088648734614, + "rewards/accuracy_reward": 0.053031887859106064, + "step": 208 + }, + { + "clip_ratio": 0.0, + "completion_length": 21.20833396911621, + "epoch": 0.16296296296296298, + "grad_norm": 1.264916036059429, + "kl": 1.625, + "learning_rate": 9.89351624575457e-07, + "loss": 0.0858, + "reward": 0.12429441511631012, + "reward_std": 0.0008846581331454217, + "rewards/accuracy_reward": 0.12429441511631012, + "step": 209 + }, + { + "clip_ratio": 0.0, + "completion_length": 25.000000953674316, + "epoch": 0.16374269005847952, + "grad_norm": 1.7177509565631972, + "kl": 1.013671875, + "learning_rate": 9.890848395243409e-07, + "loss": 0.0565, + "reward": 0.11526317521929741, + "reward_std": 0.0005852422909811139, + "rewards/accuracy_reward": 0.11526317521929741, + "step": 210 + }, + { + "clip_ratio": 0.0, + "completion_length": 22.687500953674316, + "epoch": 0.1645224171539961, + "grad_norm": 2.998875915683939, + "kl": 1.26953125, + "learning_rate": 9.888147946856173e-07, + "loss": 0.0846, + "reward": 0.11437266692519188, + "reward_std": 0.0021981876925565302, + "rewards/accuracy_reward": 0.11437266692519188, + "step": 211 + }, + { + "clip_ratio": 0.0, + "completion_length": 19.187500953674316, + "epoch": 0.16530214424951267, + "grad_norm": 3.9208512584811426, + "kl": 1.34375, + "learning_rate": 9.885414920641122e-07, + "loss": 0.0698, + "reward": 0.06773879006505013, + "reward_std": 0.0006662883388344198, + "rewards/accuracy_reward": 0.06773879006505013, + "step": 212 + }, + { + "clip_ratio": 0.0, + "completion_length": 26.9375, + "epoch": 0.16608187134502925, + "grad_norm": 2.9525260615911257, + "kl": 0.90625, + "learning_rate": 9.88264933688838e-07, + "loss": 0.0569, + "reward": 0.09685194306075573, + "reward_std": 0.0014135038363747299, + "rewards/accuracy_reward": 0.09685194306075573, + "step": 213 + }, + { + "clip_ratio": 0.0, + "completion_length": 22.291667938232422, + "epoch": 0.16686159844054582, + "grad_norm": 5.227246556111158, + "kl": 0.94140625, + "learning_rate": 9.87985121612978e-07, + "loss": 0.1015, + "reward": 0.05899123102426529, + "reward_std": 0.001518539502285421, + "rewards/accuracy_reward": 0.05899123102426529, + "step": 214 + }, + { + "clip_ratio": 0.0, + "completion_length": 25.08333396911621, + "epoch": 0.16764132553606237, + "grad_norm": 2.326342301240418, + "kl": 1.08984375, + "learning_rate": 9.877020579138707e-07, + "loss": 0.0782, + "reward": 0.08564617857336998, + "reward_std": 0.0011196026753168553, + "rewards/accuracy_reward": 0.08564617857336998, + "step": 215 + }, + { + "clip_ratio": 0.0, + "completion_length": 25.687500953674316, + "epoch": 0.16842105263157894, + "grad_norm": 1.2358534767102056, + "kl": 1.08984375, + "learning_rate": 9.874157446929951e-07, + "loss": 0.0803, + "reward": 0.08892780169844627, + "reward_std": 0.0007336726412177086, + "rewards/accuracy_reward": 0.08892780169844627, + "step": 216 + }, + { + "clip_ratio": 0.0, + "completion_length": 22.52083396911621, + "epoch": 0.16920077972709552, + "grad_norm": 3.436935261638758, + "kl": 1.33203125, + "learning_rate": 9.871261840759545e-07, + "loss": 0.0834, + "reward": 0.08073633164167404, + "reward_std": 0.00847914395853877, + "rewards/accuracy_reward": 0.08073633164167404, + "step": 217 + }, + { + "clip_ratio": 0.0, + "completion_length": 22.64583396911621, + "epoch": 0.1699805068226121, + "grad_norm": 2.1566623189134253, + "kl": 1.71484375, + "learning_rate": 9.868333782124617e-07, + "loss": 0.1519, + "reward": 0.11506958678364754, + "reward_std": 0.0026018971111625433, + "rewards/accuracy_reward": 0.11506958678364754, + "step": 218 + }, + { + "clip_ratio": 0.0, + "completion_length": 20.95833396911621, + "epoch": 0.17076023391812867, + "grad_norm": 3.5488910051635676, + "kl": 1.4609375, + "learning_rate": 9.865373292763214e-07, + "loss": 0.0457, + "reward": 0.07090311497449875, + "reward_std": 0.0042670833063311875, + "rewards/accuracy_reward": 0.07090311497449875, + "step": 219 + }, + { + "clip_ratio": 0.0, + "completion_length": 22.52083396911621, + "epoch": 0.17153996101364521, + "grad_norm": 2.8053650736648077, + "kl": 1.3046875, + "learning_rate": 9.862380394654162e-07, + "loss": 0.0427, + "reward": 0.13785403966903687, + "reward_std": 0.007753290235996246, + "rewards/accuracy_reward": 0.13785403966903687, + "step": 220 + }, + { + "clip_ratio": 0.0, + "completion_length": 23.604167938232422, + "epoch": 0.1723196881091618, + "grad_norm": 2.2426806514275213, + "kl": 1.4375, + "learning_rate": 9.85935511001688e-07, + "loss": 0.1283, + "reward": 0.11332451924681664, + "reward_std": 0.0006854766397736967, + "rewards/accuracy_reward": 0.11332451924681664, + "step": 221 + }, + { + "clip_ratio": 0.0, + "completion_length": 23.875000953674316, + "epoch": 0.17309941520467836, + "grad_norm": 2.7303830779076885, + "kl": 1.59375, + "learning_rate": 9.856297461311233e-07, + "loss": 0.1, + "reward": 0.06828703731298447, + "reward_std": 0.004009376803878695, + "rewards/accuracy_reward": 0.06828703731298447, + "step": 222 + }, + { + "clip_ratio": 0.0, + "completion_length": 21.33333396911621, + "epoch": 0.17387914230019494, + "grad_norm": 1.7534862814182692, + "kl": 1.8125, + "learning_rate": 9.853207471237357e-07, + "loss": 0.1116, + "reward": 0.08874018490314484, + "reward_std": 0.0019992401357740164, + "rewards/accuracy_reward": 0.08874018490314484, + "step": 223 + }, + { + "clip_ratio": 0.0, + "completion_length": 22.39583396911621, + "epoch": 0.1746588693957115, + "grad_norm": 2.920053943391909, + "kl": 1.73828125, + "learning_rate": 9.85008516273549e-07, + "loss": 0.1007, + "reward": 0.09884331189095974, + "reward_std": 0.005048536462709308, + "rewards/accuracy_reward": 0.09884331189095974, + "step": 224 + }, + { + "clip_ratio": 0.0, + "completion_length": 23.89583396911621, + "epoch": 0.17543859649122806, + "grad_norm": 2.899972090799929, + "kl": 1.328125, + "learning_rate": 9.846930558985806e-07, + "loss": 0.0758, + "reward": 0.06051616370677948, + "reward_std": 0.005370165046770126, + "rewards/accuracy_reward": 0.06051616370677948, + "step": 225 + }, + { + "clip_ratio": 0.0, + "completion_length": 23.104166984558105, + "epoch": 0.17621832358674464, + "grad_norm": 3.466343735070648, + "kl": 1.48828125, + "learning_rate": 9.84374368340824e-07, + "loss": 0.0647, + "reward": 0.10638459771871567, + "reward_std": 0.00696826761122793, + "rewards/accuracy_reward": 0.10638459771871567, + "step": 226 + }, + { + "clip_ratio": 0.0, + "completion_length": 22.562500953674316, + "epoch": 0.1769980506822612, + "grad_norm": 0.5345048985939578, + "kl": 1.40234375, + "learning_rate": 9.840524559662313e-07, + "loss": 0.0557, + "reward": 0.0833333395421505, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.0833333395421505, + "step": 227 + }, + { + "clip_ratio": 0.0, + "completion_length": 20.291666984558105, + "epoch": 0.17777777777777778, + "grad_norm": 2.5430356173050876, + "kl": 1.75, + "learning_rate": 9.83727321164696e-07, + "loss": 0.0834, + "reward": 0.07123610191047192, + "reward_std": 0.004868295218329877, + "rewards/accuracy_reward": 0.07123610191047192, + "step": 228 + }, + { + "clip_ratio": 0.0, + "completion_length": 24.5625, + "epoch": 0.17855750487329436, + "grad_norm": 3.2574801690348227, + "kl": 1.765625, + "learning_rate": 9.83398966350035e-07, + "loss": 0.125, + "reward": 0.10326099023222923, + "reward_std": 0.004647268156986684, + "rewards/accuracy_reward": 0.10326099023222923, + "step": 229 + }, + { + "clip_ratio": 0.0, + "completion_length": 21.187500953674316, + "epoch": 0.1793372319688109, + "grad_norm": 1.266097751271849, + "kl": 1.390625, + "learning_rate": 9.830673939599705e-07, + "loss": 0.0599, + "reward": 0.08679194748401642, + "reward_std": 0.0003180142375640571, + "rewards/accuracy_reward": 0.08679194748401642, + "step": 230 + }, + { + "clip_ratio": 0.0, + "completion_length": 23.08333396911621, + "epoch": 0.18011695906432748, + "grad_norm": 2.8167759580489222, + "kl": 1.60546875, + "learning_rate": 9.827326064561125e-07, + "loss": 0.0762, + "reward": 0.0730651281774044, + "reward_std": 0.007135274587199092, + "rewards/accuracy_reward": 0.0730651281774044, + "step": 231 + }, + { + "clip_ratio": 0.0, + "completion_length": 14.500000953674316, + "epoch": 0.18089668615984406, + "grad_norm": 0.16113981868029764, + "kl": 2.31640625, + "learning_rate": 9.823946063239402e-07, + "loss": 0.0915, + "reward": 0.129901971668005, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.129901971668005, + "step": 232 + }, + { + "clip_ratio": 0.0, + "completion_length": 21.541667938232422, + "epoch": 0.18167641325536063, + "grad_norm": 1.332369107753439, + "kl": 1.61328125, + "learning_rate": 9.820533960727832e-07, + "loss": 0.0441, + "reward": 0.1135011799633503, + "reward_std": 0.00021101981110405177, + "rewards/accuracy_reward": 0.1135011799633503, + "step": 233 + }, + { + "clip_ratio": 0.0, + "completion_length": 23.64583396911621, + "epoch": 0.1824561403508772, + "grad_norm": 1.5317502560363618, + "kl": 1.19921875, + "learning_rate": 9.817089782358032e-07, + "loss": 0.0519, + "reward": 0.11487788707017899, + "reward_std": 0.004591436590999365, + "rewards/accuracy_reward": 0.11487788707017899, + "step": 234 + }, + { + "clip_ratio": 0.0, + "completion_length": 17.52083396911621, + "epoch": 0.18323586744639375, + "grad_norm": 0.2671234363243526, + "kl": 2.04296875, + "learning_rate": 9.813613553699755e-07, + "loss": 0.0808, + "reward": 0.09689543023705482, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.09689543023705482, + "step": 235 + }, + { + "clip_ratio": 0.0, + "completion_length": 22.52083396911621, + "epoch": 0.18401559454191033, + "grad_norm": 2.661407358134365, + "kl": 2.02734375, + "learning_rate": 9.810105300560692e-07, + "loss": 0.0968, + "reward": 0.08690152689814568, + "reward_std": 0.0016228330787271261, + "rewards/accuracy_reward": 0.08690152689814568, + "step": 236 + }, + { + "clip_ratio": 0.0, + "completion_length": 23.979167938232422, + "epoch": 0.1847953216374269, + "grad_norm": 1.6977362227800048, + "kl": 1.8203125, + "learning_rate": 9.80656504898629e-07, + "loss": 0.0952, + "reward": 0.0872369036078453, + "reward_std": 0.0006903500179760158, + "rewards/accuracy_reward": 0.0872369036078453, + "step": 237 + }, + { + "clip_ratio": 0.0, + "completion_length": 20.437500953674316, + "epoch": 0.18557504873294348, + "grad_norm": 3.6993083959009114, + "kl": 1.66015625, + "learning_rate": 9.802992825259553e-07, + "loss": 0.0843, + "reward": 0.07765689864754677, + "reward_std": 0.011905404506251216, + "rewards/accuracy_reward": 0.07765689864754677, + "step": 238 + }, + { + "clip_ratio": 0.0, + "completion_length": 25.750000953674316, + "epoch": 0.18635477582846005, + "grad_norm": 0.237259194490552, + "kl": 1.3359375, + "learning_rate": 9.799388655900844e-07, + "loss": 0.0533, + "reward": 0.08496732264757156, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.08496732264757156, + "step": 239 + }, + { + "clip_ratio": 0.0, + "completion_length": 24.166667938232422, + "epoch": 0.1871345029239766, + "grad_norm": 0.325618890659599, + "kl": 1.123046875, + "learning_rate": 9.7957525676677e-07, + "loss": 0.0445, + "reward": 0.11388889327645302, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.11388889327645302, + "step": 240 + }, + { + "clip_ratio": 0.0, + "completion_length": 22.312500953674316, + "epoch": 0.18791423001949317, + "grad_norm": 3.0014091913037557, + "kl": 1.37890625, + "learning_rate": 9.792084587554613e-07, + "loss": 0.0678, + "reward": 0.06449008546769619, + "reward_std": 0.0007475692837033421, + "rewards/accuracy_reward": 0.06449008546769619, + "step": 241 + }, + { + "clip_ratio": 0.0, + "completion_length": 18.937500953674316, + "epoch": 0.18869395711500975, + "grad_norm": 4.347230766020519, + "kl": 2.1640625, + "learning_rate": 9.788384742792855e-07, + "loss": 0.1175, + "reward": 0.08908803015947342, + "reward_std": 0.012355273123830557, + "rewards/accuracy_reward": 0.08908803015947342, + "step": 242 + }, + { + "clip_ratio": 0.0, + "completion_length": 21.437500953674316, + "epoch": 0.18947368421052632, + "grad_norm": 0.25889614498853536, + "kl": 1.50390625, + "learning_rate": 9.784653060850256e-07, + "loss": 0.0576, + "reward": 0.11633987352252007, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.11633987352252007, + "step": 243 + }, + { + "clip_ratio": 0.0, + "completion_length": 22.33333396911621, + "epoch": 0.1902534113060429, + "grad_norm": 1.5983174576317682, + "kl": 1.7890625, + "learning_rate": 9.780889569431006e-07, + "loss": 0.112, + "reward": 0.09527980536222458, + "reward_std": 0.005020395619794726, + "rewards/accuracy_reward": 0.09527980536222458, + "step": 244 + }, + { + "clip_ratio": 0.0, + "completion_length": 20.666667938232422, + "epoch": 0.19103313840155944, + "grad_norm": 2.6408241280763, + "kl": 1.515625, + "learning_rate": 9.77709429647545e-07, + "loss": 0.057, + "reward": 0.08550266921520233, + "reward_std": 0.0074745421297848225, + "rewards/accuracy_reward": 0.08550266921520233, + "step": 245 + }, + { + "clip_ratio": 0.0, + "completion_length": 21.875000953674316, + "epoch": 0.19181286549707602, + "grad_norm": 1.9120022670006256, + "kl": 1.58203125, + "learning_rate": 9.77326727015989e-07, + "loss": 0.0382, + "reward": 0.12758715078234673, + "reward_std": 0.006278668995946646, + "rewards/accuracy_reward": 0.12758715078234673, + "step": 246 + }, + { + "clip_ratio": 0.0, + "completion_length": 16.64583396911621, + "epoch": 0.1925925925925926, + "grad_norm": 3.9104437289728287, + "kl": 1.9296875, + "learning_rate": 9.76940851889635e-07, + "loss": 0.1131, + "reward": 0.09734449535608292, + "reward_std": 0.01164670754224062, + "rewards/accuracy_reward": 0.09734449535608292, + "step": 247 + }, + { + "clip_ratio": 0.0, + "completion_length": 23.02083396911621, + "epoch": 0.19337231968810917, + "grad_norm": 3.846049762456662, + "kl": 1.65625, + "learning_rate": 9.765518071332396e-07, + "loss": 0.0755, + "reward": 0.0830635204911232, + "reward_std": 0.015773563645780087, + "rewards/accuracy_reward": 0.0830635204911232, + "step": 248 + }, + { + "clip_ratio": 0.0, + "completion_length": 22.9375, + "epoch": 0.19415204678362574, + "grad_norm": 2.3131674568250538, + "kl": 1.537109375, + "learning_rate": 9.7615959563509e-07, + "loss": 0.0206, + "reward": 0.12442300841212273, + "reward_std": 0.007980600406881422, + "rewards/accuracy_reward": 0.12442300841212273, + "step": 249 + }, + { + "clip_ratio": 0.0, + "completion_length": 25.479167938232422, + "epoch": 0.1949317738791423, + "grad_norm": 4.195497169434552, + "kl": 1.62109375, + "learning_rate": 9.757642203069843e-07, + "loss": 0.035, + "reward": 0.12204422801733017, + "reward_std": 0.0224958173930645, + "rewards/accuracy_reward": 0.12204422801733017, + "step": 250 + }, + { + "clip_ratio": 0.0, + "completion_length": 19.604166984558105, + "epoch": 0.19571150097465886, + "grad_norm": 5.2103577929033555, + "kl": 2.1015625, + "learning_rate": 9.753656840842082e-07, + "loss": 0.0305, + "reward": 0.1147073544561863, + "reward_std": 0.025445946492254734, + "rewards/accuracy_reward": 0.1147073544561863, + "step": 251 + }, + { + "clip_ratio": 0.0, + "completion_length": 25.812500953674316, + "epoch": 0.19649122807017544, + "grad_norm": 2.3995622137473163, + "kl": 1.53515625, + "learning_rate": 9.749639899255143e-07, + "loss": 0.0673, + "reward": 0.12678389996290207, + "reward_std": 0.003454159013926983, + "rewards/accuracy_reward": 0.12678389996290207, + "step": 252 + }, + { + "clip_ratio": 0.0, + "completion_length": 22.104166984558105, + "epoch": 0.197270955165692, + "grad_norm": 1.7428432615273857, + "kl": 1.77734375, + "learning_rate": 9.745591408131e-07, + "loss": 0.0729, + "reward": 0.12410277128219604, + "reward_std": 0.0005306522361934185, + "rewards/accuracy_reward": 0.12410277128219604, + "step": 253 + }, + { + "clip_ratio": 0.0, + "completion_length": 24.104167938232422, + "epoch": 0.1980506822612086, + "grad_norm": 1.8545050575670603, + "kl": 1.3359375, + "learning_rate": 9.741511397525852e-07, + "loss": 0.0902, + "reward": 0.13580767065286636, + "reward_std": 0.0009267845889553428, + "rewards/accuracy_reward": 0.13580767065286636, + "step": 254 + }, + { + "clip_ratio": 0.0, + "completion_length": 23.604166984558105, + "epoch": 0.19883040935672514, + "grad_norm": 3.245255032482792, + "kl": 1.8203125, + "learning_rate": 9.737399897729902e-07, + "loss": 0.1621, + "reward": 0.13663917034864426, + "reward_std": 0.0038499051588587463, + "rewards/accuracy_reward": 0.13663917034864426, + "step": 255 + }, + { + "clip_ratio": 0.0, + "completion_length": 21.937500953674316, + "epoch": 0.1996101364522417, + "grad_norm": 3.381318839529289, + "kl": 1.890625, + "learning_rate": 9.733256939267127e-07, + "loss": 0.2123, + "reward": 0.13044091314077377, + "reward_std": 0.005797044548671693, + "rewards/accuracy_reward": 0.13044091314077377, + "step": 256 + }, + { + "clip_ratio": 0.0, + "completion_length": 20.666666984558105, + "epoch": 0.20038986354775828, + "grad_norm": 3.473785023099609, + "kl": 1.91015625, + "learning_rate": 9.72908255289505e-07, + "loss": 0.1755, + "reward": 0.1318226270377636, + "reward_std": 0.0018371137557551265, + "rewards/accuracy_reward": 0.1318226270377636, + "step": 257 + }, + { + "clip_ratio": 0.0, + "completion_length": 20.83333396911621, + "epoch": 0.20116959064327486, + "grad_norm": 2.813642793968136, + "kl": 1.8359375, + "learning_rate": 9.724876769604534e-07, + "loss": 0.0434, + "reward": 0.13815467059612274, + "reward_std": 0.0005697530577890575, + "rewards/accuracy_reward": 0.13815467059612274, + "step": 258 + }, + { + "clip_ratio": 0.0, + "completion_length": 22.291666984558105, + "epoch": 0.20194931773879143, + "grad_norm": 3.9107649012845216, + "kl": 2.0390625, + "learning_rate": 9.72063962061951e-07, + "loss": 0.1536, + "reward": 0.137812077999115, + "reward_std": 0.002921649720519781, + "rewards/accuracy_reward": 0.137812077999115, + "step": 259 + }, + { + "clip_ratio": 0.0, + "completion_length": 25.02083396911621, + "epoch": 0.20272904483430798, + "grad_norm": 1.4346009750815534, + "kl": 1.58203125, + "learning_rate": 9.716371137396793e-07, + "loss": 0.0892, + "reward": 0.13681723177433014, + "reward_std": 0.0009050843073055148, + "rewards/accuracy_reward": 0.13681723177433014, + "step": 260 + }, + { + "clip_ratio": 0.0, + "completion_length": 22.291667938232422, + "epoch": 0.20350877192982456, + "grad_norm": 3.4866115323473976, + "kl": 1.74609375, + "learning_rate": 9.712071351625806e-07, + "loss": 0.0738, + "reward": 0.13259827345609665, + "reward_std": 0.003800537611823529, + "rewards/accuracy_reward": 0.13259827345609665, + "step": 261 + }, + { + "clip_ratio": 0.0, + "completion_length": 23.89583396911621, + "epoch": 0.20428849902534113, + "grad_norm": 1.7333177534526967, + "kl": 1.57421875, + "learning_rate": 9.707740295228377e-07, + "loss": 0.0596, + "reward": 0.14202070236206055, + "reward_std": 0.00047169136814773083, + "rewards/accuracy_reward": 0.14202070236206055, + "step": 262 + }, + { + "clip_ratio": 0.0, + "completion_length": 22.08333396911621, + "epoch": 0.2050682261208577, + "grad_norm": 1.8505394432271243, + "kl": 1.74609375, + "learning_rate": 9.70337800035848e-07, + "loss": 0.0727, + "reward": 0.12503890693187714, + "reward_std": 0.0011856432538479567, + "rewards/accuracy_reward": 0.12503890693187714, + "step": 263 + }, + { + "clip_ratio": 0.0, + "completion_length": 22.14583396911621, + "epoch": 0.20584795321637428, + "grad_norm": 1.970418354488007, + "kl": 1.875, + "learning_rate": 9.698984499402009e-07, + "loss": 0.0725, + "reward": 0.12785088270902634, + "reward_std": 0.0012852847576141357, + "rewards/accuracy_reward": 0.12785088270902634, + "step": 264 + }, + { + "clip_ratio": 0.0, + "completion_length": 21.75, + "epoch": 0.20662768031189083, + "grad_norm": 3.8673657513171866, + "kl": 1.8515625, + "learning_rate": 9.694559824976534e-07, + "loss": 0.1285, + "reward": 0.13662027940154076, + "reward_std": 0.0060649634106084704, + "rewards/accuracy_reward": 0.13662027940154076, + "step": 265 + }, + { + "clip_ratio": 0.0, + "completion_length": 23.45833396911621, + "epoch": 0.2074074074074074, + "grad_norm": 1.7690362347869697, + "kl": 1.50390625, + "learning_rate": 9.690104009931055e-07, + "loss": 0.0542, + "reward": 0.13606832176446915, + "reward_std": 0.00047169136814773083, + "rewards/accuracy_reward": 0.13606832176446915, + "step": 266 + }, + { + "clip_ratio": 0.0, + "completion_length": 21.187500953674316, + "epoch": 0.20818713450292398, + "grad_norm": 1.82646931051954, + "kl": 1.71484375, + "learning_rate": 9.685617087345766e-07, + "loss": 0.1062, + "reward": 0.13011812418699265, + "reward_std": 0.0031333768274635077, + "rewards/accuracy_reward": 0.13011812418699265, + "step": 267 + }, + { + "clip_ratio": 0.0, + "completion_length": 18.416666984558105, + "epoch": 0.20896686159844055, + "grad_norm": 1.480990254801878, + "kl": 1.99609375, + "learning_rate": 9.681099090531803e-07, + "loss": 0.1256, + "reward": 0.15551593899726868, + "reward_std": 0.0017183038871735334, + "rewards/accuracy_reward": 0.15551593899726868, + "step": 268 + }, + { + "clip_ratio": 0.0, + "completion_length": 61.979166984558105, + "epoch": 0.20974658869395713, + "grad_norm": 4.45416174052944, + "kl": 2.03515625, + "learning_rate": 9.676550053030993e-07, + "loss": 0.6251, + "reward": 0.14152297377586365, + "reward_std": 0.010142141720280051, + "rewards/accuracy_reward": 0.14152297377586365, + "step": 269 + }, + { + "clip_ratio": 0.0, + "completion_length": 17.916667938232422, + "epoch": 0.21052631578947367, + "grad_norm": 3.672160807656365, + "kl": 2.2890625, + "learning_rate": 9.671970008615625e-07, + "loss": 0.1664, + "reward": 0.15890305489301682, + "reward_std": 0.008111248724162579, + "rewards/accuracy_reward": 0.15890305489301682, + "step": 270 + }, + { + "clip_ratio": 0.0, + "completion_length": 63.4375, + "epoch": 0.21130604288499025, + "grad_norm": 2.6396408305994674, + "kl": 1.921875, + "learning_rate": 9.667358991288167e-07, + "loss": 0.542, + "reward": 0.12156940624117851, + "reward_std": 0.009339521639049053, + "rewards/accuracy_reward": 0.12156940624117851, + "step": 271 + }, + { + "clip_ratio": 0.0, + "completion_length": 19.541667938232422, + "epoch": 0.21208576998050682, + "grad_norm": 3.8827213064852795, + "kl": 1.8125, + "learning_rate": 9.662717035281044e-07, + "loss": 0.1395, + "reward": 0.16217350959777832, + "reward_std": 0.00623874575830996, + "rewards/accuracy_reward": 0.16217350959777832, + "step": 272 + }, + { + "clip_ratio": 0.0, + "completion_length": 20.89583396911621, + "epoch": 0.2128654970760234, + "grad_norm": 4.872856767039602, + "kl": 1.85546875, + "learning_rate": 9.658044175056367e-07, + "loss": 0.0915, + "reward": 0.15208658576011658, + "reward_std": 0.005818813806399703, + "rewards/accuracy_reward": 0.15208658576011658, + "step": 273 + }, + { + "clip_ratio": 0.0, + "completion_length": 16.416666984558105, + "epoch": 0.21364522417153997, + "grad_norm": 6.858118696733246, + "kl": 2.328125, + "learning_rate": 9.653340445305682e-07, + "loss": 0.3973, + "reward": 0.13756439834833145, + "reward_std": 0.011764792259782553, + "rewards/accuracy_reward": 0.13756439834833145, + "step": 274 + }, + { + "clip_ratio": 0.0, + "completion_length": 17.979166984558105, + "epoch": 0.21442495126705652, + "grad_norm": 6.702304859912742, + "kl": 2.04296875, + "learning_rate": 9.648605880949713e-07, + "loss": 0.16, + "reward": 0.1567860245704651, + "reward_std": 0.010588282719254494, + "rewards/accuracy_reward": 0.1567860245704651, + "step": 275 + }, + { + "clip_ratio": 0.0, + "completion_length": 17.166667461395264, + "epoch": 0.2152046783625731, + "grad_norm": 4.878099611797554, + "kl": 2.0078125, + "learning_rate": 9.643840517138098e-07, + "loss": 0.1372, + "reward": 0.12988395988941193, + "reward_std": 0.00418971199542284, + "rewards/accuracy_reward": 0.12988395988941193, + "step": 276 + }, + { + "clip_ratio": 0.0, + "completion_length": 18.375000953674316, + "epoch": 0.21598440545808967, + "grad_norm": 3.755944790669856, + "kl": 2.3046875, + "learning_rate": 9.639044389249134e-07, + "loss": 0.1197, + "reward": 0.16065938770771027, + "reward_std": 0.007759660831652582, + "rewards/accuracy_reward": 0.16065938770771027, + "step": 277 + }, + { + "clip_ratio": 0.0, + "completion_length": 13.541666984558105, + "epoch": 0.21676413255360624, + "grad_norm": 5.4220528733524604, + "kl": 2.921875, + "learning_rate": 9.634217532889515e-07, + "loss": 0.2103, + "reward": 0.1402498111128807, + "reward_std": 0.008417865494266152, + "rewards/accuracy_reward": 0.1402498111128807, + "step": 278 + }, + { + "clip_ratio": 0.0, + "completion_length": 12.291666984558105, + "epoch": 0.21754385964912282, + "grad_norm": 7.0235756541437855, + "kl": 3.2109375, + "learning_rate": 9.62935998389406e-07, + "loss": 0.3883, + "reward": 0.1477484181523323, + "reward_std": 0.011410552775487304, + "rewards/accuracy_reward": 0.1477484181523323, + "step": 279 + }, + { + "clip_ratio": 0.0, + "completion_length": 55.89583492279053, + "epoch": 0.21832358674463936, + "grad_norm": 10.110573353293253, + "kl": 2.90625, + "learning_rate": 9.624471778325455e-07, + "loss": 0.5111, + "reward": 0.16363994777202606, + "reward_std": 0.011551804840564728, + "rewards/accuracy_reward": 0.16363994777202606, + "step": 280 + }, + { + "clip_ratio": 0.0, + "completion_length": 52.375001430511475, + "epoch": 0.21910331384015594, + "grad_norm": 22.235508699230277, + "kl": 3.234375, + "learning_rate": 9.619552952473976e-07, + "loss": -0.2041, + "reward": 0.14994164556264877, + "reward_std": 0.002838627900928259, + "rewards/accuracy_reward": 0.14994164556264877, + "step": 281 + }, + { + "clip_ratio": 0.0, + "completion_length": 11.750000476837158, + "epoch": 0.2198830409356725, + "grad_norm": 8.846563763087351, + "kl": 3.1171875, + "learning_rate": 9.614603542857235e-07, + "loss": -0.1139, + "reward": 0.16706421971321106, + "reward_std": 0.01608453644439578, + "rewards/accuracy_reward": 0.16706421971321106, + "step": 282 + }, + { + "clip_ratio": 0.0, + "completion_length": 16.020833492279053, + "epoch": 0.2206627680311891, + "grad_norm": 5.839429613378161, + "kl": 2.7421875, + "learning_rate": 9.609623586219892e-07, + "loss": 0.0327, + "reward": 0.15950050950050354, + "reward_std": 0.014836331363767385, + "rewards/accuracy_reward": 0.15950050950050354, + "step": 283 + }, + { + "clip_ratio": 0.0, + "completion_length": 14.791666984558105, + "epoch": 0.22144249512670566, + "grad_norm": 5.743529221608405, + "kl": 3.0078125, + "learning_rate": 9.604613119533389e-07, + "loss": 0.4062, + "reward": 0.1408693641424179, + "reward_std": 0.013236512430012226, + "rewards/accuracy_reward": 0.1408693641424179, + "step": 284 + }, + { + "clip_ratio": 0.0, + "completion_length": 15.3125, + "epoch": 0.2222222222222222, + "grad_norm": 3.2998580002726587, + "kl": 2.8828125, + "learning_rate": 9.599572179995684e-07, + "loss": 0.299, + "reward": 0.14530812948942184, + "reward_std": 0.004396125441417098, + "rewards/accuracy_reward": 0.14530812948942184, + "step": 285 + }, + { + "clip_ratio": 0.0, + "completion_length": 12.770833969116211, + "epoch": 0.22300194931773878, + "grad_norm": 8.29951318737398, + "kl": 3.046875, + "learning_rate": 9.59450080503096e-07, + "loss": 0.3234, + "reward": 0.14096882194280624, + "reward_std": 0.006384085863828659, + "rewards/accuracy_reward": 0.14096882194280624, + "step": 286 + }, + { + "clip_ratio": 0.0, + "completion_length": 12.229166984558105, + "epoch": 0.22378167641325536, + "grad_norm": 6.7364372054781825, + "kl": 2.9921875, + "learning_rate": 9.589399032289355e-07, + "loss": 0.2129, + "reward": 0.15641389042139053, + "reward_std": 0.008956428384408355, + "rewards/accuracy_reward": 0.15641389042139053, + "step": 287 + }, + { + "clip_ratio": 0.0, + "completion_length": 16.125000476837158, + "epoch": 0.22456140350877193, + "grad_norm": 7.416919295448386, + "kl": 2.31640625, + "learning_rate": 9.584266899646684e-07, + "loss": 0.3471, + "reward": 0.15673290938138962, + "reward_std": 0.010735958348959684, + "rewards/accuracy_reward": 0.15673290938138962, + "step": 288 + }, + { + "clip_ratio": 0.0, + "completion_length": 12.375, + "epoch": 0.2253411306042885, + "grad_norm": 6.4774087181717075, + "kl": 2.890625, + "learning_rate": 9.579104445204155e-07, + "loss": 0.2685, + "reward": 0.1649218201637268, + "reward_std": 0.0024056266993284225, + "rewards/accuracy_reward": 0.1649218201637268, + "step": 289 + }, + { + "clip_ratio": 0.0, + "completion_length": 12.479166984558105, + "epoch": 0.22612085769980506, + "grad_norm": 6.171742085607571, + "kl": 3.0625, + "learning_rate": 9.573911707288082e-07, + "loss": 0.2615, + "reward": 0.15222876518964767, + "reward_std": 0.00795425078831613, + "rewards/accuracy_reward": 0.15222876518964767, + "step": 290 + }, + { + "clip_ratio": 0.0, + "completion_length": 11.375, + "epoch": 0.22690058479532163, + "grad_norm": 2.8424365519530763, + "kl": 2.9921875, + "learning_rate": 9.568688724449613e-07, + "loss": 0.1239, + "reward": 0.13931213319301605, + "reward_std": 0.004959697602316737, + "rewards/accuracy_reward": 0.13931213319301605, + "step": 291 + }, + { + "clip_ratio": 0.0, + "completion_length": 11.729166984558105, + "epoch": 0.2276803118908382, + "grad_norm": 4.783219578380445, + "kl": 3.2265625, + "learning_rate": 9.563435535464427e-07, + "loss": 0.3647, + "reward": 0.13829627633094788, + "reward_std": 0.011282553663477302, + "rewards/accuracy_reward": 0.13829627633094788, + "step": 292 + }, + { + "clip_ratio": 0.0, + "completion_length": 15.0625, + "epoch": 0.22846003898635478, + "grad_norm": 1.7699679624581726, + "kl": 2.515625, + "learning_rate": 9.558152179332463e-07, + "loss": 0.0712, + "reward": 0.1633118912577629, + "reward_std": 0.008225119207054377, + "rewards/accuracy_reward": 0.1633118912577629, + "step": 293 + }, + { + "clip_ratio": 0.0, + "completion_length": 13.458333969116211, + "epoch": 0.22923976608187135, + "grad_norm": 5.607499819766133, + "kl": 2.640625, + "learning_rate": 9.552838695277617e-07, + "loss": 0.2946, + "reward": 0.1461936980485916, + "reward_std": 0.00803038408048451, + "rewards/accuracy_reward": 0.1461936980485916, + "step": 294 + }, + { + "clip_ratio": 0.0, + "completion_length": 12.562500476837158, + "epoch": 0.2300194931773879, + "grad_norm": 1.5714562706963744, + "kl": 2.984375, + "learning_rate": 9.547495122747457e-07, + "loss": 0.0915, + "reward": 0.1667862981557846, + "reward_std": 0.005498574580997229, + "rewards/accuracy_reward": 0.1667862981557846, + "step": 295 + }, + { + "clip_ratio": 0.0, + "completion_length": 13.1875, + "epoch": 0.23079922027290448, + "grad_norm": 3.620635251174383, + "kl": 3.15625, + "learning_rate": 9.542121501412933e-07, + "loss": 0.261, + "reward": 0.15238095819950104, + "reward_std": 0.005222418578341603, + "rewards/accuracy_reward": 0.15238095819950104, + "step": 296 + }, + { + "clip_ratio": 0.0, + "completion_length": 14.541667461395264, + "epoch": 0.23157894736842105, + "grad_norm": 4.4303120652811625, + "kl": 2.6171875, + "learning_rate": 9.536717871168076e-07, + "loss": 0.1994, + "reward": 0.17520426213741302, + "reward_std": 0.005448035895824432, + "rewards/accuracy_reward": 0.17520426213741302, + "step": 297 + }, + { + "clip_ratio": 0.0, + "completion_length": 16.08333396911621, + "epoch": 0.23235867446393763, + "grad_norm": 4.022228405944924, + "kl": 2.6484375, + "learning_rate": 9.5312842721297e-07, + "loss": 0.133, + "reward": 0.15377682447433472, + "reward_std": 0.011845491826534271, + "rewards/accuracy_reward": 0.15377682447433472, + "step": 298 + }, + { + "clip_ratio": 0.0, + "completion_length": 13.333333969116211, + "epoch": 0.2331384015594542, + "grad_norm": 11.681673402523783, + "kl": 3.921875, + "learning_rate": 9.525820744637119e-07, + "loss": 0.4493, + "reward": 0.1410665586590767, + "reward_std": 0.014458210207521915, + "rewards/accuracy_reward": 0.1410665586590767, + "step": 299 + }, + { + "clip_ratio": 0.0, + "completion_length": 14.708333969116211, + "epoch": 0.23391812865497075, + "grad_norm": 1.4599098168537827, + "kl": 2.6015625, + "learning_rate": 9.520327329251831e-07, + "loss": 0.0997, + "reward": 0.17216148227453232, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.17216148227453232, + "step": 300 + }, + { + "clip_ratio": 0.0, + "completion_length": 12.645833969116211, + "epoch": 0.23469785575048732, + "grad_norm": 4.73839760766751, + "kl": 2.8359375, + "learning_rate": 9.514804066757219e-07, + "loss": 0.2478, + "reward": 0.14820358902215958, + "reward_std": 0.007825780659914017, + "rewards/accuracy_reward": 0.14820358902215958, + "step": 301 + }, + { + "clip_ratio": 0.0, + "completion_length": 12.437500476837158, + "epoch": 0.2354775828460039, + "grad_norm": 4.983771234470083, + "kl": 2.6875, + "learning_rate": 9.509250998158262e-07, + "loss": 0.1925, + "reward": 0.16442348062992096, + "reward_std": 0.005435790400952101, + "rewards/accuracy_reward": 0.16442348062992096, + "step": 302 + }, + { + "clip_ratio": 0.0, + "completion_length": 12.125000476837158, + "epoch": 0.23625730994152047, + "grad_norm": 6.2020739563672045, + "kl": 3.0703125, + "learning_rate": 9.503668164681219e-07, + "loss": 0.271, + "reward": 0.14058363065123558, + "reward_std": 0.011176002211868763, + "rewards/accuracy_reward": 0.14058363065123558, + "step": 303 + }, + { + "clip_ratio": 0.0, + "completion_length": 11.770833730697632, + "epoch": 0.23703703703703705, + "grad_norm": 0.4211076751532243, + "kl": 3.00390625, + "learning_rate": 9.498055607773316e-07, + "loss": 0.1176, + "reward": 0.17432598769664764, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.17432598769664764, + "step": 304 + }, + { + "clip_ratio": 0.0, + "completion_length": 11.020833492279053, + "epoch": 0.2378167641325536, + "grad_norm": 3.1760270982848424, + "kl": 3.140625, + "learning_rate": 9.492413369102458e-07, + "loss": 0.1373, + "reward": 0.1416749358177185, + "reward_std": 0.001002344535663724, + "rewards/accuracy_reward": 0.1416749358177185, + "step": 305 + }, + { + "clip_ratio": 0.0, + "completion_length": 12.062500476837158, + "epoch": 0.23859649122807017, + "grad_norm": 5.763208067499548, + "kl": 3.125, + "learning_rate": 9.486741490556902e-07, + "loss": 0.1803, + "reward": 0.14505772292613983, + "reward_std": 0.0010257635731250048, + "rewards/accuracy_reward": 0.14505772292613983, + "step": 306 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.25, + "epoch": 0.23937621832358674, + "grad_norm": 0.2239305538920244, + "kl": 3.65625, + "learning_rate": 9.481040014244958e-07, + "loss": 0.1438, + "reward": 0.15018796920776367, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.15018796920776367, + "step": 307 + }, + { + "clip_ratio": 0.0, + "completion_length": 11.1875, + "epoch": 0.24015594541910332, + "grad_norm": 0.39492674037455167, + "kl": 3.05078125, + "learning_rate": 9.475308982494667e-07, + "loss": 0.1221, + "reward": 0.15859133005142212, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.15859133005142212, + "step": 308 + }, + { + "clip_ratio": 0.0, + "completion_length": 13.354166984558105, + "epoch": 0.2409356725146199, + "grad_norm": 6.062163696431718, + "kl": 2.921875, + "learning_rate": 9.469548437853491e-07, + "loss": 0.2749, + "reward": 0.14673330634832382, + "reward_std": 0.00951569527387619, + "rewards/accuracy_reward": 0.14673330634832382, + "step": 309 + }, + { + "clip_ratio": 0.0, + "completion_length": 12.020833492279053, + "epoch": 0.24171539961013644, + "grad_norm": 11.790569655461159, + "kl": 4.0234375, + "learning_rate": 9.463758423088002e-07, + "loss": 0.2291, + "reward": 0.1294768750667572, + "reward_std": 0.007609860273078084, + "rewards/accuracy_reward": 0.1294768750667572, + "step": 310 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.166666984558105, + "epoch": 0.242495126705653, + "grad_norm": 5.684964061783532, + "kl": 3.21875, + "learning_rate": 9.457938981183556e-07, + "loss": 0.0548, + "reward": 0.16567786782979965, + "reward_std": 0.011467931792140007, + "rewards/accuracy_reward": 0.16567786782979965, + "step": 311 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.375, + "epoch": 0.2432748538011696, + "grad_norm": 4.733368822179079, + "kl": 3.5390625, + "learning_rate": 9.452090155343978e-07, + "loss": 0.1685, + "reward": 0.1623220592737198, + "reward_std": 0.01105591095983982, + "rewards/accuracy_reward": 0.1623220592737198, + "step": 312 + }, + { + "clip_ratio": 0.0, + "completion_length": 15.208333492279053, + "epoch": 0.24405458089668616, + "grad_norm": 10.099511461886369, + "kl": 2.13671875, + "learning_rate": 9.446211988991239e-07, + "loss": 0.0399, + "reward": 0.16994038969278336, + "reward_std": 0.005950515856966376, + "rewards/accuracy_reward": 0.16994038969278336, + "step": 313 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.145833492279053, + "epoch": 0.24483430799220274, + "grad_norm": 2.7921139224320517, + "kl": 3.59375, + "learning_rate": 9.440304525765139e-07, + "loss": 0.1239, + "reward": 0.1476425603032112, + "reward_std": 0.0005697530577890575, + "rewards/accuracy_reward": 0.1476425603032112, + "step": 314 + }, + { + "clip_ratio": 0.0, + "completion_length": 11.875, + "epoch": 0.24561403508771928, + "grad_norm": 2.555359893692319, + "kl": 2.953125, + "learning_rate": 9.434367809522978e-07, + "loss": 0.1581, + "reward": 0.14604929089546204, + "reward_std": 0.003157075960189104, + "rewards/accuracy_reward": 0.14604929089546204, + "step": 315 + }, + { + "clip_ratio": 0.0, + "completion_length": 13.645833492279053, + "epoch": 0.24639376218323586, + "grad_norm": 6.368718435772907, + "kl": 2.7265625, + "learning_rate": 9.428401884339231e-07, + "loss": 0.0229, + "reward": 0.1592159867286682, + "reward_std": 0.01695099240168929, + "rewards/accuracy_reward": 0.1592159867286682, + "step": 316 + }, + { + "clip_ratio": 0.0, + "completion_length": 15.770833969116211, + "epoch": 0.24717348927875243, + "grad_norm": 4.946457453057061, + "kl": 2.6640625, + "learning_rate": 9.422406794505221e-07, + "loss": 0.1924, + "reward": 0.16760166734457016, + "reward_std": 0.011069273576140404, + "rewards/accuracy_reward": 0.16760166734457016, + "step": 317 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.520833492279053, + "epoch": 0.247953216374269, + "grad_norm": 1.8874857401256442, + "kl": 3.671875, + "learning_rate": 9.416382584528794e-07, + "loss": 0.1572, + "reward": 0.15528500080108643, + "reward_std": 0.0030617068987339735, + "rewards/accuracy_reward": 0.15528500080108643, + "step": 318 + }, + { + "clip_ratio": 0.0, + "completion_length": 11.708333492279053, + "epoch": 0.24873294346978558, + "grad_norm": 1.8136733529034799, + "kl": 2.859375, + "learning_rate": 9.410329299133982e-07, + "loss": 0.1069, + "reward": 0.14370537549257278, + "reward_std": 0.003252904862165451, + "rewards/accuracy_reward": 0.14370537549257278, + "step": 319 + }, + { + "clip_ratio": 0.0, + "completion_length": 13.166666984558105, + "epoch": 0.24951267056530213, + "grad_norm": 2.6927140379913945, + "kl": 2.8515625, + "learning_rate": 9.404246983260677e-07, + "loss": 0.1004, + "reward": 0.1607775092124939, + "reward_std": 0.004481068812310696, + "rewards/accuracy_reward": 0.1607775092124939, + "step": 320 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.083333730697632, + "epoch": 0.25029239766081873, + "grad_norm": 4.340242300695142, + "kl": 3.140625, + "learning_rate": 9.398135682064292e-07, + "loss": 0.1486, + "reward": 0.15591206401586533, + "reward_std": 0.001132059609517455, + "rewards/accuracy_reward": 0.15591206401586533, + "step": 321 + }, + { + "clip_ratio": 0.0, + "completion_length": 14.250000476837158, + "epoch": 0.2510721247563353, + "grad_norm": 3.9537646142295215, + "kl": 3.2578125, + "learning_rate": 9.391995440915431e-07, + "loss": 0.104, + "reward": 0.14638651907444, + "reward_std": 0.009758115746080875, + "rewards/accuracy_reward": 0.14638651907444, + "step": 322 + }, + { + "clip_ratio": 0.0, + "completion_length": 13.104167461395264, + "epoch": 0.2518518518518518, + "grad_norm": 2.4833154466196956, + "kl": 2.8359375, + "learning_rate": 9.385826305399549e-07, + "loss": 0.1415, + "reward": 0.16239789128303528, + "reward_std": 0.0011320605408400297, + "rewards/accuracy_reward": 0.16239789128303528, + "step": 323 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.125, + "epoch": 0.25263157894736843, + "grad_norm": 6.95972183622307, + "kl": 4.4765625, + "learning_rate": 9.379628321316612e-07, + "loss": 0.3666, + "reward": 0.14668265730142593, + "reward_std": 0.01086803199723363, + "rewards/accuracy_reward": 0.14668265730142593, + "step": 324 + }, + { + "clip_ratio": 0.0, + "completion_length": 11.541666984558105, + "epoch": 0.253411306042885, + "grad_norm": 0.4617833333827291, + "kl": 2.84375, + "learning_rate": 9.373401534680764e-07, + "loss": 0.1075, + "reward": 0.15859133750200272, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.15859133750200272, + "step": 325 + }, + { + "clip_ratio": 0.0, + "completion_length": 17.20833396911621, + "epoch": 0.2541910331384016, + "grad_norm": 1.8132498916325728, + "kl": 2.20703125, + "learning_rate": 9.367145991719973e-07, + "loss": 0.0625, + "reward": 0.17597854137420654, + "reward_std": 0.00710753258317709, + "rewards/accuracy_reward": 0.17597854137420654, + "step": 326 + }, + { + "clip_ratio": 0.0, + "completion_length": 14.395833969116211, + "epoch": 0.2549707602339181, + "grad_norm": 2.977715983331237, + "kl": 2.8359375, + "learning_rate": 9.360861738875704e-07, + "loss": 0.1902, + "reward": 0.1683361977338791, + "reward_std": 0.007850188296288252, + "rewards/accuracy_reward": 0.1683361977338791, + "step": 327 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.125000238418579, + "epoch": 0.2557504873294347, + "grad_norm": 3.452349275804109, + "kl": 3.75, + "learning_rate": 9.354548822802559e-07, + "loss": 0.1766, + "reward": 0.13416200131177902, + "reward_std": 0.005377282854169607, + "rewards/accuracy_reward": 0.13416200131177902, + "step": 328 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.250000476837158, + "epoch": 0.2565302144249513, + "grad_norm": 4.67086414848647, + "kl": 3.9453125, + "learning_rate": 9.348207290367941e-07, + "loss": 0.3177, + "reward": 0.1363159865140915, + "reward_std": 0.0035982707049697638, + "rewards/accuracy_reward": 0.1363159865140915, + "step": 329 + }, + { + "clip_ratio": 0.0, + "completion_length": 11.625, + "epoch": 0.2573099415204678, + "grad_norm": 4.299597889128723, + "kl": 3.4140625, + "learning_rate": 9.341837188651703e-07, + "loss": 0.2106, + "reward": 0.15561115741729736, + "reward_std": 0.002914828830398619, + "rewards/accuracy_reward": 0.15561115741729736, + "step": 330 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.729166984558105, + "epoch": 0.2580896686159844, + "grad_norm": 2.140848393044737, + "kl": 2.9765625, + "learning_rate": 9.335438564945791e-07, + "loss": 0.0593, + "reward": 0.16010206937789917, + "reward_std": 0.0031131631694734097, + "rewards/accuracy_reward": 0.16010206937789917, + "step": 331 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.583333730697632, + "epoch": 0.25886939571150097, + "grad_norm": 1.78145936593844, + "kl": 3.4765625, + "learning_rate": 9.329011466753908e-07, + "loss": 0.1375, + "reward": 0.14783550053834915, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.14783550053834915, + "step": 332 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.833333492279053, + "epoch": 0.2596491228070175, + "grad_norm": 9.249352111457235, + "kl": 4.421875, + "learning_rate": 9.322555941791149e-07, + "loss": 0.2628, + "reward": 0.14634396135807037, + "reward_std": 0.0032874359749257565, + "rewards/accuracy_reward": 0.14634396135807037, + "step": 333 + }, + { + "clip_ratio": 0.0, + "completion_length": 11.020833730697632, + "epoch": 0.2604288499025341, + "grad_norm": 6.269717979673763, + "kl": 3.1171875, + "learning_rate": 9.316072037983649e-07, + "loss": 0.2294, + "reward": 0.16324052959680557, + "reward_std": 0.008229773491621017, + "rewards/accuracy_reward": 0.16324052959680557, + "step": 334 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.187500238418579, + "epoch": 0.26120857699805067, + "grad_norm": 1.75105088193478, + "kl": 3.9375, + "learning_rate": 9.309559803468235e-07, + "loss": 0.1671, + "reward": 0.14475446939468384, + "reward_std": 0.005799277685582638, + "rewards/accuracy_reward": 0.14475446939468384, + "step": 335 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.500000476837158, + "epoch": 0.26198830409356727, + "grad_norm": 4.172643295553986, + "kl": 3.2578125, + "learning_rate": 9.303019286592057e-07, + "loss": 0.1823, + "reward": 0.15000522881746292, + "reward_std": 0.005518701858818531, + "rewards/accuracy_reward": 0.15000522881746292, + "step": 336 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.9791669845581055, + "epoch": 0.2627680311890838, + "grad_norm": 0.22902570185654725, + "kl": 3.3828125, + "learning_rate": 9.296450535912236e-07, + "loss": 0.1334, + "reward": 0.14925595372915268, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.14925595372915268, + "step": 337 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.354166984558105, + "epoch": 0.26354775828460036, + "grad_norm": 0.49800815087424305, + "kl": 3.796875, + "learning_rate": 9.289853600195501e-07, + "loss": 0.1461, + "reward": 0.1659226268529892, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.1659226268529892, + "step": 338 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.000000238418579, + "epoch": 0.26432748538011697, + "grad_norm": 0.20112105370298947, + "kl": 3.46875, + "learning_rate": 9.283228528417832e-07, + "loss": 0.1385, + "reward": 0.13571429997682571, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.13571429997682571, + "step": 339 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.041666984558105, + "epoch": 0.2651072124756335, + "grad_norm": 0.5183928331293166, + "kl": 3.15625, + "learning_rate": 9.276575369764091e-07, + "loss": 0.122, + "reward": 0.14783550798892975, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.14783550798892975, + "step": 340 + }, + { + "clip_ratio": 0.0, + "completion_length": 5.958333492279053, + "epoch": 0.2658869395711501, + "grad_norm": 0.12535914977379958, + "kl": 3.6484375, + "learning_rate": 9.269894173627656e-07, + "loss": 0.1461, + "reward": 0.13571429997682571, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.13571429997682571, + "step": 341 + }, + { + "clip_ratio": 0.0, + "completion_length": 5.0, + "epoch": 0.26666666666666666, + "grad_norm": 0.11410982381062941, + "kl": 4.2578125, + "learning_rate": 9.263184989610058e-07, + "loss": 0.17, + "reward": 0.15000001341104507, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.15000001341104507, + "step": 342 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.8125, + "epoch": 0.2674463937621832, + "grad_norm": 2.8133131091506596, + "kl": 3.5859375, + "learning_rate": 9.256447867520612e-07, + "loss": 0.2071, + "reward": 0.14612681418657303, + "reward_std": 0.001217866549268365, + "rewards/accuracy_reward": 0.14612681418657303, + "step": 343 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.812500476837158, + "epoch": 0.2682261208576998, + "grad_norm": 0.48566940184021756, + "kl": 3.2265625, + "learning_rate": 9.249682857376044e-07, + "loss": 0.1237, + "reward": 0.16228070855140686, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.16228070855140686, + "step": 344 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.458333492279053, + "epoch": 0.26900584795321636, + "grad_norm": 4.965130657921159, + "kl": 4.1796875, + "learning_rate": 9.24289000940013e-07, + "loss": 0.2625, + "reward": 0.13326352089643478, + "reward_std": 0.0013975434703752398, + "rewards/accuracy_reward": 0.13326352089643478, + "step": 345 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.104166984558105, + "epoch": 0.26978557504873296, + "grad_norm": 5.914165711549631, + "kl": 3.796875, + "learning_rate": 9.236069374023304e-07, + "loss": 0.2648, + "reward": 0.16691932827234268, + "reward_std": 0.003851958317682147, + "rewards/accuracy_reward": 0.16691932827234268, + "step": 346 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.250000238418579, + "epoch": 0.2705653021442495, + "grad_norm": 6.377356066175931, + "kl": 4.84375, + "learning_rate": 9.229221001882307e-07, + "loss": 0.3321, + "reward": 0.1592262014746666, + "reward_std": 0.0021980637684464455, + "rewards/accuracy_reward": 0.1592262014746666, + "step": 347 + }, + { + "clip_ratio": 0.0, + "completion_length": 12.791666984558105, + "epoch": 0.27134502923976606, + "grad_norm": 3.883496879866113, + "kl": 3.3515625, + "learning_rate": 9.222344943819788e-07, + "loss": 0.1282, + "reward": 0.1351918950676918, + "reward_std": 0.006383164552971721, + "rewards/accuracy_reward": 0.1351918950676918, + "step": 348 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.5, + "epoch": 0.27212475633528266, + "grad_norm": 0.19387684774557348, + "kl": 3.5, + "learning_rate": 9.215441250883944e-07, + "loss": 0.138, + "reward": 0.1418546363711357, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.1418546363711357, + "step": 349 + }, + { + "clip_ratio": 0.0, + "completion_length": 12.791666984558105, + "epoch": 0.2729044834307992, + "grad_norm": 2.6934537197535136, + "kl": 3.1328125, + "learning_rate": 9.208509974328136e-07, + "loss": 0.1473, + "reward": 0.1584201380610466, + "reward_std": 0.011034275405108929, + "rewards/accuracy_reward": 0.1584201380610466, + "step": 350 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.770833730697632, + "epoch": 0.2736842105263158, + "grad_norm": 14.909744743109016, + "kl": 4.359375, + "learning_rate": 9.201551165610502e-07, + "loss": 0.3776, + "reward": 0.15026964992284775, + "reward_std": 0.010787321021780372, + "rewards/accuracy_reward": 0.15026964992284775, + "step": 351 + }, + { + "clip_ratio": 0.0, + "completion_length": 11.125000476837158, + "epoch": 0.27446393762183235, + "grad_norm": 4.346522515113751, + "kl": 3.5546875, + "learning_rate": 9.194564876393583e-07, + "loss": 0.1823, + "reward": 0.16232681274414062, + "reward_std": 0.009733288548886776, + "rewards/accuracy_reward": 0.16232681274414062, + "step": 352 + }, + { + "clip_ratio": 0.0, + "completion_length": 12.541666984558105, + "epoch": 0.2752436647173489, + "grad_norm": 3.9139850815442623, + "kl": 3.2890625, + "learning_rate": 9.187551158543938e-07, + "loss": 0.2982, + "reward": 0.15423484146595, + "reward_std": 0.004482646938413382, + "rewards/accuracy_reward": 0.15423484146595, + "step": 353 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.6041669845581055, + "epoch": 0.2760233918128655, + "grad_norm": 0.4098093213063153, + "kl": 3.6484375, + "learning_rate": 9.180510064131755e-07, + "loss": 0.1423, + "reward": 0.1478070318698883, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.1478070318698883, + "step": 354 + }, + { + "clip_ratio": 0.0, + "completion_length": 14.020833969116211, + "epoch": 0.27680311890838205, + "grad_norm": 0.33872616889624524, + "kl": 3.015625, + "learning_rate": 9.173441645430467e-07, + "loss": 0.1126, + "reward": 0.17127691954374313, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.17127691954374313, + "step": 355 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.500000476837158, + "epoch": 0.27758284600389865, + "grad_norm": 13.25726618671934, + "kl": 4.1875, + "learning_rate": 9.166345954916366e-07, + "loss": 0.1409, + "reward": 0.14254386723041534, + "reward_std": 0.003798356279730797, + "rewards/accuracy_reward": 0.14254386723041534, + "step": 356 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.895833492279053, + "epoch": 0.2783625730994152, + "grad_norm": 4.181253217697556, + "kl": 3.9453125, + "learning_rate": 9.159223045268208e-07, + "loss": 0.2647, + "reward": 0.16294778883457184, + "reward_std": 0.002057300880551338, + "rewards/accuracy_reward": 0.16294778883457184, + "step": 357 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.791666984558105, + "epoch": 0.27914230019493175, + "grad_norm": 4.5724541162695544, + "kl": 3.6328125, + "learning_rate": 9.152072969366831e-07, + "loss": 0.2806, + "reward": 0.15755286812782288, + "reward_std": 0.0021742512471973896, + "rewards/accuracy_reward": 0.15755286812782288, + "step": 358 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.979166984558105, + "epoch": 0.27992202729044835, + "grad_norm": 3.7584675453246024, + "kl": 3.4375, + "learning_rate": 9.144895780294748e-07, + "loss": 0.1788, + "reward": 0.14665570110082626, + "reward_std": 0.0017377210315316916, + "rewards/accuracy_reward": 0.14665570110082626, + "step": 359 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.416666984558105, + "epoch": 0.2807017543859649, + "grad_norm": 5.963708036467045, + "kl": 3.8359375, + "learning_rate": 9.13769153133577e-07, + "loss": 0.2013, + "reward": 0.14726510643959045, + "reward_std": 0.0023518577218055725, + "rewards/accuracy_reward": 0.14726510643959045, + "step": 360 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.395833492279053, + "epoch": 0.2814814814814815, + "grad_norm": 10.09384850614911, + "kl": 4.40625, + "learning_rate": 9.130460275974596e-07, + "loss": 0.244, + "reward": 0.14828869700431824, + "reward_std": 0.011598555836826563, + "rewards/accuracy_reward": 0.14828869700431824, + "step": 361 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.729166746139526, + "epoch": 0.28226120857699805, + "grad_norm": 2.9379058535221914, + "kl": 3.875, + "learning_rate": 9.123202067896427e-07, + "loss": 0.2484, + "reward": 0.1378328800201416, + "reward_std": 0.0030768176075071096, + "rewards/accuracy_reward": 0.1378328800201416, + "step": 362 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.166666984558105, + "epoch": 0.2830409356725146, + "grad_norm": 0.3950653711421381, + "kl": 3.5859375, + "learning_rate": 9.115916960986556e-07, + "loss": 0.1284, + "reward": 0.16692467033863068, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.16692467033863068, + "step": 363 + }, + { + "clip_ratio": 0.0, + "completion_length": 13.145833492279053, + "epoch": 0.2838206627680312, + "grad_norm": 3.6595124366360494, + "kl": 2.859375, + "learning_rate": 9.10860500932998e-07, + "loss": 0.1911, + "reward": 0.1389584057033062, + "reward_std": 0.0046866354532539845, + "rewards/accuracy_reward": 0.1389584057033062, + "step": 364 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.729166984558105, + "epoch": 0.28460038986354774, + "grad_norm": 2.884781169318846, + "kl": 3.7421875, + "learning_rate": 9.10126626721099e-07, + "loss": 0.2106, + "reward": 0.14237084984779358, + "reward_std": 0.00047169136814773083, + "rewards/accuracy_reward": 0.14237084984779358, + "step": 365 + }, + { + "clip_ratio": 0.0, + "completion_length": 14.395833492279053, + "epoch": 0.28538011695906434, + "grad_norm": 2.7804319305305816, + "kl": 2.5859375, + "learning_rate": 9.09390078911277e-07, + "loss": 0.2129, + "reward": 0.17890625447034836, + "reward_std": 0.00270632840692997, + "rewards/accuracy_reward": 0.17890625447034836, + "step": 366 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.375, + "epoch": 0.2861598440545809, + "grad_norm": 0.38207941828337255, + "kl": 3.6796875, + "learning_rate": 9.086508629716997e-07, + "loss": 0.143, + "reward": 0.15378788113594055, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.15378788113594055, + "step": 367 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.083333492279053, + "epoch": 0.28693957115009744, + "grad_norm": 3.723354909225953, + "kl": 3.875, + "learning_rate": 9.079089843903425e-07, + "loss": 0.2504, + "reward": 0.13127674162387848, + "reward_std": 0.00120510661508888, + "rewards/accuracy_reward": 0.13127674162387848, + "step": 368 + }, + { + "clip_ratio": 0.0, + "completion_length": 11.291666984558105, + "epoch": 0.28771929824561404, + "grad_norm": 1.3526253622053168, + "kl": 2.984375, + "learning_rate": 9.071644486749493e-07, + "loss": 0.1132, + "reward": 0.1708245947957039, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.1708245947957039, + "step": 369 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.041666984558105, + "epoch": 0.2884990253411306, + "grad_norm": 5.737068939101509, + "kl": 3.90625, + "learning_rate": 9.064172613529895e-07, + "loss": 0.1531, + "reward": 0.16008080542087555, + "reward_std": 0.007817985489964485, + "rewards/accuracy_reward": 0.16008080542087555, + "step": 370 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.125, + "epoch": 0.2892787524366472, + "grad_norm": 2.813923046261043, + "kl": 3.421875, + "learning_rate": 9.056674279716192e-07, + "loss": 0.1596, + "reward": 0.14751984924077988, + "reward_std": 0.001093466067686677, + "rewards/accuracy_reward": 0.14751984924077988, + "step": 371 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.833333492279053, + "epoch": 0.29005847953216374, + "grad_norm": 8.069589843107199, + "kl": 3.734375, + "learning_rate": 9.049149540976384e-07, + "loss": 0.1032, + "reward": 0.15217722207307816, + "reward_std": 0.005698587745428085, + "rewards/accuracy_reward": 0.15217722207307816, + "step": 372 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.250000238418579, + "epoch": 0.2908382066276803, + "grad_norm": 15.197853361150488, + "kl": 4.84375, + "learning_rate": 9.041598453174506e-07, + "loss": 0.261, + "reward": 0.13517910242080688, + "reward_std": 0.002728496619965881, + "rewards/accuracy_reward": 0.13517910242080688, + "step": 373 + }, + { + "clip_ratio": 0.0, + "completion_length": 5.812500238418579, + "epoch": 0.2916179337231969, + "grad_norm": 0.31898871624716346, + "kl": 3.890625, + "learning_rate": 9.034021072370207e-07, + "loss": 0.1557, + "reward": 0.14404762536287308, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.14404762536287308, + "step": 374 + }, + { + "clip_ratio": 0.0, + "completion_length": 5.75, + "epoch": 0.29239766081871343, + "grad_norm": 0.2822671515922015, + "kl": 3.859375, + "learning_rate": 9.026417454818338e-07, + "loss": 0.1543, + "reward": 0.1488095372915268, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.1488095372915268, + "step": 375 + }, + { + "clip_ratio": 0.0, + "completion_length": 11.645833730697632, + "epoch": 0.29317738791423004, + "grad_norm": 14.389676610167912, + "kl": 3.3046875, + "learning_rate": 9.018787656968533e-07, + "loss": 0.034, + "reward": 0.16039849817752838, + "reward_std": 0.009118756745010614, + "rewards/accuracy_reward": 0.16039849817752838, + "step": 376 + }, + { + "clip_ratio": 0.0, + "completion_length": 5.812500238418579, + "epoch": 0.2939571150097466, + "grad_norm": 0.22846529022108444, + "kl": 3.8203125, + "learning_rate": 9.011131735464787e-07, + "loss": 0.1519, + "reward": 0.14166668057441711, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.14166668057441711, + "step": 377 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.208333730697632, + "epoch": 0.29473684210526313, + "grad_norm": 2.11397301902399, + "kl": 3.703125, + "learning_rate": 9.003449747145043e-07, + "loss": 0.1282, + "reward": 0.15699300915002823, + "reward_std": 0.0020187075715512037, + "rewards/accuracy_reward": 0.15699300915002823, + "step": 378 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.166666746139526, + "epoch": 0.29551656920077973, + "grad_norm": 2.9055223019600143, + "kl": 3.2578125, + "learning_rate": 8.995741749040764e-07, + "loss": 0.1319, + "reward": 0.15838926285505295, + "reward_std": 0.0066304802894592285, + "rewards/accuracy_reward": 0.15838926285505295, + "step": 379 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.770833492279053, + "epoch": 0.2962962962962963, + "grad_norm": 5.40273553068801, + "kl": 3.9296875, + "learning_rate": 8.988007798376504e-07, + "loss": 0.1897, + "reward": 0.147501602768898, + "reward_std": 0.0076894816011190414, + "rewards/accuracy_reward": 0.147501602768898, + "step": 380 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.541666746139526, + "epoch": 0.2970760233918129, + "grad_norm": 4.315360565948567, + "kl": 3.8125, + "learning_rate": 8.980247952569504e-07, + "loss": 0.1129, + "reward": 0.1444043293595314, + "reward_std": 0.003221820807084441, + "rewards/accuracy_reward": 0.1444043293595314, + "step": 381 + }, + { + "clip_ratio": 0.0, + "completion_length": 12.916666984558105, + "epoch": 0.29785575048732943, + "grad_norm": 1.9824329610853104, + "kl": 2.5234375, + "learning_rate": 8.97246226922924e-07, + "loss": 0.0495, + "reward": 0.16020382195711136, + "reward_std": 0.0029046256095170975, + "rewards/accuracy_reward": 0.16020382195711136, + "step": 382 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.916666984558105, + "epoch": 0.298635477582846, + "grad_norm": 3.682796593140469, + "kl": 3.8515625, + "learning_rate": 8.964650806157008e-07, + "loss": 0.1929, + "reward": 0.17954282462596893, + "reward_std": 0.009522270411252975, + "rewards/accuracy_reward": 0.17954282462596893, + "step": 383 + }, + { + "clip_ratio": 0.0, + "completion_length": 5.625, + "epoch": 0.2994152046783626, + "grad_norm": 8.80835120764374, + "kl": 4.375, + "learning_rate": 8.956813621345501e-07, + "loss": 0.3315, + "reward": 0.15191799402236938, + "reward_std": 0.001603750861249864, + "rewards/accuracy_reward": 0.15191799402236938, + "step": 384 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.0, + "epoch": 0.3001949317738791, + "grad_norm": 10.604700425523651, + "kl": 4.546875, + "learning_rate": 8.948950772978367e-07, + "loss": 0.292, + "reward": 0.14568452537059784, + "reward_std": 0.002577458042651415, + "rewards/accuracy_reward": 0.14568452537059784, + "step": 385 + }, + { + "clip_ratio": 0.0, + "completion_length": 17.083333492279053, + "epoch": 0.30097465886939573, + "grad_norm": 3.636862344464563, + "kl": 2.8671875, + "learning_rate": 8.941062319429781e-07, + "loss": 0.1311, + "reward": 0.15528112649917603, + "reward_std": 0.01720402017235756, + "rewards/accuracy_reward": 0.15528112649917603, + "step": 386 + }, + { + "clip_ratio": 0.0, + "completion_length": 5.270833492279053, + "epoch": 0.3017543859649123, + "grad_norm": 0.28736744287416843, + "kl": 4.1640625, + "learning_rate": 8.933148319264013e-07, + "loss": 0.1665, + "reward": 0.15000001341104507, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.15000001341104507, + "step": 387 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.166666984558105, + "epoch": 0.3025341130604289, + "grad_norm": 4.808257587397571, + "kl": 3.4609375, + "learning_rate": 8.925208831234993e-07, + "loss": 0.1747, + "reward": 0.14375001192092896, + "reward_std": 0.005904720164835453, + "rewards/accuracy_reward": 0.14375001192092896, + "step": 388 + }, + { + "clip_ratio": 0.0, + "completion_length": 5.8125, + "epoch": 0.3033138401559454, + "grad_norm": 0.2676486555342594, + "kl": 3.8828125, + "learning_rate": 8.917243914285873e-07, + "loss": 0.1549, + "reward": 0.14642857760190964, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.14642857760190964, + "step": 389 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.4375, + "epoch": 0.30409356725146197, + "grad_norm": 1.7504264799915286, + "kl": 3.3671875, + "learning_rate": 8.909253627548591e-07, + "loss": 0.1268, + "reward": 0.15033482760190964, + "reward_std": 0.004510548897087574, + "rewards/accuracy_reward": 0.15033482760190964, + "step": 390 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.854166984558105, + "epoch": 0.3048732943469786, + "grad_norm": 5.492992611635677, + "kl": 3.3984375, + "learning_rate": 8.901238030343432e-07, + "loss": 0.268, + "reward": 0.15573524683713913, + "reward_std": 0.0051187395583838224, + "rewards/accuracy_reward": 0.15573524683713913, + "step": 391 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.958333492279053, + "epoch": 0.3056530214424951, + "grad_norm": 3.33193497315929, + "kl": 3.2734375, + "learning_rate": 8.893197182178587e-07, + "loss": 0.1966, + "reward": 0.14758404344320297, + "reward_std": 0.003648071549832821, + "rewards/accuracy_reward": 0.14758404344320297, + "step": 392 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.020833492279053, + "epoch": 0.3064327485380117, + "grad_norm": 0.25274717008869885, + "kl": 3.6953125, + "learning_rate": 8.885131142749712e-07, + "loss": 0.1452, + "reward": 0.16316527873277664, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.16316527873277664, + "step": 393 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.9166669845581055, + "epoch": 0.30721247563352827, + "grad_norm": 6.4668385222053075, + "kl": 4.234375, + "learning_rate": 8.877039971939481e-07, + "loss": 0.1728, + "reward": 0.1547979861497879, + "reward_std": 0.009622504934668541, + "rewards/accuracy_reward": 0.1547979861497879, + "step": 394 + }, + { + "clip_ratio": 0.0, + "completion_length": 13.041666984558105, + "epoch": 0.3079922027290448, + "grad_norm": 0.2118585348560572, + "kl": 2.6640625, + "learning_rate": 8.868923729817149e-07, + "loss": 0.1048, + "reward": 0.17120098322629929, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.17120098322629929, + "step": 395 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.291666746139526, + "epoch": 0.3087719298245614, + "grad_norm": 0.2801637016272516, + "kl": 3.515625, + "learning_rate": 8.860782476638101e-07, + "loss": 0.1398, + "reward": 0.1454545557498932, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.1454545557498932, + "step": 396 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.145833492279053, + "epoch": 0.30955165692007797, + "grad_norm": 0.3351542044371962, + "kl": 3.65625, + "learning_rate": 8.852616272843402e-07, + "loss": 0.1405, + "reward": 0.16457219421863556, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.16457219421863556, + "step": 397 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.604166746139526, + "epoch": 0.31033138401559457, + "grad_norm": 2.323477297979642, + "kl": 3.390625, + "learning_rate": 8.844425179059353e-07, + "loss": 0.1841, + "reward": 0.15334384888410568, + "reward_std": 0.0021980637684464455, + "rewards/accuracy_reward": 0.15334384888410568, + "step": 398 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.958333969116211, + "epoch": 0.3111111111111111, + "grad_norm": 3.3294560113411316, + "kl": 3.390625, + "learning_rate": 8.83620925609704e-07, + "loss": 0.2522, + "reward": 0.13990402966737747, + "reward_std": 0.0066372910514473915, + "rewards/accuracy_reward": 0.13990402966737747, + "step": 399 + }, + { + "clip_ratio": 0.0, + "completion_length": 14.937500476837158, + "epoch": 0.31189083820662766, + "grad_norm": 0.23825880365650365, + "kl": 2.484375, + "learning_rate": 8.827968564951886e-07, + "loss": 0.0983, + "reward": 0.16851955652236938, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.16851955652236938, + "step": 400 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.145833730697632, + "epoch": 0.31267056530214427, + "grad_norm": 2.280556383897255, + "kl": 3.671875, + "learning_rate": 8.819703166803186e-07, + "loss": 0.1782, + "reward": 0.15582707524299622, + "reward_std": 0.0010852443519979715, + "rewards/accuracy_reward": 0.15582707524299622, + "step": 401 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.458333492279053, + "epoch": 0.3134502923976608, + "grad_norm": 5.134190258006971, + "kl": 4.28125, + "learning_rate": 8.811413123013668e-07, + "loss": 0.2236, + "reward": 0.1684027910232544, + "reward_std": 0.012028131633996964, + "rewards/accuracy_reward": 0.1684027910232544, + "step": 402 + }, + { + "clip_ratio": 0.0, + "completion_length": 11.437500476837158, + "epoch": 0.3142300194931774, + "grad_norm": 2.9231531072288197, + "kl": 3.3125, + "learning_rate": 8.803098495129028e-07, + "loss": 0.1271, + "reward": 0.1722072809934616, + "reward_std": 0.0018650222336873412, + "rewards/accuracy_reward": 0.1722072809934616, + "step": 403 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.645833969116211, + "epoch": 0.31500974658869396, + "grad_norm": 2.8193977710210634, + "kl": 3.609375, + "learning_rate": 8.794759344877474e-07, + "loss": 0.1158, + "reward": 0.15898126363754272, + "reward_std": 0.0017131322529166937, + "rewards/accuracy_reward": 0.15898126363754272, + "step": 404 + }, + { + "clip_ratio": 0.0, + "completion_length": 5.729166746139526, + "epoch": 0.3157894736842105, + "grad_norm": 0.33673892958420754, + "kl": 4.0078125, + "learning_rate": 8.78639573416927e-07, + "loss": 0.1579, + "reward": 0.15000000596046448, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.15000000596046448, + "step": 405 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.979166984558105, + "epoch": 0.3165692007797271, + "grad_norm": 0.61116109473038, + "kl": 2.9375, + "learning_rate": 8.778007725096278e-07, + "loss": 0.1147, + "reward": 0.16097228229045868, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.16097228229045868, + "step": 406 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.0625, + "epoch": 0.31734892787524366, + "grad_norm": 0.44873850562449036, + "kl": 3.5703125, + "learning_rate": 8.769595379931495e-07, + "loss": 0.1391, + "reward": 0.15758929401636124, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.15758929401636124, + "step": 407 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.125000476837158, + "epoch": 0.31812865497076026, + "grad_norm": 5.617503977168476, + "kl": 3.59375, + "learning_rate": 8.761158761128584e-07, + "loss": 0.1939, + "reward": 0.14398247003555298, + "reward_std": 0.006505153025500476, + "rewards/accuracy_reward": 0.14398247003555298, + "step": 408 + }, + { + "clip_ratio": 0.0, + "completion_length": 11.666666984558105, + "epoch": 0.3189083820662768, + "grad_norm": 148.9446002531759, + "kl": 10.8125, + "learning_rate": 8.752697931321425e-07, + "loss": 0.5094, + "reward": 0.13543393835425377, + "reward_std": 0.01484832726418972, + "rewards/accuracy_reward": 0.13543393835425377, + "step": 409 + }, + { + "clip_ratio": 0.0, + "completion_length": 5.833333492279053, + "epoch": 0.31968810916179335, + "grad_norm": 13.060620810867475, + "kl": 4.578125, + "learning_rate": 8.744212953323634e-07, + "loss": 0.3419, + "reward": 0.14298941940069199, + "reward_std": 0.011913577443920076, + "rewards/accuracy_reward": 0.14298941940069199, + "step": 410 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.520833492279053, + "epoch": 0.32046783625730996, + "grad_norm": 0.37751306006917723, + "kl": 3.1328125, + "learning_rate": 8.735703890128112e-07, + "loss": 0.1244, + "reward": 0.16097228229045868, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.16097228229045868, + "step": 411 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.666666984558105, + "epoch": 0.3212475633528265, + "grad_norm": 7.635514043910877, + "kl": 3.703125, + "learning_rate": 8.727170804906562e-07, + "loss": 0.1901, + "reward": 0.15975462645292282, + "reward_std": 0.0006014077225700021, + "rewards/accuracy_reward": 0.15975462645292282, + "step": 412 + }, + { + "clip_ratio": 0.0, + "completion_length": 11.270833730697632, + "epoch": 0.3220272904483431, + "grad_norm": 8.49376242180647, + "kl": 3.0859375, + "learning_rate": 8.718613761009035e-07, + "loss": 0.2474, + "reward": 0.14480237662792206, + "reward_std": 0.02359803323633969, + "rewards/accuracy_reward": 0.14480237662792206, + "step": 413 + }, + { + "clip_ratio": 0.0, + "completion_length": 13.583333492279053, + "epoch": 0.32280701754385965, + "grad_norm": 3.3498673152295737, + "kl": 2.36328125, + "learning_rate": 8.710032821963446e-07, + "loss": 0.1346, + "reward": 0.16509104520082474, + "reward_std": 0.010309826582670212, + "rewards/accuracy_reward": 0.16509104520082474, + "step": 414 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.770833492279053, + "epoch": 0.3235867446393762, + "grad_norm": 0.5470724132997554, + "kl": 2.765625, + "learning_rate": 8.701428051475114e-07, + "loss": 0.11, + "reward": 0.15037594735622406, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.15037594735622406, + "step": 415 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.708333492279053, + "epoch": 0.3243664717348928, + "grad_norm": 4.524361798588892, + "kl": 3.4375, + "learning_rate": 8.692799513426282e-07, + "loss": 0.1458, + "reward": 0.1558714210987091, + "reward_std": 0.005950893857516348, + "rewards/accuracy_reward": 0.1558714210987091, + "step": 416 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.437500238418579, + "epoch": 0.32514619883040935, + "grad_norm": 0.4281626061488469, + "kl": 3.5234375, + "learning_rate": 8.68414727187564e-07, + "loss": 0.1386, + "reward": 0.1478070169687271, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.1478070169687271, + "step": 417 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.708333730697632, + "epoch": 0.32592592592592595, + "grad_norm": 0.3815308151814316, + "kl": 3.3046875, + "learning_rate": 8.675471391057862e-07, + "loss": 0.1301, + "reward": 0.16971050202846527, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.16971050202846527, + "step": 418 + }, + { + "clip_ratio": 0.0, + "completion_length": 13.604166746139526, + "epoch": 0.3267056530214425, + "grad_norm": 2.4913359083785487, + "kl": 3.26953125, + "learning_rate": 8.666771935383116e-07, + "loss": 0.3131, + "reward": 0.14368873089551926, + "reward_std": 0.008988404646515846, + "rewards/accuracy_reward": 0.14368873089551926, + "step": 419 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.291666746139526, + "epoch": 0.32748538011695905, + "grad_norm": 5.547399468303284, + "kl": 3.40625, + "learning_rate": 8.658048969436592e-07, + "loss": 0.162, + "reward": 0.14590337872505188, + "reward_std": 0.010309826582670212, + "rewards/accuracy_reward": 0.14590337872505188, + "step": 420 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.625000476837158, + "epoch": 0.32826510721247565, + "grad_norm": 0.3215622915459458, + "kl": 3.2578125, + "learning_rate": 8.649302557978022e-07, + "loss": 0.121, + "reward": 0.16692467033863068, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.16692467033863068, + "step": 421 + }, + { + "clip_ratio": 0.0, + "completion_length": 12.395833492279053, + "epoch": 0.3290448343079922, + "grad_norm": 7.027335178237231, + "kl": 3.0, + "learning_rate": 8.640532765941195e-07, + "loss": 0.1462, + "reward": 0.16403965651988983, + "reward_std": 0.010524614714086056, + "rewards/accuracy_reward": 0.16403965651988983, + "step": 422 + }, + { + "clip_ratio": 0.0, + "completion_length": 12.354166984558105, + "epoch": 0.3298245614035088, + "grad_norm": 4.179235033753008, + "kl": 3.296875, + "learning_rate": 8.631739658433485e-07, + "loss": 0.1947, + "reward": 0.181765578687191, + "reward_std": 0.005102843977510929, + "rewards/accuracy_reward": 0.181765578687191, + "step": 423 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.7291669845581055, + "epoch": 0.33060428849902534, + "grad_norm": 2.285625994483756, + "kl": 4.40625, + "learning_rate": 8.622923300735358e-07, + "loss": 0.1663, + "reward": 0.14741533994674683, + "reward_std": 0.008490445092320442, + "rewards/accuracy_reward": 0.14741533994674683, + "step": 424 + }, + { + "clip_ratio": 0.0, + "completion_length": 14.208333969116211, + "epoch": 0.3313840155945419, + "grad_norm": 3.689726399174867, + "kl": 2.5703125, + "learning_rate": 8.614083758299891e-07, + "loss": 0.0867, + "reward": 0.17276040464639664, + "reward_std": 0.008812322281301022, + "rewards/accuracy_reward": 0.17276040464639664, + "step": 425 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.25, + "epoch": 0.3321637426900585, + "grad_norm": 0.43087054871204195, + "kl": 3.5703125, + "learning_rate": 8.605221096752281e-07, + "loss": 0.1397, + "reward": 0.15018796920776367, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.15018796920776367, + "step": 426 + }, + { + "clip_ratio": 0.0, + "completion_length": 11.666666984558105, + "epoch": 0.33294346978557504, + "grad_norm": 3.0318116030178044, + "kl": 3.109375, + "learning_rate": 8.596335381889369e-07, + "loss": 0.1307, + "reward": 0.17109375447034836, + "reward_std": 0.015056601725518703, + "rewards/accuracy_reward": 0.17109375447034836, + "step": 427 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.041666984558105, + "epoch": 0.33372319688109164, + "grad_norm": 5.6166536824761915, + "kl": 3.6640625, + "learning_rate": 8.587426679679144e-07, + "loss": 0.228, + "reward": 0.13671675696969032, + "reward_std": 0.013925573555752635, + "rewards/accuracy_reward": 0.13671675696969032, + "step": 428 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.625000238418579, + "epoch": 0.3345029239766082, + "grad_norm": 4.669338580758022, + "kl": 3.609375, + "learning_rate": 8.57849505626025e-07, + "loss": 0.1877, + "reward": 0.1486913487315178, + "reward_std": 0.011565987952053547, + "rewards/accuracy_reward": 0.1486913487315178, + "step": 429 + }, + { + "clip_ratio": 0.0, + "completion_length": 11.270833492279053, + "epoch": 0.33528265107212474, + "grad_norm": 4.710731228467101, + "kl": 3.28125, + "learning_rate": 8.569540577941504e-07, + "loss": 0.1441, + "reward": 0.14722733944654465, + "reward_std": 0.009623454534448683, + "rewards/accuracy_reward": 0.14722733944654465, + "step": 430 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.604166746139526, + "epoch": 0.33606237816764134, + "grad_norm": 2.922842872301241, + "kl": 3.328125, + "learning_rate": 8.560563311201394e-07, + "loss": 0.1658, + "reward": 0.15690506249666214, + "reward_std": 0.000991815933957696, + "rewards/accuracy_reward": 0.15690506249666214, + "step": 431 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.479166984558105, + "epoch": 0.3368421052631579, + "grad_norm": 5.40669886700069, + "kl": 3.3828125, + "learning_rate": 8.551563322687596e-07, + "loss": 0.5143, + "reward": 0.1448487639427185, + "reward_std": 0.009350221138447523, + "rewards/accuracy_reward": 0.1448487639427185, + "step": 432 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.333333969116211, + "epoch": 0.3376218323586745, + "grad_norm": 6.955284672632261, + "kl": 3.890625, + "learning_rate": 8.542540679216471e-07, + "loss": 0.2239, + "reward": 0.17085595428943634, + "reward_std": 0.004075850360095501, + "rewards/accuracy_reward": 0.17085595428943634, + "step": 433 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.937500238418579, + "epoch": 0.33840155945419104, + "grad_norm": 0.4577562610711916, + "kl": 3.8359375, + "learning_rate": 8.533495447772567e-07, + "loss": 0.1481, + "reward": 0.16450217366218567, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.16450217366218567, + "step": 434 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.750000476837158, + "epoch": 0.3391812865497076, + "grad_norm": 135.6382012209044, + "kl": 7.328125, + "learning_rate": 8.524427695508136e-07, + "loss": 0.4871, + "reward": 0.15703052282333374, + "reward_std": 0.003380400245077908, + "rewards/accuracy_reward": 0.15703052282333374, + "step": 435 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.270833730697632, + "epoch": 0.3399610136452242, + "grad_norm": 3.2506215294238103, + "kl": 3.90625, + "learning_rate": 8.515337489742618e-07, + "loss": 0.2412, + "reward": 0.17852745950222015, + "reward_std": 0.0009021099540404975, + "rewards/accuracy_reward": 0.17852745950222015, + "step": 436 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.395833492279053, + "epoch": 0.34074074074074073, + "grad_norm": 4.37038860934232, + "kl": 4.421875, + "learning_rate": 8.50622489796215e-07, + "loss": 0.1736, + "reward": 0.1708763688802719, + "reward_std": 0.011707212310284376, + "rewards/accuracy_reward": 0.1708763688802719, + "step": 437 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.645833492279053, + "epoch": 0.34152046783625734, + "grad_norm": 15.208494503746422, + "kl": 4.015625, + "learning_rate": 8.497089987819064e-07, + "loss": 0.1959, + "reward": 0.1393013894557953, + "reward_std": 0.011867815104778856, + "rewards/accuracy_reward": 0.1393013894557953, + "step": 438 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.958333492279053, + "epoch": 0.3423001949317739, + "grad_norm": 2.500487241111051, + "kl": 4.359375, + "learning_rate": 8.487932827131389e-07, + "loss": 0.1615, + "reward": 0.17685356736183167, + "reward_std": 0.0007959796348586679, + "rewards/accuracy_reward": 0.17685356736183167, + "step": 439 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.520833492279053, + "epoch": 0.34307992202729043, + "grad_norm": 10.947441423527012, + "kl": 4.328125, + "learning_rate": 8.478753483882337e-07, + "loss": 0.1412, + "reward": 0.15975943952798843, + "reward_std": 0.019432506524026394, + "rewards/accuracy_reward": 0.15975943952798843, + "step": 440 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.833333492279053, + "epoch": 0.34385964912280703, + "grad_norm": 5.608765808188827, + "kl": 3.859375, + "learning_rate": 8.46955202621981e-07, + "loss": 0.2917, + "reward": 0.1515037640929222, + "reward_std": 0.0030386867001652718, + "rewards/accuracy_reward": 0.1515037640929222, + "step": 441 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.104166984558105, + "epoch": 0.3446393762183236, + "grad_norm": 5.376429802411146, + "kl": 3.8671875, + "learning_rate": 8.460328522455882e-07, + "loss": 0.1342, + "reward": 0.1472402662038803, + "reward_std": 0.010309826582670212, + "rewards/accuracy_reward": 0.1472402662038803, + "step": 442 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.75, + "epoch": 0.3454191033138402, + "grad_norm": 0.7375507125862572, + "kl": 3.8203125, + "learning_rate": 8.451083041066302e-07, + "loss": 0.1534, + "reward": 0.15899620950222015, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.15899620950222015, + "step": 443 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.291666746139526, + "epoch": 0.34619883040935673, + "grad_norm": 2.58514884483921, + "kl": 3.9609375, + "learning_rate": 8.441815650689987e-07, + "loss": 0.1441, + "reward": 0.16193846613168716, + "reward_std": 0.0006330604082904756, + "rewards/accuracy_reward": 0.16193846613168716, + "step": 444 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.333333492279053, + "epoch": 0.3469785575048733, + "grad_norm": 3.865142309488586, + "kl": 4.171875, + "learning_rate": 8.432526420128497e-07, + "loss": 0.1664, + "reward": 0.16354405879974365, + "reward_std": 0.017936360090970993, + "rewards/accuracy_reward": 0.16354405879974365, + "step": 445 + }, + { + "clip_ratio": 0.0, + "completion_length": 5.562500238418579, + "epoch": 0.3477582846003899, + "grad_norm": 0.5520638259162657, + "kl": 3.875, + "learning_rate": 8.423215418345544e-07, + "loss": 0.1542, + "reward": 0.14404762536287308, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.14404762536287308, + "step": 446 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.291666746139526, + "epoch": 0.3485380116959064, + "grad_norm": 6.287980237166874, + "kl": 4.046875, + "learning_rate": 8.413882714466463e-07, + "loss": 0.2107, + "reward": 0.1572420671582222, + "reward_std": 0.012028131633996964, + "rewards/accuracy_reward": 0.1572420671582222, + "step": 447 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.458333730697632, + "epoch": 0.349317738791423, + "grad_norm": 4.546388481488884, + "kl": 4.703125, + "learning_rate": 8.404528377777714e-07, + "loss": 0.2971, + "reward": 0.18159721791744232, + "reward_std": 0.016218727454543114, + "rewards/accuracy_reward": 0.18159721791744232, + "step": 448 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.354166984558105, + "epoch": 0.3500974658869396, + "grad_norm": 3.3143977506986015, + "kl": 3.4375, + "learning_rate": 8.395152477726354e-07, + "loss": 0.1129, + "reward": 0.15585318207740784, + "reward_std": 0.001093466067686677, + "rewards/accuracy_reward": 0.15585318207740784, + "step": 449 + }, + { + "clip_ratio": 0.0, + "completion_length": 11.3125, + "epoch": 0.3508771929824561, + "grad_norm": 0.3608819023159808, + "kl": 3.28125, + "learning_rate": 8.38575508391953e-07, + "loss": 0.1247, + "reward": 0.16003789007663727, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.16003789007663727, + "step": 450 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.125000238418579, + "epoch": 0.3516569200779727, + "grad_norm": 0.6240970965730207, + "kl": 3.546875, + "learning_rate": 8.376336266123955e-07, + "loss": 0.1415, + "reward": 0.13809525966644287, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.13809525966644287, + "step": 451 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.020833492279053, + "epoch": 0.35243664717348927, + "grad_norm": 3.8262236713205184, + "kl": 3.53125, + "learning_rate": 8.366896094265407e-07, + "loss": 0.189, + "reward": 0.1437252163887024, + "reward_std": 0.0021478808484971523, + "rewards/accuracy_reward": 0.1437252163887024, + "step": 452 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.604166746139526, + "epoch": 0.3532163742690059, + "grad_norm": 4.693590266385996, + "kl": 4.2109375, + "learning_rate": 8.357434638428183e-07, + "loss": 0.312, + "reward": 0.15640433132648468, + "reward_std": 0.006682294886559248, + "rewards/accuracy_reward": 0.15640433132648468, + "step": 453 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.1666669845581055, + "epoch": 0.3539961013645224, + "grad_norm": 16.412568739508906, + "kl": 5.234375, + "learning_rate": 8.347951968854605e-07, + "loss": 0.293, + "reward": 0.15729168057441711, + "reward_std": 0.021650638431310654, + "rewards/accuracy_reward": 0.15729168057441711, + "step": 454 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.270833492279053, + "epoch": 0.35477582846003897, + "grad_norm": 2.852199857180022, + "kl": 3.890625, + "learning_rate": 8.338448155944479e-07, + "loss": 0.1697, + "reward": 0.17543309926986694, + "reward_std": 0.0007959796348586679, + "rewards/accuracy_reward": 0.17543309926986694, + "step": 455 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.208333730697632, + "epoch": 0.35555555555555557, + "grad_norm": 2.5125320869917895, + "kl": 3.796875, + "learning_rate": 8.328923270254584e-07, + "loss": 0.1117, + "reward": 0.15251225978136063, + "reward_std": 0.0009050846565514803, + "rewards/accuracy_reward": 0.15251225978136063, + "step": 456 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.541666746139526, + "epoch": 0.3563352826510721, + "grad_norm": 5.639644968710531, + "kl": 3.9921875, + "learning_rate": 8.319377382498143e-07, + "loss": 0.4313, + "reward": 0.16532737761735916, + "reward_std": 0.010309826582670212, + "rewards/accuracy_reward": 0.16532737761735916, + "step": 457 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.645833492279053, + "epoch": 0.3571150097465887, + "grad_norm": 9.579854633126313, + "kl": 4.140625, + "learning_rate": 8.309810563544297e-07, + "loss": 0.1893, + "reward": 0.14931797236204147, + "reward_std": 0.03522123023867607, + "rewards/accuracy_reward": 0.14931797236204147, + "step": 458 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.125000238418579, + "epoch": 0.35789473684210527, + "grad_norm": 3.684533841377317, + "kl": 3.7421875, + "learning_rate": 8.300222884417591e-07, + "loss": 0.278, + "reward": 0.15312793850898743, + "reward_std": 0.005660296883434057, + "rewards/accuracy_reward": 0.15312793850898743, + "step": 459 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.6875, + "epoch": 0.3586744639376218, + "grad_norm": 6.299174771605969, + "kl": 4.078125, + "learning_rate": 8.290614416297422e-07, + "loss": 0.2222, + "reward": 0.14708946645259857, + "reward_std": 0.0018490657676011324, + "rewards/accuracy_reward": 0.14708946645259857, + "step": 460 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.583333492279053, + "epoch": 0.3594541910331384, + "grad_norm": 6.59923965919033, + "kl": 3.59375, + "learning_rate": 8.280985230517537e-07, + "loss": 0.2232, + "reward": 0.14995040744543076, + "reward_std": 0.021541289053857327, + "rewards/accuracy_reward": 0.14995040744543076, + "step": 461 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.3541669845581055, + "epoch": 0.36023391812865496, + "grad_norm": 0.5795825418946184, + "kl": 4.1328125, + "learning_rate": 8.271335398565493e-07, + "loss": 0.1657, + "reward": 0.16904763132333755, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.16904763132333755, + "step": 462 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.104166746139526, + "epoch": 0.36101364522417156, + "grad_norm": 5.49279777276861, + "kl": 4.234375, + "learning_rate": 8.261664992082112e-07, + "loss": 0.1771, + "reward": 0.16036707162857056, + "reward_std": 0.01297498308122158, + "rewards/accuracy_reward": 0.16036707162857056, + "step": 463 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.166666746139526, + "epoch": 0.3617933723196881, + "grad_norm": 2.9309597239317005, + "kl": 3.34375, + "learning_rate": 8.251974082860976e-07, + "loss": 0.141, + "reward": 0.17147818207740784, + "reward_std": 0.0047181760892271996, + "rewards/accuracy_reward": 0.17147818207740784, + "step": 464 + }, + { + "clip_ratio": 0.0, + "completion_length": 5.375000238418579, + "epoch": 0.36257309941520466, + "grad_norm": 8.422336053830872, + "kl": 3.8046875, + "learning_rate": 8.242262742847874e-07, + "loss": 0.1985, + "reward": 0.1394001916050911, + "reward_std": 0.016099190339446068, + "rewards/accuracy_reward": 0.1394001916050911, + "step": 465 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.958333492279053, + "epoch": 0.36335282651072126, + "grad_norm": 1.5131045081758705, + "kl": 4.2265625, + "learning_rate": 8.232531044140276e-07, + "loss": 0.1676, + "reward": 0.17425595223903656, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.17425595223903656, + "step": 466 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.416666746139526, + "epoch": 0.3641325536062378, + "grad_norm": 10.85177258251265, + "kl": 4.8203125, + "learning_rate": 8.222779058986797e-07, + "loss": 0.2036, + "reward": 0.1517857238650322, + "reward_std": 0.010309826582670212, + "rewards/accuracy_reward": 0.1517857238650322, + "step": 467 + }, + { + "clip_ratio": 0.0, + "completion_length": 5.854166746139526, + "epoch": 0.3649122807017544, + "grad_norm": 7.651454012467134, + "kl": 3.5078125, + "learning_rate": 8.213006859786654e-07, + "loss": 0.2309, + "reward": 0.14295635372400284, + "reward_std": 0.012028131633996964, + "rewards/accuracy_reward": 0.14295635372400284, + "step": 468 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.083333492279053, + "epoch": 0.36569200779727096, + "grad_norm": 3.810188977813238, + "kl": 4.640625, + "learning_rate": 8.203214519089142e-07, + "loss": 0.3917, + "reward": 0.16564542800188065, + "reward_std": 0.003537686076015234, + "rewards/accuracy_reward": 0.16564542800188065, + "step": 469 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.5, + "epoch": 0.3664717348927875, + "grad_norm": 0.7025200767559716, + "kl": 3.8828125, + "learning_rate": 8.193402109593081e-07, + "loss": 0.1516, + "reward": 0.14791666716337204, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.14791666716337204, + "step": 470 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.250000238418579, + "epoch": 0.3672514619883041, + "grad_norm": 7.1769899639721775, + "kl": 4.0, + "learning_rate": 8.183569704146289e-07, + "loss": 0.3655, + "reward": 0.14375001192092896, + "reward_std": 0.021650636568665504, + "rewards/accuracy_reward": 0.14375001192092896, + "step": 471 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.520833492279053, + "epoch": 0.36803118908382065, + "grad_norm": 0.5211054823348614, + "kl": 3.9609375, + "learning_rate": 8.173717375745026e-07, + "loss": 0.1568, + "reward": 0.14925595372915268, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.14925595372915268, + "step": 472 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.291666746139526, + "epoch": 0.36881091617933726, + "grad_norm": 3.6466347866363082, + "kl": 4.21875, + "learning_rate": 8.163845197533471e-07, + "loss": 0.1828, + "reward": 0.16442213207483292, + "reward_std": 0.0051978714764118195, + "rewards/accuracy_reward": 0.16442213207483292, + "step": 473 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.208333492279053, + "epoch": 0.3695906432748538, + "grad_norm": 5.619622481899318, + "kl": 3.9453125, + "learning_rate": 8.15395324280316e-07, + "loss": 0.1728, + "reward": 0.14863887429237366, + "reward_std": 0.012499823234975338, + "rewards/accuracy_reward": 0.14863887429237366, + "step": 474 + }, + { + "clip_ratio": 0.0, + "completion_length": 5.791666746139526, + "epoch": 0.37037037037037035, + "grad_norm": 6.443185239282619, + "kl": 4.5, + "learning_rate": 8.144041584992458e-07, + "loss": 0.1488, + "reward": 0.15070685744285583, + "reward_std": 0.02384147245902568, + "rewards/accuracy_reward": 0.15070685744285583, + "step": 475 + }, + { + "clip_ratio": 0.0, + "completion_length": 5.333333492279053, + "epoch": 0.37115009746588695, + "grad_norm": 0.4963476320879215, + "kl": 3.765625, + "learning_rate": 8.134110297686005e-07, + "loss": 0.1503, + "reward": 0.14404763281345367, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.14404763281345367, + "step": 476 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.937500238418579, + "epoch": 0.3719298245614035, + "grad_norm": 3.881383212104769, + "kl": 3.484375, + "learning_rate": 8.124159454614167e-07, + "loss": 0.1355, + "reward": 0.14419206976890564, + "reward_std": 0.0005306530511006713, + "rewards/accuracy_reward": 0.14419206976890564, + "step": 477 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.437500476837158, + "epoch": 0.3727095516569201, + "grad_norm": 0.4735702922949389, + "kl": 3.5234375, + "learning_rate": 8.114189129652498e-07, + "loss": 0.14, + "reward": 0.16233766078948975, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.16233766078948975, + "step": 478 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.312500476837158, + "epoch": 0.37348927875243665, + "grad_norm": 0.5317182633715427, + "kl": 3.5625, + "learning_rate": 8.104199396821183e-07, + "loss": 0.1415, + "reward": 0.14783550053834915, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.14783550053834915, + "step": 479 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.895833492279053, + "epoch": 0.3742690058479532, + "grad_norm": 3.5235062812971973, + "kl": 5.03125, + "learning_rate": 8.094190330284497e-07, + "loss": 0.2354, + "reward": 0.17164523154497147, + "reward_std": 0.0007959796348586679, + "rewards/accuracy_reward": 0.17164523154497147, + "step": 480 + }, + { + "clip_ratio": 0.0, + "completion_length": 5.458333492279053, + "epoch": 0.3750487329434698, + "grad_norm": 0.5615752188944567, + "kl": 4.015625, + "learning_rate": 8.084162004350239e-07, + "loss": 0.1592, + "reward": 0.160714291036129, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.160714291036129, + "step": 481 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.416666746139526, + "epoch": 0.37582846003898635, + "grad_norm": 6.646761619006107, + "kl": 3.8125, + "learning_rate": 8.074114493469205e-07, + "loss": 0.1806, + "reward": 0.15319941192865372, + "reward_std": 0.005412658676505089, + "rewards/accuracy_reward": 0.15319941192865372, + "step": 482 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.104166746139526, + "epoch": 0.37660818713450295, + "grad_norm": 3.960634104851548, + "kl": 3.84375, + "learning_rate": 8.064047872234611e-07, + "loss": 0.1371, + "reward": 0.15019404888153076, + "reward_std": 0.010153157985769212, + "rewards/accuracy_reward": 0.15019404888153076, + "step": 483 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.145833492279053, + "epoch": 0.3773879142300195, + "grad_norm": 0.4723750019516396, + "kl": 4.5625, + "learning_rate": 8.053962215381551e-07, + "loss": 0.1804, + "reward": 0.18399621546268463, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.18399621546268463, + "step": 484 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.583333492279053, + "epoch": 0.37816764132553604, + "grad_norm": 3.7740670405849204, + "kl": 4.296875, + "learning_rate": 8.043857597786442e-07, + "loss": 0.1442, + "reward": 0.17500000447034836, + "reward_std": 0.013121597468852997, + "rewards/accuracy_reward": 0.17500000447034836, + "step": 485 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.562500238418579, + "epoch": 0.37894736842105264, + "grad_norm": 0.29289778279495476, + "kl": 3.703125, + "learning_rate": 8.033734094466466e-07, + "loss": 0.1486, + "reward": 0.1454545557498932, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.1454545557498932, + "step": 486 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.312500238418579, + "epoch": 0.3797270955165692, + "grad_norm": 3.492405176530902, + "kl": 3.65625, + "learning_rate": 8.023591780579013e-07, + "loss": 0.2074, + "reward": 0.1638144999742508, + "reward_std": 0.0023821950890123844, + "rewards/accuracy_reward": 0.1638144999742508, + "step": 487 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.187500238418579, + "epoch": 0.3805068226120858, + "grad_norm": 2.099727811298138, + "kl": 4.1640625, + "learning_rate": 8.013430731421125e-07, + "loss": 0.1786, + "reward": 0.16051588207483292, + "reward_std": 0.000687321531586349, + "rewards/accuracy_reward": 0.16051588207483292, + "step": 488 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.145833492279053, + "epoch": 0.38128654970760234, + "grad_norm": 0.37614889520365113, + "kl": 3.515625, + "learning_rate": 8.003251022428932e-07, + "loss": 0.1406, + "reward": 0.1454545557498932, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.1454545557498932, + "step": 489 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.166666746139526, + "epoch": 0.3820662768031189, + "grad_norm": 3.2490831993692377, + "kl": 3.859375, + "learning_rate": 7.993052729177101e-07, + "loss": 0.1166, + "reward": 0.14361871033906937, + "reward_std": 0.008988404646515846, + "rewards/accuracy_reward": 0.14361871033906937, + "step": 490 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.083333492279053, + "epoch": 0.3828460038986355, + "grad_norm": 4.144005050441794, + "kl": 3.84375, + "learning_rate": 7.982835927378264e-07, + "loss": 0.1758, + "reward": 0.17722538113594055, + "reward_std": 0.005412658676505089, + "rewards/accuracy_reward": 0.17722538113594055, + "step": 491 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.354166984558105, + "epoch": 0.38362573099415204, + "grad_norm": 0.8134852287394632, + "kl": 4.328125, + "learning_rate": 7.972600692882467e-07, + "loss": 0.1695, + "reward": 0.16458334028720856, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.16458334028720856, + "step": 492 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.479166746139526, + "epoch": 0.38440545808966864, + "grad_norm": 8.111287295998025, + "kl": 3.734375, + "learning_rate": 7.962347101676593e-07, + "loss": 0.4144, + "reward": 0.14890874922275543, + "reward_std": 0.012028131633996964, + "rewards/accuracy_reward": 0.14890874922275543, + "step": 493 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.770833492279053, + "epoch": 0.3851851851851852, + "grad_norm": 7.427994817914454, + "kl": 4.1875, + "learning_rate": 7.952075229883816e-07, + "loss": 0.4443, + "reward": 0.15552400052547455, + "reward_std": 0.016583421966060996, + "rewards/accuracy_reward": 0.15552400052547455, + "step": 494 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.625000238418579, + "epoch": 0.38596491228070173, + "grad_norm": 6.887978718238284, + "kl": 4.15625, + "learning_rate": 7.941785153763017e-07, + "loss": 0.3331, + "reward": 0.1592758148908615, + "reward_std": 0.0024056262336671352, + "rewards/accuracy_reward": 0.1592758148908615, + "step": 495 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.958333492279053, + "epoch": 0.38674463937621834, + "grad_norm": 6.2963934999274365, + "kl": 4.15625, + "learning_rate": 7.931476949708237e-07, + "loss": 0.2142, + "reward": 0.16041813790798187, + "reward_std": 0.013024244457483292, + "rewards/accuracy_reward": 0.16041813790798187, + "step": 496 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.979166984558105, + "epoch": 0.3875243664717349, + "grad_norm": 9.269295451472685, + "kl": 4.421875, + "learning_rate": 7.921150694248087e-07, + "loss": 0.3698, + "reward": 0.1551213338971138, + "reward_std": 0.02220164705067873, + "rewards/accuracy_reward": 0.1551213338971138, + "step": 497 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.479166746139526, + "epoch": 0.3883040935672515, + "grad_norm": 41.599638759436765, + "kl": 5.4765625, + "learning_rate": 7.910806464045205e-07, + "loss": 0.3987, + "reward": 0.12738095968961716, + "reward_std": 0.015075568109750748, + "rewards/accuracy_reward": 0.12738095968961716, + "step": 498 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.270833492279053, + "epoch": 0.38908382066276803, + "grad_norm": 0.4715560234291706, + "kl": 3.046875, + "learning_rate": 7.900444335895664e-07, + "loss": 0.1183, + "reward": 0.13601191341876984, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.13601191341876984, + "step": 499 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.145833492279053, + "epoch": 0.3898635477582846, + "grad_norm": 0.45938131017775613, + "kl": 3.296875, + "learning_rate": 7.89006438672842e-07, + "loss": 0.1291, + "reward": 0.15854978561401367, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.15854978561401367, + "step": 500 + }, + { + "clip_ratio": 0.0, + "completion_length": 11.562500476837158, + "epoch": 0.3906432748538012, + "grad_norm": 1.832127531221603, + "kl": 3.2109375, + "learning_rate": 7.87966669360473e-07, + "loss": 0.0843, + "reward": 0.15031566470861435, + "reward_std": 0.0048112524673342705, + "rewards/accuracy_reward": 0.15031566470861435, + "step": 501 + }, + { + "clip_ratio": 0.0, + "completion_length": 50.500001430511475, + "epoch": 0.39142300194931773, + "grad_norm": 19.32139593047235, + "kl": 4.0078125, + "learning_rate": 7.86925133371758e-07, + "loss": 0.636, + "reward": 0.13084637373685837, + "reward_std": 0.025604712776839733, + "rewards/accuracy_reward": 0.13084637373685837, + "step": 502 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.4375, + "epoch": 0.39220272904483433, + "grad_norm": 4.128289333676487, + "kl": 3.9765625, + "learning_rate": 7.858818384391121e-07, + "loss": 0.3373, + "reward": 0.15272794663906097, + "reward_std": 0.010852573439478874, + "rewards/accuracy_reward": 0.15272794663906097, + "step": 503 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.270833492279053, + "epoch": 0.3929824561403509, + "grad_norm": 6.89689577293208, + "kl": 3.84375, + "learning_rate": 7.848367923080083e-07, + "loss": 0.3489, + "reward": 0.14126984775066376, + "reward_std": 0.009622504934668541, + "rewards/accuracy_reward": 0.14126984775066376, + "step": 504 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.145833730697632, + "epoch": 0.3937621832358674, + "grad_norm": 10.45347074303406, + "kl": 3.671875, + "learning_rate": 7.837900027369213e-07, + "loss": 0.5478, + "reward": 0.1450827717781067, + "reward_std": 0.021204533986747265, + "rewards/accuracy_reward": 0.1450827717781067, + "step": 505 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.875, + "epoch": 0.394541910331384, + "grad_norm": 13.539747324408372, + "kl": 4.109375, + "learning_rate": 7.827414774972684e-07, + "loss": 0.3884, + "reward": 0.1531454399228096, + "reward_std": 0.025450320914387703, + "rewards/accuracy_reward": 0.1531454399228096, + "step": 506 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.270833492279053, + "epoch": 0.3953216374269006, + "grad_norm": 4.85728328238381, + "kl": 4.0859375, + "learning_rate": 7.816912243733529e-07, + "loss": 0.2003, + "reward": 0.15173611044883728, + "reward_std": 0.012028131633996964, + "rewards/accuracy_reward": 0.15173611044883728, + "step": 507 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.708333492279053, + "epoch": 0.3961013645224172, + "grad_norm": 10.250122131032365, + "kl": 3.5390625, + "learning_rate": 7.806392511623063e-07, + "loss": 0.2273, + "reward": 0.14918291568756104, + "reward_std": 0.0038615207886323333, + "rewards/accuracy_reward": 0.14918291568756104, + "step": 508 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.229166746139526, + "epoch": 0.3968810916179337, + "grad_norm": 0.5825117318442314, + "kl": 3.9453125, + "learning_rate": 7.795855656740297e-07, + "loss": 0.1568, + "reward": 0.16904762387275696, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.16904762387275696, + "step": 509 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.395833492279053, + "epoch": 0.39766081871345027, + "grad_norm": 3.9864577401633303, + "kl": 3.7578125, + "learning_rate": 7.785301757311361e-07, + "loss": 0.1404, + "reward": 0.15848101675510406, + "reward_std": 0.005111956037580967, + "rewards/accuracy_reward": 0.15848101675510406, + "step": 510 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.291666984558105, + "epoch": 0.3984405458089669, + "grad_norm": 5.844626597048682, + "kl": 4.53125, + "learning_rate": 7.774730891688935e-07, + "loss": 0.4012, + "reward": 0.1799585521221161, + "reward_std": 0.009113149717450142, + "rewards/accuracy_reward": 0.1799585521221161, + "step": 511 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.020833492279053, + "epoch": 0.3992202729044834, + "grad_norm": 0.7290873142092051, + "kl": 4.78125, + "learning_rate": 7.764143138351641e-07, + "loss": 0.1908, + "reward": 0.171875, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.171875, + "step": 512 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.020833492279053, + "epoch": 0.4, + "grad_norm": 0.4909406743346614, + "kl": 3.578125, + "learning_rate": 7.75353857590349e-07, + "loss": 0.1434, + "reward": 0.13333335518836975, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.13333335518836975, + "step": 513 + }, + { + "clip_ratio": 0.0, + "completion_length": 5.520833492279053, + "epoch": 0.40077972709551657, + "grad_norm": 0.5378890006390055, + "kl": 4.578125, + "learning_rate": 7.742917283073276e-07, + "loss": 0.1841, + "reward": 0.16354166716337204, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.16354166716337204, + "step": 514 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.937500238418579, + "epoch": 0.4015594541910331, + "grad_norm": 0.4663833223109688, + "kl": 4.375, + "learning_rate": 7.732279338714004e-07, + "loss": 0.1736, + "reward": 0.17566289007663727, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.17566289007663727, + "step": 515 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.270833492279053, + "epoch": 0.4023391812865497, + "grad_norm": 7.038585808964307, + "kl": 4.515625, + "learning_rate": 7.721624821802301e-07, + "loss": 0.1654, + "reward": 0.1465649977326393, + "reward_std": 0.020147119648754597, + "rewards/accuracy_reward": 0.1465649977326393, + "step": 516 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.062500238418579, + "epoch": 0.40311890838206627, + "grad_norm": 6.319549014406511, + "kl": 3.828125, + "learning_rate": 7.710953811437829e-07, + "loss": 0.1979, + "reward": 0.14126984775066376, + "reward_std": 0.009622504934668541, + "rewards/accuracy_reward": 0.14126984775066376, + "step": 517 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.687500238418579, + "epoch": 0.40389863547758287, + "grad_norm": 14.443799166033836, + "kl": 6.484375, + "learning_rate": 7.700266386842695e-07, + "loss": 0.2401, + "reward": 0.15711000561714172, + "reward_std": 0.03797685354948044, + "rewards/accuracy_reward": 0.15711000561714172, + "step": 518 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.145833492279053, + "epoch": 0.4046783625730994, + "grad_norm": 0.7291630360265167, + "kl": 3.546875, + "learning_rate": 7.689562627360871e-07, + "loss": 0.1409, + "reward": 0.16829004883766174, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.16829004883766174, + "step": 519 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.854166746139526, + "epoch": 0.40545808966861596, + "grad_norm": 2.284150274568634, + "kl": 4.0859375, + "learning_rate": 7.678842612457598e-07, + "loss": 0.1693, + "reward": 0.15869100391864777, + "reward_std": 0.004431416746228933, + "rewards/accuracy_reward": 0.15869100391864777, + "step": 520 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.354166746139526, + "epoch": 0.40623781676413256, + "grad_norm": 22.641069404296434, + "kl": 4.90625, + "learning_rate": 7.668106421718798e-07, + "loss": 0.337, + "reward": 0.15684913843870163, + "reward_std": 0.005141436355188489, + "rewards/accuracy_reward": 0.15684913843870163, + "step": 521 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.166666746139526, + "epoch": 0.4070175438596491, + "grad_norm": 4.529722215791823, + "kl": 3.8359375, + "learning_rate": 7.657354134850481e-07, + "loss": 0.1501, + "reward": 0.13661955296993256, + "reward_std": 0.005111956037580967, + "rewards/accuracy_reward": 0.13661955296993256, + "step": 522 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.020833492279053, + "epoch": 0.4077972709551657, + "grad_norm": 3.5132587713319325, + "kl": 3.078125, + "learning_rate": 7.646585831678159e-07, + "loss": 0.105, + "reward": 0.1413690522313118, + "reward_std": 0.010309826582670212, + "rewards/accuracy_reward": 0.1413690522313118, + "step": 523 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.229166746139526, + "epoch": 0.40857699805068226, + "grad_norm": 6.427493426480892, + "kl": 4.21875, + "learning_rate": 7.635801592146249e-07, + "loss": 0.1411, + "reward": 0.15313854068517685, + "reward_std": 0.011724278330802917, + "rewards/accuracy_reward": 0.15313854068517685, + "step": 524 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.333333492279053, + "epoch": 0.4093567251461988, + "grad_norm": 5.535171825715842, + "kl": 4.953125, + "learning_rate": 7.625001496317478e-07, + "loss": 0.1891, + "reward": 0.15191420912742615, + "reward_std": 0.00865181372500956, + "rewards/accuracy_reward": 0.15191420912742615, + "step": 525 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.6666669845581055, + "epoch": 0.4101364522417154, + "grad_norm": 4.893961672897194, + "kl": 3.71875, + "learning_rate": 7.614185624372293e-07, + "loss": 0.183, + "reward": 0.16566459834575653, + "reward_std": 0.007699065841734409, + "rewards/accuracy_reward": 0.16566459834575653, + "step": 526 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.291666746139526, + "epoch": 0.41091617933723196, + "grad_norm": 0.6185057194613697, + "kl": 4.171875, + "learning_rate": 7.603354056608267e-07, + "loss": 0.1639, + "reward": 0.17425595223903656, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.17425595223903656, + "step": 527 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.520833969116211, + "epoch": 0.41169590643274856, + "grad_norm": 682.0184211954377, + "kl": 13.2734375, + "learning_rate": 7.592506873439491e-07, + "loss": 2.432, + "reward": 0.14337120950222015, + "reward_std": 0.007216879166662693, + "rewards/accuracy_reward": 0.14337120950222015, + "step": 528 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.250000238418579, + "epoch": 0.4124756335282651, + "grad_norm": 3555.1028884450907, + "kl": 181.03125, + "learning_rate": 7.58164415539599e-07, + "loss": 7.6453, + "reward": 0.15654495358467102, + "reward_std": 0.020854391157627106, + "rewards/accuracy_reward": 0.15654495358467102, + "step": 529 + }, + { + "clip_ratio": 0.0, + "completion_length": 5.645833492279053, + "epoch": 0.41325536062378165, + "grad_norm": 0.6055506502939557, + "kl": 4.328125, + "learning_rate": 7.570765983123125e-07, + "loss": 0.1733, + "reward": 0.1666666716337204, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.1666666716337204, + "step": 530 + }, + { + "clip_ratio": 0.0, + "completion_length": 5.666666746139526, + "epoch": 0.41403508771929826, + "grad_norm": 0.690057128263379, + "kl": 4.359375, + "learning_rate": 7.559872437380985e-07, + "loss": 0.1731, + "reward": 0.17500000447034836, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.17500000447034836, + "step": 531 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.687500238418579, + "epoch": 0.4148148148148148, + "grad_norm": 1315.802458090792, + "kl": 61.703125, + "learning_rate": 7.548963599043792e-07, + "loss": 3.0454, + "reward": 0.17348606884479523, + "reward_std": 0.018366033560596406, + "rewards/accuracy_reward": 0.17348606884479523, + "step": 532 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.958333730697632, + "epoch": 0.4155945419103314, + "grad_norm": 7.708914928216294, + "kl": 4.078125, + "learning_rate": 7.538039549099302e-07, + "loss": 0.2319, + "reward": 0.14933712780475616, + "reward_std": 0.016966513358056545, + "rewards/accuracy_reward": 0.14933712780475616, + "step": 533 + }, + { + "clip_ratio": 0.0, + "completion_length": 11.6875, + "epoch": 0.41637426900584795, + "grad_norm": 2.582191201975053, + "kl": 2.7734375, + "learning_rate": 7.527100368648205e-07, + "loss": 0.1156, + "reward": 0.1288398653268814, + "reward_std": 0.001132059609517455, + "rewards/accuracy_reward": 0.1288398653268814, + "step": 534 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.437500238418579, + "epoch": 0.4171539961013645, + "grad_norm": 7.689630134942464, + "kl": 3.5703125, + "learning_rate": 7.516146138903518e-07, + "loss": 0.2095, + "reward": 0.16102994978427887, + "reward_std": 0.012028131633996964, + "rewards/accuracy_reward": 0.16102994978427887, + "step": 535 + }, + { + "clip_ratio": 0.0, + "completion_length": 5.833333492279053, + "epoch": 0.4179337231968811, + "grad_norm": 6.255243670074004, + "kl": 4.25, + "learning_rate": 7.505176941189986e-07, + "loss": 0.2267, + "reward": 0.1492311656475067, + "reward_std": 0.010911233723163605, + "rewards/accuracy_reward": 0.1492311656475067, + "step": 536 + }, + { + "clip_ratio": 0.0, + "completion_length": 5.750000238418579, + "epoch": 0.41871345029239765, + "grad_norm": 4.185175126819574, + "kl": 4.2578125, + "learning_rate": 7.494192856943475e-07, + "loss": 0.1593, + "reward": 0.1481274887919426, + "reward_std": 0.010477647185325623, + "rewards/accuracy_reward": 0.1481274887919426, + "step": 537 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.333333492279053, + "epoch": 0.41949317738791425, + "grad_norm": 6.808007850322068, + "kl": 3.4609375, + "learning_rate": 7.483193967710372e-07, + "loss": 0.3202, + "reward": 0.1556977480649948, + "reward_std": 0.011876894859597087, + "rewards/accuracy_reward": 0.1556977480649948, + "step": 538 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.458333492279053, + "epoch": 0.4202729044834308, + "grad_norm": 86.26158691980832, + "kl": 7.875, + "learning_rate": 7.472180355146975e-07, + "loss": 0.4188, + "reward": 0.14704862236976624, + "reward_std": 0.01655529160052538, + "rewards/accuracy_reward": 0.14704862236976624, + "step": 539 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.5, + "epoch": 0.42105263157894735, + "grad_norm": 4.266988022777769, + "kl": 3.875, + "learning_rate": 7.461152101018888e-07, + "loss": 0.1728, + "reward": 0.14872636646032333, + "reward_std": 0.00679755536839366, + "rewards/accuracy_reward": 0.14872636646032333, + "step": 540 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.125000238418579, + "epoch": 0.42183235867446395, + "grad_norm": 9.790453302000945, + "kl": 4.6796875, + "learning_rate": 7.45010928720042e-07, + "loss": 0.1953, + "reward": 0.13483188301324844, + "reward_std": 0.023676427081227303, + "rewards/accuracy_reward": 0.13483188301324844, + "step": 541 + }, + { + "clip_ratio": 0.0, + "completion_length": 5.75, + "epoch": 0.4226120857699805, + "grad_norm": 0.7638438910268821, + "kl": 3.734375, + "learning_rate": 7.439051995673965e-07, + "loss": 0.1497, + "reward": 0.14404762536287308, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.14404762536287308, + "step": 542 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.3125, + "epoch": 0.4233918128654971, + "grad_norm": 6.076664081585385, + "kl": 3.8359375, + "learning_rate": 7.427980308529404e-07, + "loss": 0.1624, + "reward": 0.15981241315603256, + "reward_std": 0.003203280735760927, + "rewards/accuracy_reward": 0.15981241315603256, + "step": 543 + }, + { + "clip_ratio": 0.0, + "completion_length": 5.8125, + "epoch": 0.42417153996101364, + "grad_norm": 6.954069996234832, + "kl": 3.7265625, + "learning_rate": 7.416894307963492e-07, + "loss": 0.1379, + "reward": 0.14126984775066376, + "reward_std": 0.009622504934668541, + "rewards/accuracy_reward": 0.14126984775066376, + "step": 544 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.708333492279053, + "epoch": 0.4249512670565302, + "grad_norm": 10.959691309789722, + "kl": 3.75, + "learning_rate": 7.405794076279245e-07, + "loss": 0.18, + "reward": 0.1448412761092186, + "reward_std": 0.021244490519165993, + "rewards/accuracy_reward": 0.1448412761092186, + "step": 545 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.208333492279053, + "epoch": 0.4257309941520468, + "grad_norm": 15.575307553434769, + "kl": 4.359375, + "learning_rate": 7.394679695885333e-07, + "loss": 0.1878, + "reward": 0.18217329680919647, + "reward_std": 0.010592571459710598, + "rewards/accuracy_reward": 0.18217329680919647, + "step": 546 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.562500238418579, + "epoch": 0.42651072124756334, + "grad_norm": 8.815329392247659, + "kl": 3.8125, + "learning_rate": 7.38355124929547e-07, + "loss": 0.1348, + "reward": 0.15213586390018463, + "reward_std": 0.007432721555233002, + "rewards/accuracy_reward": 0.15213586390018463, + "step": 547 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.479166746139526, + "epoch": 0.42729044834307994, + "grad_norm": 32.96098064687768, + "kl": 6.4453125, + "learning_rate": 7.372408819127791e-07, + "loss": 0.229, + "reward": 0.1510416641831398, + "reward_std": 0.010309826582670212, + "rewards/accuracy_reward": 0.1510416641831398, + "step": 548 + }, + { + "clip_ratio": 0.0, + "completion_length": 16.333333492279053, + "epoch": 0.4280701754385965, + "grad_norm": 11.840648605106866, + "kl": 3.015625, + "learning_rate": 7.361252488104251e-07, + "loss": 0.2657, + "reward": 0.17088532447814941, + "reward_std": 0.005972900427877903, + "rewards/accuracy_reward": 0.17088532447814941, + "step": 549 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.229166746139526, + "epoch": 0.42884990253411304, + "grad_norm": 6.800456709016594, + "kl": 3.9296875, + "learning_rate": 7.350082339050006e-07, + "loss": 0.1374, + "reward": 0.1436508148908615, + "reward_std": 0.009622504934668541, + "rewards/accuracy_reward": 0.1436508148908615, + "step": 550 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.875000238418579, + "epoch": 0.42962962962962964, + "grad_norm": 8.290262336360817, + "kl": 3.40625, + "learning_rate": 7.338898454892794e-07, + "loss": 0.1172, + "reward": 0.1795128434896469, + "reward_std": 0.003976011881604791, + "rewards/accuracy_reward": 0.1795128434896469, + "step": 551 + }, + { + "clip_ratio": 0.0, + "completion_length": 5.416666746139526, + "epoch": 0.4304093567251462, + "grad_norm": 1.8505175096133353, + "kl": 3.765625, + "learning_rate": 7.327700918662325e-07, + "loss": 0.15, + "reward": 0.1460784375667572, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.1460784375667572, + "step": 552 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.75, + "epoch": 0.4311890838206628, + "grad_norm": 30.438136118565836, + "kl": 4.796875, + "learning_rate": 7.316489813489666e-07, + "loss": 0.4529, + "reward": 0.16495657712221146, + "reward_std": 0.01821788214147091, + "rewards/accuracy_reward": 0.16495657712221146, + "step": 553 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.354166746139526, + "epoch": 0.43196881091617934, + "grad_norm": 9.017816425897559, + "kl": 3.6328125, + "learning_rate": 7.305265222606613e-07, + "loss": 0.1944, + "reward": 0.15818039327859879, + "reward_std": 0.006200214847922325, + "rewards/accuracy_reward": 0.15818039327859879, + "step": 554 + }, + { + "clip_ratio": 0.0, + "completion_length": 5.395833492279053, + "epoch": 0.4327485380116959, + "grad_norm": 106.8464704607081, + "kl": 7.359375, + "learning_rate": 7.294027229345089e-07, + "loss": 0.3112, + "reward": 0.15555556118488312, + "reward_std": 0.009622504934668541, + "rewards/accuracy_reward": 0.15555556118488312, + "step": 555 + }, + { + "clip_ratio": 0.0, + "completion_length": 5.625, + "epoch": 0.4335282651072125, + "grad_norm": 2.5148575588241613, + "kl": 3.3125, + "learning_rate": 7.28277591713651e-07, + "loss": 0.1323, + "reward": 0.1547619178891182, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.1547619178891182, + "step": 556 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.187500238418579, + "epoch": 0.43430799220272903, + "grad_norm": 7.805638355295732, + "kl": 3.4375, + "learning_rate": 7.271511369511177e-07, + "loss": 0.1802, + "reward": 0.16076389700174332, + "reward_std": 0.009622504934668541, + "rewards/accuracy_reward": 0.16076389700174332, + "step": 557 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.166666984558105, + "epoch": 0.43508771929824563, + "grad_norm": 9.47863228334864, + "kl": 2.6015625, + "learning_rate": 7.26023367009765e-07, + "loss": 0.6941, + "reward": 0.14044123888015747, + "reward_std": 0.01778208464384079, + "rewards/accuracy_reward": 0.14044123888015747, + "step": 558 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.125, + "epoch": 0.4358674463937622, + "grad_norm": 8.557694457533575, + "kl": 4.609375, + "learning_rate": 7.248942902622126e-07, + "loss": 0.1994, + "reward": 0.1728670671582222, + "reward_std": 0.0048112524673342705, + "rewards/accuracy_reward": 0.1728670671582222, + "step": 559 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.354166984558105, + "epoch": 0.43664717348927873, + "grad_norm": 71.43433571432669, + "kl": 5.3984375, + "learning_rate": 7.237639150907825e-07, + "loss": 0.509, + "reward": 0.1398187130689621, + "reward_std": 0.007432510843500495, + "rewards/accuracy_reward": 0.1398187130689621, + "step": 560 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.250000238418579, + "epoch": 0.43742690058479533, + "grad_norm": 2.662085684698807, + "kl": 2.7109375, + "learning_rate": 7.226322498874357e-07, + "loss": 0.1075, + "reward": 0.15758929401636124, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.15758929401636124, + "step": 561 + }, + { + "clip_ratio": 0.0, + "completion_length": 13.583333730697632, + "epoch": 0.4382066276803119, + "grad_norm": 10.278524666464056, + "kl": 2.5078125, + "learning_rate": 7.214993030537111e-07, + "loss": 0.2708, + "reward": 0.14937517046928406, + "reward_std": 0.021752064116299152, + "rewards/accuracy_reward": 0.14937517046928406, + "step": 562 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.75, + "epoch": 0.4389863547758285, + "grad_norm": 10.598618560101574, + "kl": 3.015625, + "learning_rate": 7.203650830006619e-07, + "loss": 0.1615, + "reward": 0.17000534385442734, + "reward_std": 0.0064766849391162395, + "rewards/accuracy_reward": 0.17000534385442734, + "step": 563 + }, + { + "clip_ratio": 0.0, + "completion_length": 5.916666746139526, + "epoch": 0.439766081871345, + "grad_norm": 18.706992459057485, + "kl": 3.6875, + "learning_rate": 7.19229598148794e-07, + "loss": 0.1516, + "reward": 0.14305556565523148, + "reward_std": 0.024056261405348778, + "rewards/accuracy_reward": 0.14305556565523148, + "step": 564 + }, + { + "clip_ratio": 0.0, + "completion_length": 5.854166746139526, + "epoch": 0.4405458089668616, + "grad_norm": 42.33722894553616, + "kl": 4.21875, + "learning_rate": 7.180928569280033e-07, + "loss": 0.1635, + "reward": 0.14345239102840424, + "reward_std": 0.010309826582670212, + "rewards/accuracy_reward": 0.14345239102840424, + "step": 565 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.187500238418579, + "epoch": 0.4413255360623782, + "grad_norm": 9.503215781168503, + "kl": 1.68359375, + "learning_rate": 7.169548677775129e-07, + "loss": 0.114, + "reward": 0.14387401193380356, + "reward_std": 0.0006014079554006457, + "rewards/accuracy_reward": 0.14387401193380356, + "step": 566 + }, + { + "clip_ratio": 0.0, + "completion_length": 5.666666746139526, + "epoch": 0.4421052631578947, + "grad_norm": 31.061007770377056, + "kl": 2.984375, + "learning_rate": 7.158156391458107e-07, + "loss": 0.1577, + "reward": 0.14574654400348663, + "reward_std": 0.014734460972249508, + "rewards/accuracy_reward": 0.14574654400348663, + "step": 567 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.75, + "epoch": 0.4428849902534113, + "grad_norm": 12.502539514434291, + "kl": 3.046875, + "learning_rate": 7.146751794905864e-07, + "loss": 0.2006, + "reward": 0.17068368196487427, + "reward_std": 0.004126847488805652, + "rewards/accuracy_reward": 0.17068368196487427, + "step": 568 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.645833492279053, + "epoch": 0.4436647173489279, + "grad_norm": 36.17713421370529, + "kl": 5.390625, + "learning_rate": 7.13533497278669e-07, + "loss": 0.3954, + "reward": 0.17440824210643768, + "reward_std": 0.005313411355018616, + "rewards/accuracy_reward": 0.17440824210643768, + "step": 569 + }, + { + "clip_ratio": 0.0, + "completion_length": 11.437500476837158, + "epoch": 0.4444444444444444, + "grad_norm": 384.8178421954848, + "kl": 24.453125, + "learning_rate": 7.12390600985964e-07, + "loss": 1.1603, + "reward": 0.16798757761716843, + "reward_std": 0.027818378526717424, + "rewards/accuracy_reward": 0.16798757761716843, + "step": 570 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.833333492279053, + "epoch": 0.445224171539961, + "grad_norm": 2.4800641018854104, + "kl": 1.74609375, + "learning_rate": 7.112464990973899e-07, + "loss": 0.0703, + "reward": 0.14196428656578064, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.14196428656578064, + "step": 571 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.520833492279053, + "epoch": 0.44600389863547757, + "grad_norm": 8.462434420586678, + "kl": 2.2421875, + "learning_rate": 7.101012001068159e-07, + "loss": 0.3008, + "reward": 0.1650463044643402, + "reward_std": 0.005613128654658794, + "rewards/accuracy_reward": 0.1650463044643402, + "step": 572 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.770833492279053, + "epoch": 0.44678362573099417, + "grad_norm": 19.51349408685107, + "kl": 4.859375, + "learning_rate": 7.089547125169986e-07, + "loss": 0.1691, + "reward": 0.15353422611951828, + "reward_std": 0.011175260122399777, + "rewards/accuracy_reward": 0.15353422611951828, + "step": 573 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.0416669845581055, + "epoch": 0.4475633528265107, + "grad_norm": 17.078049285990165, + "kl": 5.703125, + "learning_rate": 7.078070448395183e-07, + "loss": 0.1995, + "reward": 0.1815476268529892, + "reward_std": 0.014433758333325386, + "rewards/accuracy_reward": 0.1815476268529892, + "step": 574 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.541666984558105, + "epoch": 0.44834307992202727, + "grad_norm": 46.846754836989255, + "kl": 5.56640625, + "learning_rate": 7.066582055947172e-07, + "loss": 0.216, + "reward": 0.14058562368154526, + "reward_std": 0.004775875713676214, + "rewards/accuracy_reward": 0.14058562368154526, + "step": 575 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.583333492279053, + "epoch": 0.44912280701754387, + "grad_norm": 997.5685150926331, + "kl": 28.6015625, + "learning_rate": 7.055082033116345e-07, + "loss": 2.3898, + "reward": 0.15674273669719696, + "reward_std": 0.0022111828438937664, + "rewards/accuracy_reward": 0.15674273669719696, + "step": 576 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.979166746139526, + "epoch": 0.4499025341130604, + "grad_norm": 20.967147453075277, + "kl": 3.1484375, + "learning_rate": 7.043570465279441e-07, + "loss": 0.5456, + "reward": 0.16420137137174606, + "reward_std": 0.017183993943035603, + "rewards/accuracy_reward": 0.16420137137174606, + "step": 577 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.708333492279053, + "epoch": 0.450682261208577, + "grad_norm": 93.87789381256485, + "kl": 5.56640625, + "learning_rate": 7.032047437898911e-07, + "loss": 0.655, + "reward": 0.14348546415567398, + "reward_std": 0.010195272974669933, + "rewards/accuracy_reward": 0.14348546415567398, + "step": 578 + }, + { + "clip_ratio": 0.0, + "completion_length": 12.104166984558105, + "epoch": 0.45146198830409356, + "grad_norm": 20.936925012062346, + "kl": 2.984375, + "learning_rate": 7.020513036522282e-07, + "loss": 0.3587, + "reward": 0.154551699757576, + "reward_std": 0.008684921078383923, + "rewards/accuracy_reward": 0.154551699757576, + "step": 579 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.479166984558105, + "epoch": 0.4522417153996101, + "grad_norm": 32.07952155390828, + "kl": 6.1171875, + "learning_rate": 7.00896734678152e-07, + "loss": 0.2775, + "reward": 0.15596140176057816, + "reward_std": 0.0150727485306561, + "rewards/accuracy_reward": 0.15596140176057816, + "step": 580 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.520833492279053, + "epoch": 0.4530214424951267, + "grad_norm": 10.493960694464665, + "kl": 3.1015625, + "learning_rate": 6.997410454392399e-07, + "loss": 0.1336, + "reward": 0.15995670855045319, + "reward_std": 0.001769316615536809, + "rewards/accuracy_reward": 0.15995670855045319, + "step": 581 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.854166984558105, + "epoch": 0.45380116959064326, + "grad_norm": 17.20521639371433, + "kl": 3.21875, + "learning_rate": 6.98584244515386e-07, + "loss": 0.4798, + "reward": 0.1522551327943802, + "reward_std": 0.03442382253706455, + "rewards/accuracy_reward": 0.1522551327943802, + "step": 582 + }, + { + "clip_ratio": 0.0, + "completion_length": 11.145833492279053, + "epoch": 0.45458089668615986, + "grad_norm": 18.88129187265365, + "kl": 3.8984375, + "learning_rate": 6.974263404947377e-07, + "loss": 0.5307, + "reward": 0.17063726484775543, + "reward_std": 0.027801679214462638, + "rewards/accuracy_reward": 0.17063726484775543, + "step": 583 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.125000238418579, + "epoch": 0.4553606237816764, + "grad_norm": 5.00385108032432, + "kl": 3.1875, + "learning_rate": 6.962673419736318e-07, + "loss": 0.1553, + "reward": 0.1709383875131607, + "reward_std": 0.0016980890650302172, + "rewards/accuracy_reward": 0.1709383875131607, + "step": 584 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.4166669845581055, + "epoch": 0.45614035087719296, + "grad_norm": 14.437853590371247, + "kl": 3.015625, + "learning_rate": 6.951072575565308e-07, + "loss": 0.2736, + "reward": 0.14713334292173386, + "reward_std": 0.015151983126997948, + "rewards/accuracy_reward": 0.14713334292173386, + "step": 585 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.729166746139526, + "epoch": 0.45692007797270956, + "grad_norm": 6.579632822723212, + "kl": 3.2890625, + "learning_rate": 6.939460958559588e-07, + "loss": 0.1675, + "reward": 0.15719697624444962, + "reward_std": 0.0020557595416903496, + "rewards/accuracy_reward": 0.15719697624444962, + "step": 586 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.958333492279053, + "epoch": 0.4576998050682261, + "grad_norm": 7.143676976157255, + "kl": 3.4453125, + "learning_rate": 6.92783865492438e-07, + "loss": 0.2347, + "reward": 0.17388392984867096, + "reward_std": 0.0012887290213257074, + "rewards/accuracy_reward": 0.17388392984867096, + "step": 587 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.666666984558105, + "epoch": 0.4584795321637427, + "grad_norm": 1.856929945772351, + "kl": 3.40625, + "learning_rate": 6.916205750944241e-07, + "loss": 0.1325, + "reward": 0.16383928060531616, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.16383928060531616, + "step": 588 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.354166746139526, + "epoch": 0.45925925925925926, + "grad_norm": 2.995201588788458, + "kl": 2.52734375, + "learning_rate": 6.904562332982428e-07, + "loss": 0.1025, + "reward": 0.14925595372915268, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.14925595372915268, + "step": 589 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.354166746139526, + "epoch": 0.4600389863547758, + "grad_norm": 6.372953796870178, + "kl": 2.20703125, + "learning_rate": 6.892908487480251e-07, + "loss": 0.1526, + "reward": 0.1655934453010559, + "reward_std": 0.0026334160938858986, + "rewards/accuracy_reward": 0.1655934453010559, + "step": 590 + }, + { + "clip_ratio": 0.0, + "completion_length": 5.875, + "epoch": 0.4608187134502924, + "grad_norm": 79.3770471401924, + "kl": 4.330078125, + "learning_rate": 6.881244300956435e-07, + "loss": 0.3731, + "reward": 0.1313258744776249, + "reward_std": 0.0016172275645658374, + "rewards/accuracy_reward": 0.1313258744776249, + "step": 591 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.416666746139526, + "epoch": 0.46159844054580895, + "grad_norm": 20.50863726574966, + "kl": 3.5703125, + "learning_rate": 6.869569860006482e-07, + "loss": 0.1614, + "reward": 0.16135913878679276, + "reward_std": 0.013928290456533432, + "rewards/accuracy_reward": 0.16135913878679276, + "step": 592 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.541666746139526, + "epoch": 0.46237816764132555, + "grad_norm": 7.188915830201685, + "kl": 2.8125, + "learning_rate": 6.85788525130202e-07, + "loss": 0.1639, + "reward": 0.1533953994512558, + "reward_std": 0.0006360284751281142, + "rewards/accuracy_reward": 0.1533953994512558, + "step": 593 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.937500238418579, + "epoch": 0.4631578947368421, + "grad_norm": 198.12095002925503, + "kl": 6.171875, + "learning_rate": 6.846190561590163e-07, + "loss": 0.6569, + "reward": 0.14233194291591644, + "reward_std": 0.001819381257519126, + "rewards/accuracy_reward": 0.14233194291591644, + "step": 594 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.479166746139526, + "epoch": 0.46393762183235865, + "grad_norm": 9.851311899725188, + "kl": 2.3359375, + "learning_rate": 6.834485877692865e-07, + "loss": 0.0951, + "reward": 0.1607142984867096, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.1607142984867096, + "step": 595 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.270833492279053, + "epoch": 0.46471734892787525, + "grad_norm": 23.489094889453526, + "kl": 3.25, + "learning_rate": 6.82277128650628e-07, + "loss": 0.1268, + "reward": 0.1563674435019493, + "reward_std": 0.024852244183421135, + "rewards/accuracy_reward": 0.1563674435019493, + "step": 596 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.666666746139526, + "epoch": 0.4654970760233918, + "grad_norm": 1103.0842668510945, + "kl": 298.046875, + "learning_rate": 6.811046875000114e-07, + "loss": 4.6009, + "reward": 0.1563657447695732, + "reward_std": 0.013424476608633995, + "rewards/accuracy_reward": 0.1563657447695732, + "step": 597 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.125, + "epoch": 0.4662768031189084, + "grad_norm": 8.04656294897895, + "kl": 1.009765625, + "learning_rate": 6.799312730216979e-07, + "loss": 0.0409, + "reward": 0.13412699848413467, + "reward_std": 0.005498574580997229, + "rewards/accuracy_reward": 0.13412699848413467, + "step": 598 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.583333492279053, + "epoch": 0.46705653021442495, + "grad_norm": 5.060463177505536, + "kl": 3.3203125, + "learning_rate": 6.787568939271746e-07, + "loss": 0.1351, + "reward": 0.18333334475755692, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.18333334475755692, + "step": 599 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.0625, + "epoch": 0.4678362573099415, + "grad_norm": 60.322601083901915, + "kl": 5.0, + "learning_rate": 6.775815589350904e-07, + "loss": 0.4171, + "reward": 0.16909722983837128, + "reward_std": 0.009622504934668541, + "rewards/accuracy_reward": 0.16909722983837128, + "step": 600 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.645833730697632, + "epoch": 0.4686159844054581, + "grad_norm": 44.481275386553236, + "kl": 4.5625, + "learning_rate": 6.764052767711903e-07, + "loss": 0.5164, + "reward": 0.15505515038967133, + "reward_std": 0.01400415413081646, + "rewards/accuracy_reward": 0.15505515038967133, + "step": 601 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.479166746139526, + "epoch": 0.46939571150097464, + "grad_norm": 8.632585486112204, + "kl": 1.859375, + "learning_rate": 6.752280561682514e-07, + "loss": 0.0929, + "reward": 0.16639957576990128, + "reward_std": 0.0009252413874492049, + "rewards/accuracy_reward": 0.16639957576990128, + "step": 602 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.020833492279053, + "epoch": 0.47017543859649125, + "grad_norm": 12.620593909317027, + "kl": 2.1484375, + "learning_rate": 6.740499058660177e-07, + "loss": 0.7253, + "reward": 0.15492628514766693, + "reward_std": 0.023291989229619503, + "rewards/accuracy_reward": 0.15492628514766693, + "step": 603 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.916666746139526, + "epoch": 0.4709551656920078, + "grad_norm": 6.023948645936651, + "kl": 3.5, + "learning_rate": 6.728708346111358e-07, + "loss": 0.1402, + "reward": 0.17861690372228622, + "reward_std": 0.005512893199920654, + "rewards/accuracy_reward": 0.17861690372228622, + "step": 604 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.750000238418579, + "epoch": 0.47173489278752434, + "grad_norm": 5.879352808794209, + "kl": 2.7421875, + "learning_rate": 6.716908511570886e-07, + "loss": 0.132, + "reward": 0.1728006899356842, + "reward_std": 0.005041202064603567, + "rewards/accuracy_reward": 0.1728006899356842, + "step": 605 + }, + { + "clip_ratio": 0.0, + "completion_length": 5.687500238418579, + "epoch": 0.47251461988304094, + "grad_norm": 7.675213068398439, + "kl": 1.48046875, + "learning_rate": 6.705099642641321e-07, + "loss": 0.1459, + "reward": 0.1426633819937706, + "reward_std": 0.0031509576365351677, + "rewards/accuracy_reward": 0.1426633819937706, + "step": 606 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.4375, + "epoch": 0.4732943469785575, + "grad_norm": 18.023879241016054, + "kl": 2.6953125, + "learning_rate": 6.693281826992291e-07, + "loss": 0.1586, + "reward": 0.13234128057956696, + "reward_std": 0.019932333379983902, + "rewards/accuracy_reward": 0.13234128057956696, + "step": 607 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.1041669845581055, + "epoch": 0.4740740740740741, + "grad_norm": 5.748733933006918, + "kl": 1.7890625, + "learning_rate": 6.681455152359844e-07, + "loss": 0.0642, + "reward": 0.16458334773778915, + "reward_std": 0.005154913291335106, + "rewards/accuracy_reward": 0.16458334773778915, + "step": 608 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.458333492279053, + "epoch": 0.47485380116959064, + "grad_norm": 9.9287640933695, + "kl": 1.640625, + "learning_rate": 6.669619706545797e-07, + "loss": 0.0755, + "reward": 0.15793652087450027, + "reward_std": 0.009622504934668541, + "rewards/accuracy_reward": 0.15793652087450027, + "step": 609 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.9375, + "epoch": 0.4756335282651072, + "grad_norm": 15.281853206542879, + "kl": 2.25, + "learning_rate": 6.657775577417089e-07, + "loss": 0.4116, + "reward": 0.1750330924987793, + "reward_std": 0.008133307099342346, + "rewards/accuracy_reward": 0.1750330924987793, + "step": 610 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.354166746139526, + "epoch": 0.4764132553606238, + "grad_norm": 12.471122766753131, + "kl": 2.984375, + "learning_rate": 6.645922852905121e-07, + "loss": 0.1106, + "reward": 0.1470194309949875, + "reward_std": 0.0005306530511006713, + "rewards/accuracy_reward": 0.1470194309949875, + "step": 611 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.937500238418579, + "epoch": 0.47719298245614034, + "grad_norm": 3.0001199756443504, + "kl": 2.15234375, + "learning_rate": 6.634061621005106e-07, + "loss": 0.0872, + "reward": 0.16417215019464493, + "reward_std": 0.0014243837213143706, + "rewards/accuracy_reward": 0.16417215019464493, + "step": 612 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.500000238418579, + "epoch": 0.47797270955165694, + "grad_norm": 4.179672100281533, + "kl": 2.78515625, + "learning_rate": 6.622191969775419e-07, + "loss": 0.1269, + "reward": 0.18232886493206024, + "reward_std": 0.0009021099540404975, + "rewards/accuracy_reward": 0.18232886493206024, + "step": 613 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.125000238418579, + "epoch": 0.4787524366471735, + "grad_norm": 8.951380579323073, + "kl": 2.0546875, + "learning_rate": 6.610313987336938e-07, + "loss": 0.0962, + "reward": 0.15555556863546371, + "reward_std": 0.009622504934668541, + "rewards/accuracy_reward": 0.15555556863546371, + "step": 614 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.770833492279053, + "epoch": 0.47953216374269003, + "grad_norm": 10.399509178769897, + "kl": 2.0859375, + "learning_rate": 6.598427761872391e-07, + "loss": 0.1449, + "reward": 0.15236419439315796, + "reward_std": 0.005728523246943951, + "rewards/accuracy_reward": 0.15236419439315796, + "step": 615 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.916666984558105, + "epoch": 0.48031189083820663, + "grad_norm": 4.422649413931237, + "kl": 2.515625, + "learning_rate": 6.586533381625709e-07, + "loss": 0.1128, + "reward": 0.17276348173618317, + "reward_std": 0.0005306530511006713, + "rewards/accuracy_reward": 0.17276348173618317, + "step": 616 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.4375, + "epoch": 0.4810916179337232, + "grad_norm": 129.67718452079302, + "kl": 5.2734375, + "learning_rate": 6.574630934901358e-07, + "loss": 0.5996, + "reward": 0.1466388925909996, + "reward_std": 0.006306389521341771, + "rewards/accuracy_reward": 0.1466388925909996, + "step": 617 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.479166746139526, + "epoch": 0.4818713450292398, + "grad_norm": 17.16827123030281, + "kl": 2.6171875, + "learning_rate": 6.562720510063694e-07, + "loss": 0.4359, + "reward": 0.1692412719130516, + "reward_std": 0.012123214080929756, + "rewards/accuracy_reward": 0.1692412719130516, + "step": 618 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.708333492279053, + "epoch": 0.48265107212475633, + "grad_norm": 2.363160291070905, + "kl": 1.546875, + "learning_rate": 6.5508021955363e-07, + "loss": 0.0631, + "reward": 0.15238095819950104, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.15238095819950104, + "step": 619 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.500000238418579, + "epoch": 0.4834307992202729, + "grad_norm": 4.739304596391792, + "kl": 1.53515625, + "learning_rate": 6.538876079801331e-07, + "loss": 0.0598, + "reward": 0.17053572088479996, + "reward_std": 0.005154913291335106, + "rewards/accuracy_reward": 0.17053572088479996, + "step": 620 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.145833492279053, + "epoch": 0.4842105263157895, + "grad_norm": 1.7255607833059352, + "kl": 1.9921875, + "learning_rate": 6.526942251398866e-07, + "loss": 0.0769, + "reward": 0.14479167759418488, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.14479167759418488, + "step": 621 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.354166746139526, + "epoch": 0.484990253411306, + "grad_norm": 47.40812056596257, + "kl": 4.578125, + "learning_rate": 6.515000798926235e-07, + "loss": 0.2508, + "reward": 0.159835085272789, + "reward_std": 0.0025448258966207504, + "rewards/accuracy_reward": 0.159835085272789, + "step": 622 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.625000238418579, + "epoch": 0.48576998050682263, + "grad_norm": 22.518783224194827, + "kl": 2.390625, + "learning_rate": 6.503051811037377e-07, + "loss": 0.3716, + "reward": 0.16134236752986908, + "reward_std": 0.015866519417613745, + "rewards/accuracy_reward": 0.16134236752986908, + "step": 623 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.7916669845581055, + "epoch": 0.4865497076023392, + "grad_norm": 30.88465054349459, + "kl": 6.75, + "learning_rate": 6.491095376442164e-07, + "loss": 0.1673, + "reward": 0.15853175520896912, + "reward_std": 0.035625264048576355, + "rewards/accuracy_reward": 0.15853175520896912, + "step": 624 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.375000238418579, + "epoch": 0.4873294346978557, + "grad_norm": 19.168217749151395, + "kl": 1.021484375, + "learning_rate": 6.479131583905766e-07, + "loss": 0.409, + "reward": 0.1417711079120636, + "reward_std": 0.01613397477194667, + "rewards/accuracy_reward": 0.1417711079120636, + "step": 625 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.333333492279053, + "epoch": 0.4881091617933723, + "grad_norm": 15.944055747770157, + "kl": 3.5078125, + "learning_rate": 6.467160522247965e-07, + "loss": 0.0806, + "reward": 0.14295197278261185, + "reward_std": 0.016854544635862112, + "rewards/accuracy_reward": 0.14295197278261185, + "step": 626 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.541666984558105, + "epoch": 0.4888888888888889, + "grad_norm": 15.846691362415955, + "kl": 2.23828125, + "learning_rate": 6.455182280342522e-07, + "loss": 0.1388, + "reward": 0.1582958921790123, + "reward_std": 0.01749379187822342, + "rewards/accuracy_reward": 0.1582958921790123, + "step": 627 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.6041669845581055, + "epoch": 0.4896686159844055, + "grad_norm": 2.2593549347639135, + "kl": 2.046875, + "learning_rate": 6.443196947116492e-07, + "loss": 0.0824, + "reward": 0.18854166567325592, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.18854166567325592, + "step": 628 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.604166746139526, + "epoch": 0.490448343079922, + "grad_norm": 5.918806433786088, + "kl": 0.94921875, + "learning_rate": 6.431204611549589e-07, + "loss": 0.0259, + "reward": 0.18194445222616196, + "reward_std": 0.0048112524673342705, + "rewards/accuracy_reward": 0.18194445222616196, + "step": 629 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.500000238418579, + "epoch": 0.49122807017543857, + "grad_norm": 12.937843315553032, + "kl": 1.33984375, + "learning_rate": 6.419205362673503e-07, + "loss": 0.116, + "reward": 0.15198414027690887, + "reward_std": 0.009622504934668541, + "rewards/accuracy_reward": 0.15198414027690887, + "step": 630 + }, + { + "clip_ratio": 0.0, + "completion_length": 5.666666746139526, + "epoch": 0.49200779727095517, + "grad_norm": 35.714845503092974, + "kl": 1.595703125, + "learning_rate": 6.407199289571254e-07, + "loss": 0.2641, + "reward": 0.13688726723194122, + "reward_std": 0.011462102644145489, + "rewards/accuracy_reward": 0.13688726723194122, + "step": 631 + }, + { + "clip_ratio": 0.0, + "completion_length": 5.812500238418579, + "epoch": 0.4927875243664717, + "grad_norm": 69.86771767706655, + "kl": 5.318359375, + "learning_rate": 6.395186481376521e-07, + "loss": 0.222, + "reward": 0.13960815221071243, + "reward_std": 0.02216244861483574, + "rewards/accuracy_reward": 0.13960815221071243, + "step": 632 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.4375, + "epoch": 0.4935672514619883, + "grad_norm": 8.01478186582231, + "kl": 2.1181640625, + "learning_rate": 6.383167027272992e-07, + "loss": 0.0617, + "reward": 0.1440332680940628, + "reward_std": 0.013452515006065369, + "rewards/accuracy_reward": 0.1440332680940628, + "step": 633 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.604166746139526, + "epoch": 0.49434697855750487, + "grad_norm": 8.795807152079473, + "kl": 1.96875, + "learning_rate": 6.371141016493684e-07, + "loss": 0.046, + "reward": 0.16944444924592972, + "reward_std": 0.01297498308122158, + "rewards/accuracy_reward": 0.16944444924592972, + "step": 634 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.375000238418579, + "epoch": 0.4951267056530214, + "grad_norm": 9.0496512155821, + "kl": 2.1796875, + "learning_rate": 6.359108538320302e-07, + "loss": 0.1689, + "reward": 0.14785150811076164, + "reward_std": 0.005993825849145651, + "rewards/accuracy_reward": 0.14785150811076164, + "step": 635 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.395833492279053, + "epoch": 0.495906432748538, + "grad_norm": 7193.058519057484, + "kl": 187.134765625, + "learning_rate": 6.347069682082558e-07, + "loss": 15.7364, + "reward": 0.160714291036129, + "reward_std": 0.009622504934668541, + "rewards/accuracy_reward": 0.160714291036129, + "step": 636 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.458333492279053, + "epoch": 0.49668615984405456, + "grad_norm": 9.988452809840618, + "kl": 1.6328125, + "learning_rate": 6.335024537157516e-07, + "loss": 0.0923, + "reward": 0.18113402277231216, + "reward_std": 0.00504120159894228, + "rewards/accuracy_reward": 0.18113402277231216, + "step": 637 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.625000238418579, + "epoch": 0.49746588693957117, + "grad_norm": 10.971990521167271, + "kl": 3.09765625, + "learning_rate": 6.322973192968931e-07, + "loss": 0.1809, + "reward": 0.1791294664144516, + "reward_std": 0.01146968174725771, + "rewards/accuracy_reward": 0.1791294664144516, + "step": 638 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.041666746139526, + "epoch": 0.4982456140350877, + "grad_norm": 7.030819740507299, + "kl": 1.2158203125, + "learning_rate": 6.310915738986581e-07, + "loss": 0.0645, + "reward": 0.16208640486001968, + "reward_std": 0.00504120159894228, + "rewards/accuracy_reward": 0.16208640486001968, + "step": 639 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.979166746139526, + "epoch": 0.49902534113060426, + "grad_norm": 1.9810138733082172, + "kl": 1.806640625, + "learning_rate": 6.298852264725597e-07, + "loss": 0.0721, + "reward": 0.1666666716337204, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.1666666716337204, + "step": 640 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.958333492279053, + "epoch": 0.49980506822612086, + "grad_norm": 17.153507425862223, + "kl": 1.984375, + "learning_rate": 6.286782859745814e-07, + "loss": 0.0376, + "reward": 0.1480654776096344, + "reward_std": 0.020619653165340424, + "rewards/accuracy_reward": 0.1480654776096344, + "step": 641 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.145833492279053, + "epoch": 0.5005847953216375, + "grad_norm": 6.674066550522118, + "kl": 1.58984375, + "learning_rate": 6.274707613651092e-07, + "loss": 0.0586, + "reward": 0.1725504994392395, + "reward_std": 0.007756044622510672, + "rewards/accuracy_reward": 0.1725504994392395, + "step": 642 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.1666669845581055, + "epoch": 0.501364522417154, + "grad_norm": 12.824860359289994, + "kl": 1.0380859375, + "learning_rate": 6.262626616088653e-07, + "loss": 0.0922, + "reward": 0.15486112236976624, + "reward_std": 0.0008109381305985153, + "rewards/accuracy_reward": 0.15486112236976624, + "step": 643 + }, + { + "clip_ratio": 0.0, + "completion_length": 5.687500238418579, + "epoch": 0.5021442495126706, + "grad_norm": 5.728281211182756, + "kl": 0.494140625, + "learning_rate": 6.250539956748425e-07, + "loss": 0.0061, + "reward": 0.16006945073604584, + "reward_std": 0.0060140653513371944, + "rewards/accuracy_reward": 0.16006945073604584, + "step": 644 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.1875, + "epoch": 0.5029239766081871, + "grad_norm": 30.717564417513398, + "kl": 4.1953125, + "learning_rate": 6.238447725362365e-07, + "loss": 0.2052, + "reward": 0.14884259551763535, + "reward_std": 0.026606866158545017, + "rewards/accuracy_reward": 0.14884259551763535, + "step": 645 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.770833492279053, + "epoch": 0.5037037037037037, + "grad_norm": 11.833191675001, + "kl": 2.2734375, + "learning_rate": 6.226350011703799e-07, + "loss": 0.1026, + "reward": 0.17846541106700897, + "reward_std": 0.005884554237127304, + "rewards/accuracy_reward": 0.17846541106700897, + "step": 646 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.604166746139526, + "epoch": 0.5044834307992203, + "grad_norm": 14.240639321293012, + "kl": 0.912109375, + "learning_rate": 6.214246905586751e-07, + "loss": 0.1685, + "reward": 0.1389467641711235, + "reward_std": 0.002205158583819866, + "rewards/accuracy_reward": 0.1389467641711235, + "step": 647 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.854166746139526, + "epoch": 0.5052631578947369, + "grad_norm": 9.647720482389149, + "kl": 1.154296875, + "learning_rate": 6.202138496865281e-07, + "loss": 0.0552, + "reward": 0.16805556416511536, + "reward_std": 0.00682500284165144, + "rewards/accuracy_reward": 0.16805556416511536, + "step": 648 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.708333492279053, + "epoch": 0.5060428849902534, + "grad_norm": 12.229031584877808, + "kl": 1.56640625, + "learning_rate": 6.190024875432818e-07, + "loss": 0.2373, + "reward": 0.1589185893535614, + "reward_std": 0.006443812511861324, + "rewards/accuracy_reward": 0.1589185893535614, + "step": 649 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.75, + "epoch": 0.50682261208577, + "grad_norm": 14.465505864382934, + "kl": 1.59375, + "learning_rate": 6.177906131221483e-07, + "loss": 0.1529, + "reward": 0.16737351566553116, + "reward_std": 0.02384147420525551, + "rewards/accuracy_reward": 0.16737351566553116, + "step": 650 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.270833492279053, + "epoch": 0.5076023391812865, + "grad_norm": 10.700838307797138, + "kl": 0.4462890625, + "learning_rate": 6.165782354201435e-07, + "loss": -0.0086, + "reward": 0.15114088356494904, + "reward_std": 0.016323892399668694, + "rewards/accuracy_reward": 0.15114088356494904, + "step": 651 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.937500238418579, + "epoch": 0.5083820662768032, + "grad_norm": 18.398483095593406, + "kl": 2.203125, + "learning_rate": 6.153653634380198e-07, + "loss": 0.1543, + "reward": 0.1621279940009117, + "reward_std": 0.023587769828736782, + "rewards/accuracy_reward": 0.1621279940009117, + "step": 652 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.812500238418579, + "epoch": 0.5091617933723197, + "grad_norm": 283.6952956407202, + "kl": 12.71875, + "learning_rate": 6.141520061801988e-07, + "loss": 0.9772, + "reward": 0.1845824345946312, + "reward_std": 0.014288676902651787, + "rewards/accuracy_reward": 0.1845824345946312, + "step": 653 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.645833492279053, + "epoch": 0.5099415204678363, + "grad_norm": 8.87848252936082, + "kl": 1.640625, + "learning_rate": 6.129381726547049e-07, + "loss": 0.065, + "reward": 0.171875, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.171875, + "step": 654 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.0, + "epoch": 0.5107212475633528, + "grad_norm": 17.34603911024246, + "kl": 2.1796875, + "learning_rate": 6.117238718730982e-07, + "loss": 0.1457, + "reward": 0.144157275557518, + "reward_std": 0.010380841908045113, + "rewards/accuracy_reward": 0.144157275557518, + "step": 655 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.458333492279053, + "epoch": 0.5115009746588693, + "grad_norm": 49.85826248694607, + "kl": 6.953125, + "learning_rate": 6.10509112850408e-07, + "loss": 0.3299, + "reward": 0.15697727352380753, + "reward_std": 0.021236851811408997, + "rewards/accuracy_reward": 0.15697727352380753, + "step": 656 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.270833492279053, + "epoch": 0.512280701754386, + "grad_norm": 5.076072964666256, + "kl": 0.595703125, + "learning_rate": 6.092939046050654e-07, + "loss": 0.0111, + "reward": 0.159722238779068, + "reward_std": 0.0048112524673342705, + "rewards/accuracy_reward": 0.159722238779068, + "step": 657 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.270833492279053, + "epoch": 0.5130604288499026, + "grad_norm": 26.830201776101685, + "kl": 2.640625, + "learning_rate": 6.080782561588366e-07, + "loss": 0.0486, + "reward": 0.170258529484272, + "reward_std": 0.028441181406378746, + "rewards/accuracy_reward": 0.170258529484272, + "step": 658 + }, + { + "clip_ratio": 0.0, + "completion_length": 5.916666746139526, + "epoch": 0.5138401559454191, + "grad_norm": 43.680268961762586, + "kl": 7.28515625, + "learning_rate": 6.068621765367563e-07, + "loss": 0.0624, + "reward": 0.15620040148496628, + "reward_std": 0.02526606386527419, + "rewards/accuracy_reward": 0.15620040148496628, + "step": 659 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.875, + "epoch": 0.5146198830409356, + "grad_norm": 9.658475945557967, + "kl": 1.146484375, + "learning_rate": 6.056456747670594e-07, + "loss": 0.0218, + "reward": 0.16845238953828812, + "reward_std": 0.016333107836544514, + "rewards/accuracy_reward": 0.16845238953828812, + "step": 660 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.041666746139526, + "epoch": 0.5153996101364522, + "grad_norm": 17.11592550448531, + "kl": 1.650390625, + "learning_rate": 6.044287598811156e-07, + "loss": 0.2725, + "reward": 0.1574452891945839, + "reward_std": 0.016178556601516902, + "rewards/accuracy_reward": 0.1574452891945839, + "step": 661 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.645833492279053, + "epoch": 0.5161793372319688, + "grad_norm": 16.750914811252894, + "kl": 0.970703125, + "learning_rate": 6.03211440913361e-07, + "loss": 0.0353, + "reward": 0.1451389044523239, + "reward_std": 0.021835491061210632, + "rewards/accuracy_reward": 0.1451389044523239, + "step": 662 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.375000238418579, + "epoch": 0.5169590643274854, + "grad_norm": 2050.8657533772443, + "kl": 61.25390625, + "learning_rate": 6.019937269012325e-07, + "loss": 2.4176, + "reward": 0.173611119389534, + "reward_std": 0.01114263292402029, + "rewards/accuracy_reward": 0.173611119389534, + "step": 663 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.666666746139526, + "epoch": 0.5177387914230019, + "grad_norm": 84.41950741746031, + "kl": 5.953125, + "learning_rate": 6.007756268850987e-07, + "loss": 0.3029, + "reward": 0.17240846902132034, + "reward_std": 0.016569858882576227, + "rewards/accuracy_reward": 0.17240846902132034, + "step": 664 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.250000238418579, + "epoch": 0.5185185185185185, + "grad_norm": 10.632124676640212, + "kl": 3.75, + "learning_rate": 5.995571499081946e-07, + "loss": 0.1058, + "reward": 0.16848959028720856, + "reward_std": 0.010446218773722649, + "rewards/accuracy_reward": 0.16848959028720856, + "step": 665 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.875000476837158, + "epoch": 0.519298245614035, + "grad_norm": 156.28888702429649, + "kl": 7.90625, + "learning_rate": 5.983383050165539e-07, + "loss": 0.3183, + "reward": 0.18854168057441711, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.18854168057441711, + "step": 666 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.104166746139526, + "epoch": 0.5200779727095517, + "grad_norm": 5087.227959639842, + "kl": 154.83984375, + "learning_rate": 5.971191012589415e-07, + "loss": 10.0331, + "reward": 0.159132719039917, + "reward_std": 0.028588212095201015, + "rewards/accuracy_reward": 0.159132719039917, + "step": 667 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.833333492279053, + "epoch": 0.5208576998050682, + "grad_norm": 24.811678337046942, + "kl": 2.40625, + "learning_rate": 5.958995476867862e-07, + "loss": 0.0705, + "reward": 0.17213542014360428, + "reward_std": 0.02588246390223503, + "rewards/accuracy_reward": 0.17213542014360428, + "step": 668 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.166666746139526, + "epoch": 0.5216374269005848, + "grad_norm": 19.89640599697181, + "kl": 3.5234375, + "learning_rate": 5.946796533541143e-07, + "loss": 0.0842, + "reward": 0.18283730000257492, + "reward_std": 0.0351847056299448, + "rewards/accuracy_reward": 0.18283730000257492, + "step": 669 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.791666746139526, + "epoch": 0.5224171539961013, + "grad_norm": 2.3450295174618097, + "kl": 1.6640625, + "learning_rate": 5.934594273174818e-07, + "loss": 0.0663, + "reward": 0.19166667759418488, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.19166667759418488, + "step": 670 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.458333492279053, + "epoch": 0.5231968810916179, + "grad_norm": 5.56837939255499, + "kl": 2.9765625, + "learning_rate": 5.922388786359073e-07, + "loss": 0.1084, + "reward": 0.18318451941013336, + "reward_std": 0.009325588122010231, + "rewards/accuracy_reward": 0.18318451941013336, + "step": 671 + }, + { + "clip_ratio": 0.0, + "completion_length": 5.833333492279053, + "epoch": 0.5239766081871345, + "grad_norm": 29.431000852624095, + "kl": 4.828125, + "learning_rate": 5.910180163708046e-07, + "loss": 0.1548, + "reward": 0.17681623995304108, + "reward_std": 0.02648637257516384, + "rewards/accuracy_reward": 0.17681623995304108, + "step": 672 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.750000238418579, + "epoch": 0.5247563352826511, + "grad_norm": 13.758226623764198, + "kl": 1.984375, + "learning_rate": 5.897968495859157e-07, + "loss": 0.0634, + "reward": 0.18368057161569595, + "reward_std": 0.022755810990929604, + "rewards/accuracy_reward": 0.18368057161569595, + "step": 673 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.916666746139526, + "epoch": 0.5255360623781676, + "grad_norm": 139.22857474442833, + "kl": 12.96875, + "learning_rate": 5.885753873472434e-07, + "loss": 0.4604, + "reward": 0.16230159997940063, + "reward_std": 0.015001907013356686, + "rewards/accuracy_reward": 0.16230159997940063, + "step": 674 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.437500476837158, + "epoch": 0.5263157894736842, + "grad_norm": 154.49189447310852, + "kl": 12.5, + "learning_rate": 5.87353638722984e-07, + "loss": 0.6283, + "reward": 0.16762153804302216, + "reward_std": 0.03621844947338104, + "rewards/accuracy_reward": 0.16762153804302216, + "step": 675 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.125000238418579, + "epoch": 0.5270955165692007, + "grad_norm": 28.054526732656722, + "kl": 2.7890625, + "learning_rate": 5.861316127834592e-07, + "loss": 0.1547, + "reward": 0.17039930820465088, + "reward_std": 0.014133053831756115, + "rewards/accuracy_reward": 0.17039930820465088, + "step": 676 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.750000238418579, + "epoch": 0.5278752436647174, + "grad_norm": 14.296662507127708, + "kl": 1.556640625, + "learning_rate": 5.849093186010509e-07, + "loss": 0.0602, + "reward": 0.17213542759418488, + "reward_std": 0.02106943028047681, + "rewards/accuracy_reward": 0.17213542759418488, + "step": 677 + }, + { + "clip_ratio": 0.0, + "completion_length": 5.5625, + "epoch": 0.5286549707602339, + "grad_norm": 130.2400127702207, + "kl": 6.99609375, + "learning_rate": 5.836867652501313e-07, + "loss": 0.3414, + "reward": 0.14906556904315948, + "reward_std": 0.020471498370170593, + "rewards/accuracy_reward": 0.14906556904315948, + "step": 678 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.812500238418579, + "epoch": 0.5294346978557505, + "grad_norm": 11.871878217320992, + "kl": 2.01171875, + "learning_rate": 5.824639618069973e-07, + "loss": 0.0839, + "reward": 0.19124504923820496, + "reward_std": 0.014874828048050404, + "rewards/accuracy_reward": 0.19124504923820496, + "step": 679 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.854166746139526, + "epoch": 0.530214424951267, + "grad_norm": 17.26443076390969, + "kl": 2.2734375, + "learning_rate": 5.812409173498019e-07, + "loss": 0.1086, + "reward": 0.1638144925236702, + "reward_std": 0.02170541975647211, + "rewards/accuracy_reward": 0.1638144925236702, + "step": 680 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.75, + "epoch": 0.5309941520467836, + "grad_norm": 28.133039339991658, + "kl": 3.16015625, + "learning_rate": 5.80017640958488e-07, + "loss": 0.1581, + "reward": 0.1653050184249878, + "reward_std": 0.006909038871526718, + "rewards/accuracy_reward": 0.1653050184249878, + "step": 681 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.750000238418579, + "epoch": 0.5317738791423002, + "grad_norm": 11.186608179646798, + "kl": 1.28515625, + "learning_rate": 5.7879414171472e-07, + "loss": 0.0213, + "reward": 0.18263889104127884, + "reward_std": 0.017628328874707222, + "rewards/accuracy_reward": 0.18263889104127884, + "step": 682 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.312500238418579, + "epoch": 0.5325536062378168, + "grad_norm": 13.021101597241106, + "kl": 2.8359375, + "learning_rate": 5.775704287018171e-07, + "loss": 0.1604, + "reward": 0.1791973114013672, + "reward_std": 0.012062542024068534, + "rewards/accuracy_reward": 0.1791973114013672, + "step": 683 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.104166746139526, + "epoch": 0.5333333333333333, + "grad_norm": 53.50209530804248, + "kl": 4.1484375, + "learning_rate": 5.763465110046853e-07, + "loss": 0.1854, + "reward": 0.18683037161827087, + "reward_std": 0.025573696941137314, + "rewards/accuracy_reward": 0.18683037161827087, + "step": 684 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.604166984558105, + "epoch": 0.5341130604288499, + "grad_norm": 23.466449976806057, + "kl": 2.828125, + "learning_rate": 5.751223977097501e-07, + "loss": 0.4945, + "reward": 0.14113732427358627, + "reward_std": 0.02884014882147312, + "rewards/accuracy_reward": 0.14113732427358627, + "step": 685 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.000000238418579, + "epoch": 0.5348927875243664, + "grad_norm": 44.963489753777345, + "kl": 4.515625, + "learning_rate": 5.738980979048895e-07, + "loss": 0.1324, + "reward": 0.17809193581342697, + "reward_std": 0.029883730225265026, + "rewards/accuracy_reward": 0.17809193581342697, + "step": 686 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.166666746139526, + "epoch": 0.5356725146198831, + "grad_norm": 49.31301511235178, + "kl": 6.2109375, + "learning_rate": 5.726736206793657e-07, + "loss": 0.3362, + "reward": 0.1874600201845169, + "reward_std": 0.01942795980721712, + "rewards/accuracy_reward": 0.1874600201845169, + "step": 687 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.145833492279053, + "epoch": 0.5364522417153996, + "grad_norm": 16.179513267930382, + "kl": 3.765625, + "learning_rate": 5.714489751237582e-07, + "loss": 0.1491, + "reward": 0.1918584108352661, + "reward_std": 0.021227598190307617, + "rewards/accuracy_reward": 0.1918584108352661, + "step": 688 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.145833492279053, + "epoch": 0.5372319688109162, + "grad_norm": 13.499458967313133, + "kl": 2.26171875, + "learning_rate": 5.702241703298966e-07, + "loss": 0.0808, + "reward": 0.1741752177476883, + "reward_std": 0.01613913895562291, + "rewards/accuracy_reward": 0.1741752177476883, + "step": 689 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.000000238418579, + "epoch": 0.5380116959064327, + "grad_norm": 8.157827902881921, + "kl": 3.578125, + "learning_rate": 5.689992153907924e-07, + "loss": 0.1595, + "reward": 0.18358135968446732, + "reward_std": 0.022893482819199562, + "rewards/accuracy_reward": 0.18358135968446732, + "step": 690 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.583333492279053, + "epoch": 0.5387914230019493, + "grad_norm": 22.99930865181368, + "kl": 2.9453125, + "learning_rate": 5.677741194005715e-07, + "loss": 0.1294, + "reward": 0.19384921342134476, + "reward_std": 0.0048112524673342705, + "rewards/accuracy_reward": 0.19384921342134476, + "step": 691 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.854166746139526, + "epoch": 0.5395711500974659, + "grad_norm": 162.69802776558012, + "kl": 7.90625, + "learning_rate": 5.665488914544074e-07, + "loss": 1.3169, + "reward": 0.19633013010025024, + "reward_std": 0.019426160492002964, + "rewards/accuracy_reward": 0.19633013010025024, + "step": 692 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.145833492279053, + "epoch": 0.5403508771929825, + "grad_norm": 641.6373505465062, + "kl": 37.140625, + "learning_rate": 5.653235406484531e-07, + "loss": 1.8916, + "reward": 0.19438251852989197, + "reward_std": 0.03529222495853901, + "rewards/accuracy_reward": 0.19438251852989197, + "step": 693 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.9791669845581055, + "epoch": 0.541130604288499, + "grad_norm": 542.3112645059924, + "kl": 20.328125, + "learning_rate": 5.640980760797737e-07, + "loss": 1.3815, + "reward": 0.19339163601398468, + "reward_std": 0.007063568918965757, + "rewards/accuracy_reward": 0.19339163601398468, + "step": 694 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.854166746139526, + "epoch": 0.5419103313840156, + "grad_norm": 24.741152175313765, + "kl": 4.953125, + "learning_rate": 5.628725068462787e-07, + "loss": 0.4404, + "reward": 0.19107971340417862, + "reward_std": 0.02465215139091015, + "rewards/accuracy_reward": 0.19107971340417862, + "step": 695 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.145833492279053, + "epoch": 0.5426900584795321, + "grad_norm": 7.452326717408337, + "kl": 1.71484375, + "learning_rate": 5.616468420466551e-07, + "loss": 0.0925, + "reward": 0.19727183133363724, + "reward_std": 0.007431631907820702, + "rewards/accuracy_reward": 0.19727183133363724, + "step": 696 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.916666746139526, + "epoch": 0.5434697855750488, + "grad_norm": 14.993960149756191, + "kl": 3.703125, + "learning_rate": 5.60421090780299e-07, + "loss": 0.1764, + "reward": 0.1950024962425232, + "reward_std": 0.022151668556034565, + "rewards/accuracy_reward": 0.1950024962425232, + "step": 697 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.833333492279053, + "epoch": 0.5442495126705653, + "grad_norm": 11.883147731601257, + "kl": 4.578125, + "learning_rate": 5.591952621472485e-07, + "loss": 0.191, + "reward": 0.20087379217147827, + "reward_std": 0.03229519259184599, + "rewards/accuracy_reward": 0.20087379217147827, + "step": 698 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.0, + "epoch": 0.5450292397660819, + "grad_norm": 104.15676464754617, + "kl": 10.6796875, + "learning_rate": 5.579693652481161e-07, + "loss": 0.6969, + "reward": 0.19982202351093292, + "reward_std": 0.02143909689038992, + "rewards/accuracy_reward": 0.19982202351093292, + "step": 699 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.770833492279053, + "epoch": 0.5458089668615984, + "grad_norm": 24162.883116184807, + "kl": 1673.453125, + "learning_rate": 5.567434091840212e-07, + "loss": 74.2185, + "reward": 0.17545372247695923, + "reward_std": 0.017187719931825995, + "rewards/accuracy_reward": 0.17545372247695923, + "step": 700 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.083333492279053, + "epoch": 0.546588693957115, + "grad_norm": 311.0922699460503, + "kl": 23.4765625, + "learning_rate": 5.555174030565222e-07, + "loss": 1.3108, + "reward": 0.20801282674074173, + "reward_std": 0.043087988160550594, + "rewards/accuracy_reward": 0.20801282674074173, + "step": 701 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.645833730697632, + "epoch": 0.5473684210526316, + "grad_norm": 52.34143136163874, + "kl": 9.6484375, + "learning_rate": 5.542913559675497e-07, + "loss": 0.3221, + "reward": 0.17341270297765732, + "reward_std": 0.02454973664134741, + "rewards/accuracy_reward": 0.17341270297765732, + "step": 702 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.770833730697632, + "epoch": 0.5481481481481482, + "grad_norm": 63.190213433185555, + "kl": 7.4453125, + "learning_rate": 5.530652770193379e-07, + "loss": 0.4314, + "reward": 0.19103111326694489, + "reward_std": 0.018122155219316483, + "rewards/accuracy_reward": 0.19103111326694489, + "step": 703 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.770833492279053, + "epoch": 0.5489278752436647, + "grad_norm": 57.51952961665184, + "kl": 2.78125, + "learning_rate": 5.518391753143574e-07, + "loss": 0.2208, + "reward": 0.1961694210767746, + "reward_std": 0.019015775993466377, + "rewards/accuracy_reward": 0.1961694210767746, + "step": 704 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.145833492279053, + "epoch": 0.5497076023391813, + "grad_norm": 180.4042428106239, + "kl": 8.7548828125, + "learning_rate": 5.506130599552483e-07, + "loss": 0.8192, + "reward": 0.17864811420440674, + "reward_std": 0.008954516611993313, + "rewards/accuracy_reward": 0.17864811420440674, + "step": 705 + }, + { + "clip_ratio": 0.0, + "completion_length": 14.833333969116211, + "epoch": 0.5504873294346978, + "grad_norm": 4418.477234155181, + "kl": 157.359375, + "learning_rate": 5.493869400447518e-07, + "loss": 10.2428, + "reward": 0.15092042833566666, + "reward_std": 0.0321147795766592, + "rewards/accuracy_reward": 0.15092042833566666, + "step": 706 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.0416669845581055, + "epoch": 0.5512670565302145, + "grad_norm": 17.197449789383192, + "kl": 3.4765625, + "learning_rate": 5.481608246856427e-07, + "loss": 0.2102, + "reward": 0.22228986769914627, + "reward_std": 0.020675611682236195, + "rewards/accuracy_reward": 0.22228986769914627, + "step": 707 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.395833492279053, + "epoch": 0.552046783625731, + "grad_norm": 24.75895248248711, + "kl": 5.03125, + "learning_rate": 5.469347229806622e-07, + "loss": 0.2661, + "reward": 0.21269650757312775, + "reward_std": 0.030957388691604137, + "rewards/accuracy_reward": 0.21269650757312775, + "step": 708 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.0625, + "epoch": 0.5528265107212476, + "grad_norm": 12.262279297133833, + "kl": 3.6328125, + "learning_rate": 5.457086440324503e-07, + "loss": 0.1487, + "reward": 0.2014680802822113, + "reward_std": 0.012678503233473748, + "rewards/accuracy_reward": 0.2014680802822113, + "step": 709 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.604166746139526, + "epoch": 0.5536062378167641, + "grad_norm": 14.267412304401052, + "kl": 3.390625, + "learning_rate": 5.444825969434777e-07, + "loss": 0.3105, + "reward": 0.20235450565814972, + "reward_std": 0.023722592275589705, + "rewards/accuracy_reward": 0.20235450565814972, + "step": 710 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.062500238418579, + "epoch": 0.5543859649122806, + "grad_norm": 3.098212699507371, + "kl": 2.0390625, + "learning_rate": 5.432565908159789e-07, + "loss": 0.0804, + "reward": 0.19099266082048416, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.19099266082048416, + "step": 711 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.020833492279053, + "epoch": 0.5551656920077973, + "grad_norm": 12.61422364904933, + "kl": 3.6796875, + "learning_rate": 5.420306347518841e-07, + "loss": 0.1088, + "reward": 0.2105216085910797, + "reward_std": 0.03166159801185131, + "rewards/accuracy_reward": 0.2105216085910797, + "step": 712 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.7291669845581055, + "epoch": 0.5559454191033139, + "grad_norm": 12.778117750548397, + "kl": 2.52734375, + "learning_rate": 5.408047378527515e-07, + "loss": 0.1102, + "reward": 0.18544147163629532, + "reward_std": 0.02681065909564495, + "rewards/accuracy_reward": 0.18544147163629532, + "step": 713 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.666666746139526, + "epoch": 0.5567251461988304, + "grad_norm": 19.36387005810071, + "kl": 2.9375, + "learning_rate": 5.39578909219701e-07, + "loss": 0.1413, + "reward": 0.1749693676829338, + "reward_std": 0.0251549887470901, + "rewards/accuracy_reward": 0.1749693676829338, + "step": 714 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.020833730697632, + "epoch": 0.557504873294347, + "grad_norm": 15.732964286537323, + "kl": 2.5, + "learning_rate": 5.383531579533448e-07, + "loss": 0.2041, + "reward": 0.19231152534484863, + "reward_std": 0.018857845105230808, + "rewards/accuracy_reward": 0.19231152534484863, + "step": 715 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.958333492279053, + "epoch": 0.5582846003898635, + "grad_norm": 27.36454473112182, + "kl": 4.203125, + "learning_rate": 5.371274931537212e-07, + "loss": 0.3749, + "reward": 0.20143437385559082, + "reward_std": 0.029813488014042377, + "rewards/accuracy_reward": 0.20143437385559082, + "step": 716 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.291666746139526, + "epoch": 0.5590643274853802, + "grad_norm": 29.97490811347302, + "kl": 5.203125, + "learning_rate": 5.359019239202262e-07, + "loss": 0.5586, + "reward": 0.16298101097345352, + "reward_std": 0.022775106132030487, + "rewards/accuracy_reward": 0.16298101097345352, + "step": 717 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.000000238418579, + "epoch": 0.5598440545808967, + "grad_norm": 524.2191846566968, + "kl": 26.203125, + "learning_rate": 5.346764593515468e-07, + "loss": 1.2854, + "reward": 0.2096840739250183, + "reward_std": 0.044943878427147865, + "rewards/accuracy_reward": 0.2096840739250183, + "step": 718 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.479166984558105, + "epoch": 0.5606237816764132, + "grad_norm": 16.78877130061686, + "kl": 1.6953125, + "learning_rate": 5.334511085455925e-07, + "loss": 0.2368, + "reward": 0.19805872440338135, + "reward_std": 0.006724819540977478, + "rewards/accuracy_reward": 0.19805872440338135, + "step": 719 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.958333492279053, + "epoch": 0.5614035087719298, + "grad_norm": 18.92408129852908, + "kl": 4.71875, + "learning_rate": 5.322258805994284e-07, + "loss": 0.2904, + "reward": 0.20917277038097382, + "reward_std": 0.015409362851642072, + "rewards/accuracy_reward": 0.20917277038097382, + "step": 720 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.8125, + "epoch": 0.5621832358674463, + "grad_norm": 10.665002131264536, + "kl": 2.453125, + "learning_rate": 5.310007846092077e-07, + "loss": 0.1507, + "reward": 0.19428324699401855, + "reward_std": 0.0033077364787459373, + "rewards/accuracy_reward": 0.19428324699401855, + "step": 721 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.687500238418579, + "epoch": 0.562962962962963, + "grad_norm": 40.64523919462699, + "kl": 3.6015625, + "learning_rate": 5.297758296701033e-07, + "loss": 0.2089, + "reward": 0.1823449209332466, + "reward_std": 0.022853082045912743, + "rewards/accuracy_reward": 0.1823449209332466, + "step": 722 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.875, + "epoch": 0.5637426900584795, + "grad_norm": 13.365020075746704, + "kl": 2.71875, + "learning_rate": 5.285510248762418e-07, + "loss": 0.139, + "reward": 0.21067707240581512, + "reward_std": 0.00542085338383913, + "rewards/accuracy_reward": 0.21067707240581512, + "step": 723 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.708333969116211, + "epoch": 0.5645224171539961, + "grad_norm": 1111.0698899104896, + "kl": 51.9375, + "learning_rate": 5.273263793206345e-07, + "loss": 3.3511, + "reward": 0.20131982117891312, + "reward_std": 0.028339996933937073, + "rewards/accuracy_reward": 0.20131982117891312, + "step": 724 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.125, + "epoch": 0.5653021442495126, + "grad_norm": 23.95313847129953, + "kl": 3.6015625, + "learning_rate": 5.261019020951106e-07, + "loss": 0.3006, + "reward": 0.21709401160478592, + "reward_std": 0.021138248965144157, + "rewards/accuracy_reward": 0.21709401160478592, + "step": 725 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.916666746139526, + "epoch": 0.5660818713450292, + "grad_norm": 34.92136709623118, + "kl": 5.640625, + "learning_rate": 5.2487760229025e-07, + "loss": 0.2838, + "reward": 0.18407223373651505, + "reward_std": 0.02077169483527541, + "rewards/accuracy_reward": 0.18407223373651505, + "step": 726 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.4375, + "epoch": 0.5668615984405458, + "grad_norm": 21.273447907786856, + "kl": 4.2734375, + "learning_rate": 5.236534889953149e-07, + "loss": 0.7199, + "reward": 0.19003383815288544, + "reward_std": 0.025045832619071007, + "rewards/accuracy_reward": 0.19003383815288544, + "step": 727 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.9375, + "epoch": 0.5676413255360624, + "grad_norm": 921.0420812355022, + "kl": 46.65234375, + "learning_rate": 5.224295712981829e-07, + "loss": 3.088, + "reward": 0.17517749965190887, + "reward_std": 0.027015223167836666, + "rewards/accuracy_reward": 0.17517749965190887, + "step": 728 + }, + { + "clip_ratio": 0.0, + "completion_length": 50.3541693687439, + "epoch": 0.5684210526315789, + "grad_norm": 20.23029607238146, + "kl": 2.765625, + "learning_rate": 5.2120585828528e-07, + "loss": 0.4956, + "reward": 0.18020396679639816, + "reward_std": 0.0344222504645586, + "rewards/accuracy_reward": 0.18020396679639816, + "step": 729 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.666666984558105, + "epoch": 0.5692007797270955, + "grad_norm": 167.7698861790345, + "kl": 8.21875, + "learning_rate": 5.19982359041512e-07, + "loss": 0.7493, + "reward": 0.16905220597982407, + "reward_std": 0.013457732740789652, + "rewards/accuracy_reward": 0.16905220597982407, + "step": 730 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.729166746139526, + "epoch": 0.569980506822612, + "grad_norm": 26.219479385465597, + "kl": 3.8359375, + "learning_rate": 5.187590826501982e-07, + "loss": 0.2268, + "reward": 0.1933784931898117, + "reward_std": 0.03980731964111328, + "rewards/accuracy_reward": 0.1933784931898117, + "step": 731 + }, + { + "clip_ratio": 0.0, + "completion_length": 14.541666984558105, + "epoch": 0.5707602339181287, + "grad_norm": 211.92177757459132, + "kl": 12.78125, + "learning_rate": 5.175360381930028e-07, + "loss": 1.0304, + "reward": 0.15167047083377838, + "reward_std": 0.040984055027365685, + "rewards/accuracy_reward": 0.15167047083377838, + "step": 732 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.4375, + "epoch": 0.5715399610136452, + "grad_norm": 46.23421803023925, + "kl": 5.6015625, + "learning_rate": 5.163132347498686e-07, + "loss": 0.5211, + "reward": 0.16398446261882782, + "reward_std": 0.03464341349899769, + "rewards/accuracy_reward": 0.16398446261882782, + "step": 733 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.770833492279053, + "epoch": 0.5723196881091618, + "grad_norm": 22.4583145542789, + "kl": 3.953125, + "learning_rate": 5.15090681398949e-07, + "loss": 0.3211, + "reward": 0.20709607005119324, + "reward_std": 0.053975099697709084, + "rewards/accuracy_reward": 0.20709607005119324, + "step": 734 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.979166746139526, + "epoch": 0.5730994152046783, + "grad_norm": 14.503801968147574, + "kl": 3.0703125, + "learning_rate": 5.138683872165408e-07, + "loss": 0.118, + "reward": 0.20572249591350555, + "reward_std": 0.011753120459616184, + "rewards/accuracy_reward": 0.20572249591350555, + "step": 735 + }, + { + "clip_ratio": 0.0, + "completion_length": 50.9166693687439, + "epoch": 0.5738791423001949, + "grad_norm": 62.1279647946864, + "kl": 6.0, + "learning_rate": 5.126463612770161e-07, + "loss": 0.8819, + "reward": 0.19868828356266022, + "reward_std": 0.03587975725531578, + "rewards/accuracy_reward": 0.19868828356266022, + "step": 736 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.166666746139526, + "epoch": 0.5746588693957115, + "grad_norm": 15.643713565093693, + "kl": 4.2734375, + "learning_rate": 5.114246126527565e-07, + "loss": 0.3693, + "reward": 0.17953062057495117, + "reward_std": 0.020495770033448935, + "rewards/accuracy_reward": 0.17953062057495117, + "step": 737 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.625000238418579, + "epoch": 0.5754385964912281, + "grad_norm": 77.25183113273191, + "kl": 7.546875, + "learning_rate": 5.102031504140842e-07, + "loss": 0.3659, + "reward": 0.19230081140995026, + "reward_std": 0.02740845491643995, + "rewards/accuracy_reward": 0.19230081140995026, + "step": 738 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.395833492279053, + "epoch": 0.5762183235867446, + "grad_norm": 7.000475441817, + "kl": 2.9765625, + "learning_rate": 5.089819836291954e-07, + "loss": 0.1014, + "reward": 0.18896903097629547, + "reward_std": 0.014748050831258297, + "rewards/accuracy_reward": 0.18896903097629547, + "step": 739 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.625000238418579, + "epoch": 0.5769980506822612, + "grad_norm": 28.470134149326157, + "kl": 3.8828125, + "learning_rate": 5.077611213640928e-07, + "loss": 0.2367, + "reward": 0.1828887090086937, + "reward_std": 0.01444562803953886, + "rewards/accuracy_reward": 0.1828887090086937, + "step": 740 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.8125, + "epoch": 0.5777777777777777, + "grad_norm": 25.482433222694905, + "kl": 6.109375, + "learning_rate": 5.065405726825182e-07, + "loss": 0.2564, + "reward": 0.2014196440577507, + "reward_std": 0.024420201312750578, + "rewards/accuracy_reward": 0.2014196440577507, + "step": 741 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.3125, + "epoch": 0.5785575048732944, + "grad_norm": 833.6018682132706, + "kl": 23.53515625, + "learning_rate": 5.053203466458858e-07, + "loss": 1.9449, + "reward": 0.18860293924808502, + "reward_std": 0.019932100549340248, + "rewards/accuracy_reward": 0.18860293924808502, + "step": 742 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.729166984558105, + "epoch": 0.5793372319688109, + "grad_norm": 118.74058673592324, + "kl": 12.5625, + "learning_rate": 5.041004523132139e-07, + "loss": 0.6016, + "reward": 0.17768223583698273, + "reward_std": 0.0218703243881464, + "rewards/accuracy_reward": 0.17768223583698273, + "step": 743 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.145833492279053, + "epoch": 0.5801169590643275, + "grad_norm": 5.797584009416236, + "kl": 3.09375, + "learning_rate": 5.028808987410586e-07, + "loss": 0.1029, + "reward": 0.2204861119389534, + "reward_std": 0.012501556426286697, + "rewards/accuracy_reward": 0.2204861119389534, + "step": 744 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.291666984558105, + "epoch": 0.580896686159844, + "grad_norm": 33.51764348931488, + "kl": 6.3359375, + "learning_rate": 5.01661694983446e-07, + "loss": 0.3254, + "reward": 0.17308900505304337, + "reward_std": 0.01642521470785141, + "rewards/accuracy_reward": 0.17308900505304337, + "step": 745 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.125, + "epoch": 0.5816764132553606, + "grad_norm": 32.4677377313916, + "kl": 6.046875, + "learning_rate": 5.004428500918054e-07, + "loss": 0.587, + "reward": 0.16805507242679596, + "reward_std": 0.027270404621958733, + "rewards/accuracy_reward": 0.16805507242679596, + "step": 746 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.750000238418579, + "epoch": 0.5824561403508772, + "grad_norm": 12.9733401908474, + "kl": 3.375, + "learning_rate": 4.992243731149014e-07, + "loss": 0.2741, + "reward": 0.19418112188577652, + "reward_std": 0.015036219730973244, + "rewards/accuracy_reward": 0.19418112188577652, + "step": 747 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.562500476837158, + "epoch": 0.5832358674463938, + "grad_norm": 12.604818759759834, + "kl": 4.96875, + "learning_rate": 4.980062730987677e-07, + "loss": 0.1591, + "reward": 0.18438390642404556, + "reward_std": 0.03260108456015587, + "rewards/accuracy_reward": 0.18438390642404556, + "step": 748 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.541666984558105, + "epoch": 0.5840155945419103, + "grad_norm": 17.315812431905435, + "kl": 4.796875, + "learning_rate": 4.967885590866388e-07, + "loss": 0.3777, + "reward": 0.16472266614437103, + "reward_std": 0.04542579501867294, + "rewards/accuracy_reward": 0.16472266614437103, + "step": 749 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.250000238418579, + "epoch": 0.5847953216374269, + "grad_norm": 513.1154296842493, + "kl": 35.796875, + "learning_rate": 4.955712401188844e-07, + "loss": 1.4642, + "reward": 0.18676988035440445, + "reward_std": 0.02362931426614523, + "rewards/accuracy_reward": 0.18676988035440445, + "step": 750 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.041666984558105, + "epoch": 0.5855750487329434, + "grad_norm": 280.46370623111204, + "kl": 11.25, + "learning_rate": 4.943543252329406e-07, + "loss": 1.4801, + "reward": 0.19414883852005005, + "reward_std": 0.03875085152685642, + "rewards/accuracy_reward": 0.19414883852005005, + "step": 751 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.708333492279053, + "epoch": 0.5863547758284601, + "grad_norm": 29.89352853771839, + "kl": 3.359375, + "learning_rate": 4.931378234632438e-07, + "loss": 0.254, + "reward": 0.21168728172779083, + "reward_std": 0.01946496916934848, + "rewards/accuracy_reward": 0.21168728172779083, + "step": 752 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.187500238418579, + "epoch": 0.5871345029239766, + "grad_norm": 18.12528820630025, + "kl": 3.609375, + "learning_rate": 4.919217438411633e-07, + "loss": 0.2833, + "reward": 0.1397571787238121, + "reward_std": 0.016146399080753326, + "rewards/accuracy_reward": 0.1397571787238121, + "step": 753 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.291666984558105, + "epoch": 0.5879142300194932, + "grad_norm": 106.80881891467668, + "kl": 5.22265625, + "learning_rate": 4.907060953949347e-07, + "loss": 0.666, + "reward": 0.19727333635091782, + "reward_std": 0.02822993416339159, + "rewards/accuracy_reward": 0.19727333635091782, + "step": 754 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.083333492279053, + "epoch": 0.5886939571150097, + "grad_norm": 27.75147661184074, + "kl": 3.3671875, + "learning_rate": 4.894908871495922e-07, + "loss": 0.4653, + "reward": 0.1966698244214058, + "reward_std": 0.01028318889439106, + "rewards/accuracy_reward": 0.1966698244214058, + "step": 755 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.062500476837158, + "epoch": 0.5894736842105263, + "grad_norm": 10.722757471616433, + "kl": 1.58984375, + "learning_rate": 4.882761281269019e-07, + "loss": 0.0505, + "reward": 0.1849554032087326, + "reward_std": 0.009616607800126076, + "rewards/accuracy_reward": 0.1849554032087326, + "step": 756 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.7291669845581055, + "epoch": 0.5902534113060429, + "grad_norm": 102.8928844801119, + "kl": 12.3125, + "learning_rate": 4.870618273452952e-07, + "loss": 0.6361, + "reward": 0.2091982513666153, + "reward_std": 0.04147295653820038, + "rewards/accuracy_reward": 0.2091982513666153, + "step": 757 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.270833969116211, + "epoch": 0.5910331384015595, + "grad_norm": 12.046063435267275, + "kl": 1.85546875, + "learning_rate": 4.858479938198012e-07, + "loss": 0.1541, + "reward": 0.1697048619389534, + "reward_std": 0.016207124979700893, + "rewards/accuracy_reward": 0.1697048619389534, + "step": 758 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.8541669845581055, + "epoch": 0.591812865497076, + "grad_norm": 39.507207248358554, + "kl": 6.3984375, + "learning_rate": 4.8463463656198e-07, + "loss": 0.4891, + "reward": 0.1688900664448738, + "reward_std": 0.040857626125216484, + "rewards/accuracy_reward": 0.1688900664448738, + "step": 759 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.729166746139526, + "epoch": 0.5925925925925926, + "grad_norm": 11.626844783892748, + "kl": 2.19140625, + "learning_rate": 4.834217645798564e-07, + "loss": 0.1351, + "reward": 0.19343341886997223, + "reward_std": 0.01102245319634676, + "rewards/accuracy_reward": 0.19343341886997223, + "step": 760 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.958333492279053, + "epoch": 0.5933723196881091, + "grad_norm": 57.713097720505786, + "kl": 7.0, + "learning_rate": 4.822093868778517e-07, + "loss": 0.5137, + "reward": 0.19546708464622498, + "reward_std": 0.03336811903864145, + "rewards/accuracy_reward": 0.19546708464622498, + "step": 761 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.25, + "epoch": 0.5941520467836258, + "grad_norm": 11.053631456072825, + "kl": 3.890625, + "learning_rate": 4.809975124567184e-07, + "loss": 0.1512, + "reward": 0.17614223062992096, + "reward_std": 0.029737313278019428, + "rewards/accuracy_reward": 0.17614223062992096, + "step": 762 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.875000476837158, + "epoch": 0.5949317738791423, + "grad_norm": 13.964064600395327, + "kl": 5.109375, + "learning_rate": 4.797861503134719e-07, + "loss": 0.4196, + "reward": 0.19560137391090393, + "reward_std": 0.0304703451693058, + "rewards/accuracy_reward": 0.19560137391090393, + "step": 763 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.791666746139526, + "epoch": 0.5957115009746589, + "grad_norm": 5.810858456372481, + "kl": 2.6953125, + "learning_rate": 4.78575309441325e-07, + "loss": 0.0868, + "reward": 0.20079366117715836, + "reward_std": 0.009622504934668541, + "rewards/accuracy_reward": 0.20079366117715836, + "step": 764 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.395833492279053, + "epoch": 0.5964912280701754, + "grad_norm": 1049.0042068010284, + "kl": 59.2265625, + "learning_rate": 4.773649988296201e-07, + "loss": 2.2499, + "reward": 0.21168051660060883, + "reward_std": 0.03866158239543438, + "rewards/accuracy_reward": 0.21168051660060883, + "step": 765 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.416666746139526, + "epoch": 0.597270955165692, + "grad_norm": 21.85254158598344, + "kl": 3.078125, + "learning_rate": 4.7615522746376345e-07, + "loss": 0.2842, + "reward": 0.2145531252026558, + "reward_std": 0.016547015868127346, + "rewards/accuracy_reward": 0.2145531252026558, + "step": 766 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.25, + "epoch": 0.5980506822612086, + "grad_norm": 10.024992837660362, + "kl": 4.03125, + "learning_rate": 4.7494600432515754e-07, + "loss": 0.193, + "reward": 0.17795942723751068, + "reward_std": 0.022083266347181052, + "rewards/accuracy_reward": 0.17795942723751068, + "step": 767 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.000000476837158, + "epoch": 0.5988304093567252, + "grad_norm": 7.8565078659367, + "kl": 1.14453125, + "learning_rate": 4.737373383911347e-07, + "loss": 0.0518, + "reward": 0.1793087124824524, + "reward_std": 0.0058309040032327175, + "rewards/accuracy_reward": 0.1793087124824524, + "step": 768 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.354166984558105, + "epoch": 0.5996101364522417, + "grad_norm": 79.39213074490343, + "kl": 7.0625, + "learning_rate": 4.72529238634891e-07, + "loss": 0.4959, + "reward": 0.16901394724845886, + "reward_std": 0.01649160380475223, + "rewards/accuracy_reward": 0.16901394724845886, + "step": 769 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.791666746139526, + "epoch": 0.6003898635477583, + "grad_norm": 17.942624401665725, + "kl": 4.75, + "learning_rate": 4.7132171402541865e-07, + "loss": 0.1882, + "reward": 0.22621528804302216, + "reward_std": 0.025411183014512062, + "rewards/accuracy_reward": 0.22621528804302216, + "step": 770 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.166666746139526, + "epoch": 0.6011695906432749, + "grad_norm": 29.255201760504846, + "kl": 4.4375, + "learning_rate": 4.7011477352744034e-07, + "loss": 0.3436, + "reward": 0.1923586130142212, + "reward_std": 0.012202110141515732, + "rewards/accuracy_reward": 0.1923586130142212, + "step": 771 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.833333492279053, + "epoch": 0.6019493177387915, + "grad_norm": 59.112660107637076, + "kl": 10.53125, + "learning_rate": 4.689084261013421e-07, + "loss": 0.6029, + "reward": 0.20741159468889236, + "reward_std": 0.02666310779750347, + "rewards/accuracy_reward": 0.20741159468889236, + "step": 772 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.958333492279053, + "epoch": 0.602729044834308, + "grad_norm": 68.00304081397594, + "kl": 10.71875, + "learning_rate": 4.677026807031067e-07, + "loss": 0.55, + "reward": 0.20736417174339294, + "reward_std": 0.03963995724916458, + "rewards/accuracy_reward": 0.20736417174339294, + "step": 773 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.3541669845581055, + "epoch": 0.6035087719298246, + "grad_norm": 15.505441590336915, + "kl": 4.671875, + "learning_rate": 4.664975462842483e-07, + "loss": 0.2714, + "reward": 0.1954365149140358, + "reward_std": 0.007877668365836143, + "rewards/accuracy_reward": 0.1954365149140358, + "step": 774 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.750000476837158, + "epoch": 0.6042884990253411, + "grad_norm": 45.13047924003801, + "kl": 5.09375, + "learning_rate": 4.652930317917443e-07, + "loss": 0.448, + "reward": 0.19026663899421692, + "reward_std": 0.02981062512844801, + "rewards/accuracy_reward": 0.19026663899421692, + "step": 775 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.7916669845581055, + "epoch": 0.6050682261208578, + "grad_norm": 7.958277092307518, + "kl": 1.8984375, + "learning_rate": 4.640891461679698e-07, + "loss": 0.0864, + "reward": 0.19340278208255768, + "reward_std": 0.006357727572321892, + "rewards/accuracy_reward": 0.19340278208255768, + "step": 776 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.916666746139526, + "epoch": 0.6058479532163743, + "grad_norm": 560.4114274304104, + "kl": 19.1328125, + "learning_rate": 4.6288589835063155e-07, + "loss": 1.3135, + "reward": 0.19378283619880676, + "reward_std": 0.017120971344411373, + "rewards/accuracy_reward": 0.19378283619880676, + "step": 777 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.333333492279053, + "epoch": 0.6066276803118908, + "grad_norm": 14.676701775679524, + "kl": 4.375, + "learning_rate": 4.616832972727009e-07, + "loss": 0.3108, + "reward": 0.19689543545246124, + "reward_std": 0.024047876708209515, + "rewards/accuracy_reward": 0.19689543545246124, + "step": 778 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.541666746139526, + "epoch": 0.6074074074074074, + "grad_norm": 28.79449619848036, + "kl": 5.3671875, + "learning_rate": 4.604813518623478e-07, + "loss": 0.1833, + "reward": 0.2140377089381218, + "reward_std": 0.02147880382835865, + "rewards/accuracy_reward": 0.2140377089381218, + "step": 779 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.083333492279053, + "epoch": 0.6081871345029239, + "grad_norm": 3.4839908125831687, + "kl": 3.8359375, + "learning_rate": 4.592800710428747e-07, + "loss": 0.1725, + "reward": 0.2162698581814766, + "reward_std": 0.007877668365836143, + "rewards/accuracy_reward": 0.2162698581814766, + "step": 780 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.375000238418579, + "epoch": 0.6089668615984406, + "grad_norm": 11.339906943788513, + "kl": 2.7265625, + "learning_rate": 4.580794637326497e-07, + "loss": 0.1529, + "reward": 0.195563443005085, + "reward_std": 0.017552451230585575, + "rewards/accuracy_reward": 0.195563443005085, + "step": 781 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.375, + "epoch": 0.6097465886939571, + "grad_norm": 16.706542975142643, + "kl": 3.0859375, + "learning_rate": 4.5687953884504117e-07, + "loss": 0.2591, + "reward": 0.1861708089709282, + "reward_std": 0.033440083265304565, + "rewards/accuracy_reward": 0.1861708089709282, + "step": 782 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.166666746139526, + "epoch": 0.6105263157894737, + "grad_norm": 10.366552350377624, + "kl": 2.984375, + "learning_rate": 4.556803052883508e-07, + "loss": 0.3351, + "reward": 0.1939774826169014, + "reward_std": 0.01127982884645462, + "rewards/accuracy_reward": 0.1939774826169014, + "step": 783 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.1666669845581055, + "epoch": 0.6113060428849902, + "grad_norm": 6.820595275329998, + "kl": 2.48828125, + "learning_rate": 4.54481771965748e-07, + "loss": 0.0946, + "reward": 0.2082669585943222, + "reward_std": 0.013235677033662796, + "rewards/accuracy_reward": 0.2082669585943222, + "step": 784 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.187500476837158, + "epoch": 0.6120857699805068, + "grad_norm": 8.114162777111806, + "kl": 2.3046875, + "learning_rate": 4.532839477752035e-07, + "loss": 0.0906, + "reward": 0.19740156829357147, + "reward_std": 0.014632525853812695, + "rewards/accuracy_reward": 0.19740156829357147, + "step": 785 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.2916669845581055, + "epoch": 0.6128654970760234, + "grad_norm": 51.445169185273556, + "kl": 7.5078125, + "learning_rate": 4.520868416094236e-07, + "loss": 0.3441, + "reward": 0.20311260968446732, + "reward_std": 0.026415514759719372, + "rewards/accuracy_reward": 0.20311260968446732, + "step": 786 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.458333730697632, + "epoch": 0.61364522417154, + "grad_norm": 24.493535986190917, + "kl": 3.625, + "learning_rate": 4.5089046235578345e-07, + "loss": 0.19, + "reward": 0.19780371338129044, + "reward_std": 0.016349583864212036, + "rewards/accuracy_reward": 0.19780371338129044, + "step": 787 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.208333492279053, + "epoch": 0.6144249512670565, + "grad_norm": 6.395559042532656, + "kl": 2.34375, + "learning_rate": 4.4969481889626224e-07, + "loss": 0.0852, + "reward": 0.19479166716337204, + "reward_std": 0.006966505199670792, + "rewards/accuracy_reward": 0.19479166716337204, + "step": 788 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.5416669845581055, + "epoch": 0.6152046783625731, + "grad_norm": 9.290677501031373, + "kl": 3.5390625, + "learning_rate": 4.4849992010737625e-07, + "loss": 0.229, + "reward": 0.17592810839414597, + "reward_std": 0.022496329620480537, + "rewards/accuracy_reward": 0.17592810839414597, + "step": 789 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.916666746139526, + "epoch": 0.6159844054580896, + "grad_norm": 94.37831153244247, + "kl": 8.9296875, + "learning_rate": 4.4730577486011333e-07, + "loss": 0.4132, + "reward": 0.19950980693101883, + "reward_std": 0.03226372506469488, + "rewards/accuracy_reward": 0.19950980693101883, + "step": 790 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.5625, + "epoch": 0.6167641325536063, + "grad_norm": 5.581001454717509, + "kl": 3.0, + "learning_rate": 4.4611239201986685e-07, + "loss": 0.1298, + "reward": 0.20035651326179504, + "reward_std": 0.0013121606316417456, + "rewards/accuracy_reward": 0.20035651326179504, + "step": 791 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.770833492279053, + "epoch": 0.6175438596491228, + "grad_norm": 15.291703216686232, + "kl": 3.1640625, + "learning_rate": 4.4491978044637015e-07, + "loss": 0.2724, + "reward": 0.20971546322107315, + "reward_std": 0.01972260605543852, + "rewards/accuracy_reward": 0.20971546322107315, + "step": 792 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.5416669845581055, + "epoch": 0.6183235867446394, + "grad_norm": 84.93749548444256, + "kl": 13.34375, + "learning_rate": 4.437279489936306e-07, + "loss": 0.5183, + "reward": 0.20228589326143265, + "reward_std": 0.030025272630155087, + "rewards/accuracy_reward": 0.20228589326143265, + "step": 793 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.083333492279053, + "epoch": 0.6191033138401559, + "grad_norm": 29.10800078204113, + "kl": 5.296875, + "learning_rate": 4.4253690650986417e-07, + "loss": 0.4785, + "reward": 0.19759321957826614, + "reward_std": 0.019533390179276466, + "rewards/accuracy_reward": 0.19759321957826614, + "step": 794 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.270833492279053, + "epoch": 0.6198830409356725, + "grad_norm": 383.067637919153, + "kl": 17.21875, + "learning_rate": 4.413466618374291e-07, + "loss": 1.3129, + "reward": 0.19665924459695816, + "reward_std": 0.03253862354904413, + "rewards/accuracy_reward": 0.19665924459695816, + "step": 795 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.7916669845581055, + "epoch": 0.6206627680311891, + "grad_norm": 10.216804573778802, + "kl": 2.7578125, + "learning_rate": 4.4015722381276086e-07, + "loss": 0.1883, + "reward": 0.2028769925236702, + "reward_std": 0.0024056262336671352, + "rewards/accuracy_reward": 0.2028769925236702, + "step": 796 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.395833492279053, + "epoch": 0.6214424951267057, + "grad_norm": 385.8434929742846, + "kl": 25.375, + "learning_rate": 4.389686012663063e-07, + "loss": 1.3037, + "reward": 0.2043689414858818, + "reward_std": 0.023358877166174352, + "rewards/accuracy_reward": 0.2043689414858818, + "step": 797 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.187500238418579, + "epoch": 0.6222222222222222, + "grad_norm": 38.59957111208649, + "kl": 6.484375, + "learning_rate": 4.377808030224581e-07, + "loss": 0.3835, + "reward": 0.2030586376786232, + "reward_std": 0.030849136412143707, + "rewards/accuracy_reward": 0.2030586376786232, + "step": 798 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.333333492279053, + "epoch": 0.6230019493177388, + "grad_norm": 107.42478125369642, + "kl": 9.265625, + "learning_rate": 4.3659383789948934e-07, + "loss": 0.539, + "reward": 0.19165126979351044, + "reward_std": 0.01961417868733406, + "rewards/accuracy_reward": 0.19165126979351044, + "step": 799 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.958333492279053, + "epoch": 0.6237816764132553, + "grad_norm": 19.352124542532792, + "kl": 5.921875, + "learning_rate": 4.35407714709488e-07, + "loss": 0.2386, + "reward": 0.1721200942993164, + "reward_std": 0.047283776104450226, + "rewards/accuracy_reward": 0.1721200942993164, + "step": 800 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.229166746139526, + "epoch": 0.624561403508772, + "grad_norm": 42.39111872358708, + "kl": 5.1875, + "learning_rate": 4.342224422582911e-07, + "loss": 0.3254, + "reward": 0.2216346263885498, + "reward_std": 0.011658036150038242, + "rewards/accuracy_reward": 0.2216346263885498, + "step": 801 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.5, + "epoch": 0.6253411306042885, + "grad_norm": 42.29530611712892, + "kl": 5.296875, + "learning_rate": 4.330380293454202e-07, + "loss": 0.2243, + "reward": 0.19074634462594986, + "reward_std": 0.023415545001626015, + "rewards/accuracy_reward": 0.19074634462594986, + "step": 802 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.25, + "epoch": 0.6261208576998051, + "grad_norm": 19.1864664407353, + "kl": 5.828125, + "learning_rate": 4.318544847640158e-07, + "loss": 0.3136, + "reward": 0.2264232411980629, + "reward_std": 0.0345494095236063, + "rewards/accuracy_reward": 0.2264232411980629, + "step": 803 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.104166984558105, + "epoch": 0.6269005847953216, + "grad_norm": 12.053964076178366, + "kl": 5.96875, + "learning_rate": 4.30671817300771e-07, + "loss": 0.401, + "reward": 0.19188113510608673, + "reward_std": 0.04166575148701668, + "rewards/accuracy_reward": 0.19188113510608673, + "step": 804 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.875000238418579, + "epoch": 0.6276803118908382, + "grad_norm": 15.369913762064051, + "kl": 2.7109375, + "learning_rate": 4.2949003573586785e-07, + "loss": 0.1415, + "reward": 0.1992822140455246, + "reward_std": 0.017546577379107475, + "rewards/accuracy_reward": 0.1992822140455246, + "step": 805 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.312500476837158, + "epoch": 0.6284600389863548, + "grad_norm": 20.08835528033312, + "kl": 3.359375, + "learning_rate": 4.283091488429115e-07, + "loss": 0.3665, + "reward": 0.19808261841535568, + "reward_std": 0.019510054029524326, + "rewards/accuracy_reward": 0.19808261841535568, + "step": 806 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.041666984558105, + "epoch": 0.6292397660818714, + "grad_norm": 14.746151854082994, + "kl": 3.1484375, + "learning_rate": 4.271291653888642e-07, + "loss": 0.2432, + "reward": 0.20466720312833786, + "reward_std": 0.014170052949339151, + "rewards/accuracy_reward": 0.20466720312833786, + "step": 807 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.375000238418579, + "epoch": 0.6300194931773879, + "grad_norm": 9.651169896639045, + "kl": 3.734375, + "learning_rate": 4.259500941339822e-07, + "loss": 0.2547, + "reward": 0.20700864493846893, + "reward_std": 0.014448235742747784, + "rewards/accuracy_reward": 0.20700864493846893, + "step": 808 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.770833492279053, + "epoch": 0.6307992202729045, + "grad_norm": 136.78566738068835, + "kl": 11.65625, + "learning_rate": 4.2477194383174863e-07, + "loss": 0.6354, + "reward": 0.20887459069490433, + "reward_std": 0.02471503522247076, + "rewards/accuracy_reward": 0.20887459069490433, + "step": 809 + }, + { + "clip_ratio": 0.0, + "completion_length": 50.04166793823242, + "epoch": 0.631578947368421, + "grad_norm": 167.52586776433836, + "kl": 8.140625, + "learning_rate": 4.2359472322880973e-07, + "loss": 1.1554, + "reward": 0.20988915115594864, + "reward_std": 0.03373613581061363, + "rewards/accuracy_reward": 0.20988915115594864, + "step": 810 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.729166746139526, + "epoch": 0.6323586744639377, + "grad_norm": 11.30581276764659, + "kl": 3.875, + "learning_rate": 4.2241844106490967e-07, + "loss": 0.1823, + "reward": 0.21582088619470596, + "reward_std": 0.01094868266955018, + "rewards/accuracy_reward": 0.21582088619470596, + "step": 811 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.7916669845581055, + "epoch": 0.6331384015594542, + "grad_norm": 280.4746717335008, + "kl": 15.515625, + "learning_rate": 4.2124310607282543e-07, + "loss": 1.2256, + "reward": 0.20998981595039368, + "reward_std": 0.034942809492349625, + "rewards/accuracy_reward": 0.20998981595039368, + "step": 812 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.208333969116211, + "epoch": 0.6339181286549708, + "grad_norm": 14.719046726718693, + "kl": 2.52734375, + "learning_rate": 4.200687269783023e-07, + "loss": 0.1315, + "reward": 0.1912563219666481, + "reward_std": 0.018876119516789913, + "rewards/accuracy_reward": 0.1912563219666481, + "step": 813 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.083333492279053, + "epoch": 0.6346978557504873, + "grad_norm": 80.5964186128411, + "kl": 12.34375, + "learning_rate": 4.188953124999888e-07, + "loss": 0.8233, + "reward": 0.1867903769016266, + "reward_std": 0.02752780169248581, + "rewards/accuracy_reward": 0.1867903769016266, + "step": 814 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.750000476837158, + "epoch": 0.6354775828460039, + "grad_norm": 39.42691012375579, + "kl": 8.1640625, + "learning_rate": 4.1772287134937223e-07, + "loss": 0.4886, + "reward": 0.19144070148468018, + "reward_std": 0.020385267212986946, + "rewards/accuracy_reward": 0.19144070148468018, + "step": 815 + }, + { + "clip_ratio": 0.0, + "completion_length": 49.41666793823242, + "epoch": 0.6362573099415205, + "grad_norm": 11.547947822180952, + "kl": 4.609375, + "learning_rate": 4.1655141223071366e-07, + "loss": 0.6891, + "reward": 0.20407475531101227, + "reward_std": 0.03265130240470171, + "rewards/accuracy_reward": 0.20407475531101227, + "step": 816 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.187500476837158, + "epoch": 0.6370370370370371, + "grad_norm": 24.576295798710138, + "kl": 7.1015625, + "learning_rate": 4.1538094384098363e-07, + "loss": 0.4078, + "reward": 0.1960323303937912, + "reward_std": 0.025398317025974393, + "rewards/accuracy_reward": 0.1960323303937912, + "step": 817 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.208333492279053, + "epoch": 0.6378167641325536, + "grad_norm": 16.855327869999773, + "kl": 4.71875, + "learning_rate": 4.142114748697979e-07, + "loss": 0.2384, + "reward": 0.21702534705400467, + "reward_std": 0.02624906785786152, + "rewards/accuracy_reward": 0.21702534705400467, + "step": 818 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.145833492279053, + "epoch": 0.6385964912280702, + "grad_norm": 18.606683912213757, + "kl": 4.984375, + "learning_rate": 4.130430139993516e-07, + "loss": 0.3394, + "reward": 0.2314903885126114, + "reward_std": 0.016296265181154013, + "rewards/accuracy_reward": 0.2314903885126114, + "step": 819 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.625000238418579, + "epoch": 0.6393762183235867, + "grad_norm": 35.95326779247107, + "kl": 5.5, + "learning_rate": 4.1187556990435635e-07, + "loss": 0.3098, + "reward": 0.2029413878917694, + "reward_std": 0.01674424426164478, + "rewards/accuracy_reward": 0.2029413878917694, + "step": 820 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.729166984558105, + "epoch": 0.6401559454191034, + "grad_norm": 196.53792273316563, + "kl": 15.46875, + "learning_rate": 4.1070915125197504e-07, + "loss": 0.8768, + "reward": 0.19404908269643784, + "reward_std": 0.02998365741223097, + "rewards/accuracy_reward": 0.19404908269643784, + "step": 821 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.8125, + "epoch": 0.6409356725146199, + "grad_norm": 5.60700152481248, + "kl": 3.3671875, + "learning_rate": 4.0954376670175727e-07, + "loss": 0.1593, + "reward": 0.2088274285197258, + "reward_std": 0.004716914147138596, + "rewards/accuracy_reward": 0.2088274285197258, + "step": 822 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.145833492279053, + "epoch": 0.6417153996101365, + "grad_norm": 112.55388322425284, + "kl": 12.453125, + "learning_rate": 4.0837942490557577e-07, + "loss": 0.6601, + "reward": 0.22587250918149948, + "reward_std": 0.034454891458153725, + "rewards/accuracy_reward": 0.22587250918149948, + "step": 823 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.520833492279053, + "epoch": 0.642495126705653, + "grad_norm": 20.777146687622558, + "kl": 4.109375, + "learning_rate": 4.0721613450756196e-07, + "loss": 0.2603, + "reward": 0.22010359913110733, + "reward_std": 0.02082142001017928, + "rewards/accuracy_reward": 0.22010359913110733, + "step": 824 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.979166746139526, + "epoch": 0.6432748538011696, + "grad_norm": 51.26771572352514, + "kl": 5.203125, + "learning_rate": 4.0605390414404117e-07, + "loss": 0.2251, + "reward": 0.19832590222358704, + "reward_std": 0.017583255423232913, + "rewards/accuracy_reward": 0.19832590222358704, + "step": 825 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.083333492279053, + "epoch": 0.6440545808966862, + "grad_norm": 69.41846787049884, + "kl": 6.06640625, + "learning_rate": 4.0489274244346925e-07, + "loss": 0.2655, + "reward": 0.19951199740171432, + "reward_std": 0.020145550835877657, + "rewards/accuracy_reward": 0.19951199740171432, + "step": 826 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.812500238418579, + "epoch": 0.6448343079922028, + "grad_norm": 23.910114426636852, + "kl": 4.71875, + "learning_rate": 4.037326580263682e-07, + "loss": 0.3594, + "reward": 0.19929338246583939, + "reward_std": 0.019803845789283514, + "rewards/accuracy_reward": 0.19929338246583939, + "step": 827 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.125000238418579, + "epoch": 0.6456140350877193, + "grad_norm": 19.421358737779105, + "kl": 3.2421875, + "learning_rate": 4.0257365950526243e-07, + "loss": 0.1116, + "reward": 0.19143907725811005, + "reward_std": 0.013990202452987432, + "rewards/accuracy_reward": 0.19143907725811005, + "step": 828 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.583333492279053, + "epoch": 0.6463937621832359, + "grad_norm": 7.941813633380404, + "kl": 2.8671875, + "learning_rate": 4.0141575548461403e-07, + "loss": 0.1071, + "reward": 0.2020089402794838, + "reward_std": 0.005412658676505089, + "rewards/accuracy_reward": 0.2020089402794838, + "step": 829 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.666666984558105, + "epoch": 0.6471734892787524, + "grad_norm": 11.737896987154013, + "kl": 3.390625, + "learning_rate": 4.002589545607602e-07, + "loss": 0.1886, + "reward": 0.19675233960151672, + "reward_std": 0.013309437199495733, + "rewards/accuracy_reward": 0.19675233960151672, + "step": 830 + }, + { + "clip_ratio": 0.0, + "completion_length": 17.812500476837158, + "epoch": 0.6479532163742691, + "grad_norm": 18.366980925881403, + "kl": 5.046875, + "learning_rate": 3.99103265321848e-07, + "loss": 0.496, + "reward": 0.20842014253139496, + "reward_std": 0.05011184327304363, + "rewards/accuracy_reward": 0.20842014253139496, + "step": 831 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.020833969116211, + "epoch": 0.6487329434697856, + "grad_norm": 13.405002609765074, + "kl": 2.76171875, + "learning_rate": 3.9794869634777174e-07, + "loss": 0.2071, + "reward": 0.1993274837732315, + "reward_std": 0.012340146116912365, + "rewards/accuracy_reward": 0.1993274837732315, + "step": 832 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.791666746139526, + "epoch": 0.6495126705653022, + "grad_norm": 15.494509868888603, + "kl": 4.234375, + "learning_rate": 3.967952562101088e-07, + "loss": 0.1557, + "reward": 0.22708333283662796, + "reward_std": 0.0051549142226576805, + "rewards/accuracy_reward": 0.22708333283662796, + "step": 833 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.104166746139526, + "epoch": 0.6502923976608187, + "grad_norm": 2.084070941523775, + "kl": 3.0078125, + "learning_rate": 3.956429534720559e-07, + "loss": 0.1195, + "reward": 0.21607143431901932, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.21607143431901932, + "step": 834 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.187500476837158, + "epoch": 0.6510721247563352, + "grad_norm": 8.08640444249246, + "kl": 2.2421875, + "learning_rate": 3.944917966883655e-07, + "loss": 0.1531, + "reward": 0.2024982050061226, + "reward_std": 0.0026334156282246113, + "rewards/accuracy_reward": 0.2024982050061226, + "step": 835 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.312500238418579, + "epoch": 0.6518518518518519, + "grad_norm": 13.786405653279184, + "kl": 3.6796875, + "learning_rate": 3.933417944052828e-07, + "loss": 0.1478, + "reward": 0.22604165971279144, + "reward_std": 0.01539972797036171, + "rewards/accuracy_reward": 0.22604165971279144, + "step": 836 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.041666746139526, + "epoch": 0.6526315789473685, + "grad_norm": 1.1748814247215527, + "kl": 3.6015625, + "learning_rate": 3.9219295516048155e-07, + "loss": 0.1442, + "reward": 0.19485295563936234, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.19485295563936234, + "step": 837 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.145833492279053, + "epoch": 0.653411306042885, + "grad_norm": 2237.912229922971, + "kl": 139.1875, + "learning_rate": 3.910452874830015e-07, + "loss": 4.8967, + "reward": 0.1780773475766182, + "reward_std": 0.03933209553360939, + "rewards/accuracy_reward": 0.1780773475766182, + "step": 838 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.6666669845581055, + "epoch": 0.6541910331384015, + "grad_norm": 10.473272553926513, + "kl": 4.1953125, + "learning_rate": 3.8989879989318407e-07, + "loss": 0.1865, + "reward": 0.20865502208471298, + "reward_std": 0.023965464439243078, + "rewards/accuracy_reward": 0.20865502208471298, + "step": 839 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.604166746139526, + "epoch": 0.6549707602339181, + "grad_norm": 83.32627325211686, + "kl": 8.25, + "learning_rate": 3.8875350090261014e-07, + "loss": 0.3729, + "reward": 0.21747076511383057, + "reward_std": 0.016058134846389294, + "rewards/accuracy_reward": 0.21747076511383057, + "step": 840 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.4791669845581055, + "epoch": 0.6557504873294348, + "grad_norm": 58.93143334333492, + "kl": 7.03125, + "learning_rate": 3.8760939901403613e-07, + "loss": 0.5712, + "reward": 0.21835222840309143, + "reward_std": 0.030122676864266396, + "rewards/accuracy_reward": 0.21835222840309143, + "step": 841 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.229166746139526, + "epoch": 0.6565302144249513, + "grad_norm": 71.40128600202888, + "kl": 8.390625, + "learning_rate": 3.864665027213311e-07, + "loss": 0.5488, + "reward": 0.20483766496181488, + "reward_std": 0.023112010210752487, + "rewards/accuracy_reward": 0.20483766496181488, + "step": 842 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.916666984558105, + "epoch": 0.6573099415204678, + "grad_norm": 81.00529931390395, + "kl": 12.046875, + "learning_rate": 3.8532482050941374e-07, + "loss": 0.7515, + "reward": 0.186067596077919, + "reward_std": 0.03991219587624073, + "rewards/accuracy_reward": 0.186067596077919, + "step": 843 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.000000238418579, + "epoch": 0.6580896686159844, + "grad_norm": 28.540090037116666, + "kl": 5.8125, + "learning_rate": 3.8418436085418935e-07, + "loss": 0.3534, + "reward": 0.1989043578505516, + "reward_std": 0.026317469775676727, + "rewards/accuracy_reward": 0.1989043578505516, + "step": 844 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.375000476837158, + "epoch": 0.6588693957115009, + "grad_norm": 49.22818092955027, + "kl": 10.984375, + "learning_rate": 3.8304513222248713e-07, + "loss": 0.4965, + "reward": 0.18014463782310486, + "reward_std": 0.014690164476633072, + "rewards/accuracy_reward": 0.18014463782310486, + "step": 845 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.2916669845581055, + "epoch": 0.6596491228070176, + "grad_norm": 41.01620676230909, + "kl": 5.2421875, + "learning_rate": 3.8190714307199655e-07, + "loss": 0.353, + "reward": 0.20514292269945145, + "reward_std": 0.01859601680189371, + "rewards/accuracy_reward": 0.20514292269945145, + "step": 846 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.145833730697632, + "epoch": 0.6604288499025341, + "grad_norm": 15.50305603216997, + "kl": 4.171875, + "learning_rate": 3.807704018512059e-07, + "loss": 0.2939, + "reward": 0.2043001651763916, + "reward_std": 0.023173989728093147, + "rewards/accuracy_reward": 0.2043001651763916, + "step": 847 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.895833492279053, + "epoch": 0.6612085769980507, + "grad_norm": 9.893991042900543, + "kl": 3.4296875, + "learning_rate": 3.79634916999338e-07, + "loss": 0.1809, + "reward": 0.21078431606292725, + "reward_std": 0.005943312309682369, + "rewards/accuracy_reward": 0.21078431606292725, + "step": 848 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.520833492279053, + "epoch": 0.6619883040935672, + "grad_norm": 31.156907102921853, + "kl": 5.1015625, + "learning_rate": 3.7850069694628884e-07, + "loss": 0.3965, + "reward": 0.20739000290632248, + "reward_std": 0.025404288433492184, + "rewards/accuracy_reward": 0.20739000290632248, + "step": 849 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.083333492279053, + "epoch": 0.6627680311890838, + "grad_norm": 11.988670132221133, + "kl": 4.5625, + "learning_rate": 3.7736775011256415e-07, + "loss": 0.1814, + "reward": 0.20704366266727448, + "reward_std": 0.020715244114398956, + "rewards/accuracy_reward": 0.20704366266727448, + "step": 850 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.729166746139526, + "epoch": 0.6635477582846004, + "grad_norm": 15.094477497666018, + "kl": 5.875, + "learning_rate": 3.762360849092175e-07, + "loss": 0.2685, + "reward": 0.1950061321258545, + "reward_std": 0.027128692716360092, + "rewards/accuracy_reward": 0.1950061321258545, + "step": 851 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.666666984558105, + "epoch": 0.664327485380117, + "grad_norm": 47.49385296307977, + "kl": 5.59375, + "learning_rate": 3.7510570973778743e-07, + "loss": 0.5546, + "reward": 0.19470486044883728, + "reward_std": 0.04438193514943123, + "rewards/accuracy_reward": 0.19470486044883728, + "step": 852 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.041666984558105, + "epoch": 0.6651072124756335, + "grad_norm": 49.403938914448794, + "kl": 5.109375, + "learning_rate": 3.7397663299023507e-07, + "loss": 0.5808, + "reward": 0.18113365024328232, + "reward_std": 0.027653340250253677, + "rewards/accuracy_reward": 0.18113365024328232, + "step": 853 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.833333730697632, + "epoch": 0.6658869395711501, + "grad_norm": 115.05846461575804, + "kl": 5.15625, + "learning_rate": 3.728488630488823e-07, + "loss": 0.1882, + "reward": 0.17945753037929535, + "reward_std": 0.01749674673192203, + "rewards/accuracy_reward": 0.17945753037929535, + "step": 854 + }, + { + "clip_ratio": 0.0, + "completion_length": 21.562500715255737, + "epoch": 0.6666666666666666, + "grad_norm": 19.08448230040922, + "kl": 4.859375, + "learning_rate": 3.7172240828634907e-07, + "loss": 0.5295, + "reward": 0.2150610387325287, + "reward_std": 0.02549346862360835, + "rewards/accuracy_reward": 0.2150610387325287, + "step": 855 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.9791669845581055, + "epoch": 0.6674463937621833, + "grad_norm": 583.5786239380601, + "kl": 23.7421875, + "learning_rate": 3.705972770654912e-07, + "loss": 1.3665, + "reward": 0.2295539528131485, + "reward_std": 0.02044660970568657, + "rewards/accuracy_reward": 0.2295539528131485, + "step": 856 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.187500476837158, + "epoch": 0.6682261208576998, + "grad_norm": 11.440404797312098, + "kl": 3.9921875, + "learning_rate": 3.694734777393387e-07, + "loss": 0.2865, + "reward": 0.21353499591350555, + "reward_std": 0.014220003969967365, + "rewards/accuracy_reward": 0.21353499591350555, + "step": 857 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.4166669845581055, + "epoch": 0.6690058479532164, + "grad_norm": 18.288574984719386, + "kl": 5.3515625, + "learning_rate": 3.6835101865103347e-07, + "loss": 0.2077, + "reward": 0.21395864337682724, + "reward_std": 0.029082434251904488, + "rewards/accuracy_reward": 0.21395864337682724, + "step": 858 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.833333492279053, + "epoch": 0.6697855750487329, + "grad_norm": 99.72355524750172, + "kl": 7.65625, + "learning_rate": 3.6722990813376744e-07, + "loss": 0.7471, + "reward": 0.19210276752710342, + "reward_std": 0.030884217470884323, + "rewards/accuracy_reward": 0.19210276752710342, + "step": 859 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.1875, + "epoch": 0.6705653021442495, + "grad_norm": 62.567748839286416, + "kl": 6.34375, + "learning_rate": 3.661101545107207e-07, + "loss": 0.6136, + "reward": 0.1789841502904892, + "reward_std": 0.044487953186035156, + "rewards/accuracy_reward": 0.1789841502904892, + "step": 860 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.8541669845581055, + "epoch": 0.6713450292397661, + "grad_norm": 9.421069945210936, + "kl": 3.1484375, + "learning_rate": 3.649917660949995e-07, + "loss": 0.1673, + "reward": 0.2123817279934883, + "reward_std": 0.012781540397554636, + "rewards/accuracy_reward": 0.2123817279934883, + "step": 861 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.104166746139526, + "epoch": 0.6721247563352827, + "grad_norm": 11.37393268822973, + "kl": 3.3671875, + "learning_rate": 3.638747511895748e-07, + "loss": 0.1343, + "reward": 0.20081845670938492, + "reward_std": 0.020817924290895462, + "rewards/accuracy_reward": 0.20081845670938492, + "step": 862 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.229166984558105, + "epoch": 0.6729044834307992, + "grad_norm": 19.10937238360906, + "kl": 4.328125, + "learning_rate": 3.627591180872209e-07, + "loss": 0.2971, + "reward": 0.19378720968961716, + "reward_std": 0.033046112395823, + "rewards/accuracy_reward": 0.19378720968961716, + "step": 863 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.166666984558105, + "epoch": 0.6736842105263158, + "grad_norm": 16.243544201159025, + "kl": 5.0, + "learning_rate": 3.616448750704529e-07, + "loss": 0.294, + "reward": 0.2008969932794571, + "reward_std": 0.025891023222357035, + "rewards/accuracy_reward": 0.2008969932794571, + "step": 864 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.375000476837158, + "epoch": 0.6744639376218323, + "grad_norm": 36.36679969189664, + "kl": 6.515625, + "learning_rate": 3.605320304114665e-07, + "loss": 0.2803, + "reward": 0.18615100532770157, + "reward_std": 0.026490109972655773, + "rewards/accuracy_reward": 0.18615100532770157, + "step": 865 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.5, + "epoch": 0.675243664717349, + "grad_norm": 10.953750395305315, + "kl": 4.046875, + "learning_rate": 3.594205923720754e-07, + "loss": 0.2638, + "reward": 0.2095734253525734, + "reward_std": 0.026855789124965668, + "rewards/accuracy_reward": 0.2095734253525734, + "step": 866 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.75, + "epoch": 0.6760233918128655, + "grad_norm": 19.11690069447073, + "kl": 5.9921875, + "learning_rate": 3.583105692036508e-07, + "loss": 0.2916, + "reward": 0.17860960215330124, + "reward_std": 0.014079881832003593, + "rewards/accuracy_reward": 0.17860960215330124, + "step": 867 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.416666984558105, + "epoch": 0.6768031189083821, + "grad_norm": 26.351598894194492, + "kl": 5.15625, + "learning_rate": 3.572019691470596e-07, + "loss": 0.3035, + "reward": 0.18929192423820496, + "reward_std": 0.01562455779640004, + "rewards/accuracy_reward": 0.18929192423820496, + "step": 868 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.708333730697632, + "epoch": 0.6775828460038986, + "grad_norm": 24.05928915180133, + "kl": 5.03125, + "learning_rate": 3.5609480043260355e-07, + "loss": 0.3947, + "reward": 0.20867124199867249, + "reward_std": 0.02751572383567691, + "rewards/accuracy_reward": 0.20867124199867249, + "step": 869 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.520833492279053, + "epoch": 0.6783625730994152, + "grad_norm": 10.54519917794754, + "kl": 4.078125, + "learning_rate": 3.5498907127995804e-07, + "loss": 0.2366, + "reward": 0.1898021697998047, + "reward_std": 0.014521957375109196, + "rewards/accuracy_reward": 0.1898021697998047, + "step": 870 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.3541669845581055, + "epoch": 0.6791423001949318, + "grad_norm": 9.150597797246903, + "kl": 3.984375, + "learning_rate": 3.538847898981112e-07, + "loss": 0.193, + "reward": 0.2298201099038124, + "reward_std": 0.009535149089060724, + "rewards/accuracy_reward": 0.2298201099038124, + "step": 871 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.187500238418579, + "epoch": 0.6799220272904484, + "grad_norm": 11.265811806168355, + "kl": 4.4921875, + "learning_rate": 3.527819644853026e-07, + "loss": 0.3085, + "reward": 0.18132660537958145, + "reward_std": 0.013307837769389153, + "rewards/accuracy_reward": 0.18132660537958145, + "step": 872 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.270833730697632, + "epoch": 0.6807017543859649, + "grad_norm": 27.280552585994847, + "kl": 6.296875, + "learning_rate": 3.5168060322896296e-07, + "loss": 0.4025, + "reward": 0.1876101717352867, + "reward_std": 0.030521446838974953, + "rewards/accuracy_reward": 0.1876101717352867, + "step": 873 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.375000238418579, + "epoch": 0.6814814814814815, + "grad_norm": 29.09441002295356, + "kl": 5.125, + "learning_rate": 3.5058071430565264e-07, + "loss": 0.309, + "reward": 0.18311680853366852, + "reward_std": 0.01297963922843337, + "rewards/accuracy_reward": 0.18311680853366852, + "step": 874 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.479166984558105, + "epoch": 0.682261208576998, + "grad_norm": 17.95798638973203, + "kl": 4.703125, + "learning_rate": 3.4948230588100154e-07, + "loss": 0.4875, + "reward": 0.2000204399228096, + "reward_std": 0.03611239977180958, + "rewards/accuracy_reward": 0.2000204399228096, + "step": 875 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.062500476837158, + "epoch": 0.6830409356725147, + "grad_norm": 16.598550671346135, + "kl": 4.96875, + "learning_rate": 3.4838538610964805e-07, + "loss": 0.241, + "reward": 0.18047089129686356, + "reward_std": 0.03364178724586964, + "rewards/accuracy_reward": 0.18047089129686356, + "step": 876 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.083333730697632, + "epoch": 0.6838206627680312, + "grad_norm": 10.380660860394476, + "kl": 2.2578125, + "learning_rate": 3.4728996313517937e-07, + "loss": 0.169, + "reward": 0.211860790848732, + "reward_std": 0.0022142704110592604, + "rewards/accuracy_reward": 0.211860790848732, + "step": 877 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.354166984558105, + "epoch": 0.6846003898635478, + "grad_norm": 14.396135490520734, + "kl": 3.9296875, + "learning_rate": 3.4619604509006974e-07, + "loss": 0.2923, + "reward": 0.19804134964942932, + "reward_std": 0.023806564509868622, + "rewards/accuracy_reward": 0.19804134964942932, + "step": 878 + }, + { + "clip_ratio": 0.0, + "completion_length": 11.479166984558105, + "epoch": 0.6853801169590643, + "grad_norm": 12.399009445589902, + "kl": 4.0859375, + "learning_rate": 3.451036400956208e-07, + "loss": 0.2746, + "reward": 0.18712309002876282, + "reward_std": 0.02333628199994564, + "rewards/accuracy_reward": 0.18712309002876282, + "step": 879 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.729166984558105, + "epoch": 0.6861598440545809, + "grad_norm": 11.78205400828776, + "kl": 3.6015625, + "learning_rate": 3.4401275626190144e-07, + "loss": 0.148, + "reward": 0.17492123693227768, + "reward_std": 0.014532366767525673, + "rewards/accuracy_reward": 0.17492123693227768, + "step": 880 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.145833730697632, + "epoch": 0.6869395711500975, + "grad_norm": 338.0151734976019, + "kl": 24.6875, + "learning_rate": 3.429234016876874e-07, + "loss": 0.6819, + "reward": 0.21351857483386993, + "reward_std": 0.03548992797732353, + "rewards/accuracy_reward": 0.21351857483386993, + "step": 881 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.812500476837158, + "epoch": 0.6877192982456141, + "grad_norm": 151.99267479623956, + "kl": 10.328125, + "learning_rate": 3.4183558446040096e-07, + "loss": 0.8245, + "reward": 0.1790117770433426, + "reward_std": 0.027704279869794846, + "rewards/accuracy_reward": 0.1790117770433426, + "step": 882 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.166666984558105, + "epoch": 0.6884990253411306, + "grad_norm": 13.12288886853419, + "kl": 3.046875, + "learning_rate": 3.40749312656051e-07, + "loss": 0.2597, + "reward": 0.1871459111571312, + "reward_std": 0.020442471373826265, + "rewards/accuracy_reward": 0.1871459111571312, + "step": 883 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.708333969116211, + "epoch": 0.6892787524366472, + "grad_norm": 125.94566298406964, + "kl": 8.29296875, + "learning_rate": 3.396645943391734e-07, + "loss": 0.5188, + "reward": 0.1927696168422699, + "reward_std": 0.010672137141227722, + "rewards/accuracy_reward": 0.1927696168422699, + "step": 884 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.270833492279053, + "epoch": 0.6900584795321637, + "grad_norm": 18.80558081305964, + "kl": 4.96875, + "learning_rate": 3.3858143756277057e-07, + "loss": 0.3059, + "reward": 0.20359624177217484, + "reward_std": 0.012285875156521797, + "rewards/accuracy_reward": 0.20359624177217484, + "step": 885 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.083333492279053, + "epoch": 0.6908382066276804, + "grad_norm": 9.88574551897838, + "kl": 3.96875, + "learning_rate": 3.374998503682522e-07, + "loss": 0.2362, + "reward": 0.22083333134651184, + "reward_std": 0.014433756470680237, + "rewards/accuracy_reward": 0.22083333134651184, + "step": 886 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.3125, + "epoch": 0.6916179337231969, + "grad_norm": 956.8158767549055, + "kl": 55.71484375, + "learning_rate": 3.3641984078537524e-07, + "loss": 3.2511, + "reward": 0.17928192764520645, + "reward_std": 0.010196511633694172, + "rewards/accuracy_reward": 0.17928192764520645, + "step": 887 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.270833492279053, + "epoch": 0.6923976608187135, + "grad_norm": 22.18857442314468, + "kl": 4.21875, + "learning_rate": 3.353414168321842e-07, + "loss": 0.3368, + "reward": 0.2063341811299324, + "reward_std": 0.01536745554767549, + "rewards/accuracy_reward": 0.2063341811299324, + "step": 888 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.166666746139526, + "epoch": 0.69317738791423, + "grad_norm": 13.289208716880553, + "kl": 3.53125, + "learning_rate": 3.34264586514952e-07, + "loss": 0.1577, + "reward": 0.21908482909202576, + "reward_std": 0.001933091669343412, + "rewards/accuracy_reward": 0.21908482909202576, + "step": 889 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.833333492279053, + "epoch": 0.6939571150097466, + "grad_norm": 16.414544697328814, + "kl": 5.875, + "learning_rate": 3.331893578281203e-07, + "loss": 0.3265, + "reward": 0.21650509536266327, + "reward_std": 0.026580046862363815, + "rewards/accuracy_reward": 0.21650509536266327, + "step": 890 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.875000238418579, + "epoch": 0.6947368421052632, + "grad_norm": 55.329987616850566, + "kl": 6.3984375, + "learning_rate": 3.3211573875424005e-07, + "loss": 0.3431, + "reward": 0.19722440838813782, + "reward_std": 0.0016096474137157202, + "rewards/accuracy_reward": 0.19722440838813782, + "step": 891 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.2291669845581055, + "epoch": 0.6955165692007798, + "grad_norm": 16.370349301666863, + "kl": 5.140625, + "learning_rate": 3.310437372639128e-07, + "loss": 0.4026, + "reward": 0.20340585708618164, + "reward_std": 0.02194612193852663, + "rewards/accuracy_reward": 0.20340585708618164, + "step": 892 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.041666746139526, + "epoch": 0.6962962962962963, + "grad_norm": 14.943007133298616, + "kl": 5.1875, + "learning_rate": 3.2997336131573047e-07, + "loss": 0.2734, + "reward": 0.20707210153341293, + "reward_std": 0.016659541055560112, + "rewards/accuracy_reward": 0.20707210153341293, + "step": 893 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.4166669845581055, + "epoch": 0.6970760233918128, + "grad_norm": 29.232016253147407, + "kl": 5.71875, + "learning_rate": 3.289046188562171e-07, + "loss": 0.2327, + "reward": 0.2031308338046074, + "reward_std": 0.039826130494475365, + "rewards/accuracy_reward": 0.2031308338046074, + "step": 894 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.520833492279053, + "epoch": 0.6978557504873294, + "grad_norm": 25.562862068261172, + "kl": 6.53125, + "learning_rate": 3.2783751781976986e-07, + "loss": 0.3669, + "reward": 0.1791870892047882, + "reward_std": 0.03576790168881416, + "rewards/accuracy_reward": 0.1791870892047882, + "step": 895 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.6875, + "epoch": 0.698635477582846, + "grad_norm": 15.20260004328982, + "kl": 5.8125, + "learning_rate": 3.2677206612859954e-07, + "loss": 0.2967, + "reward": 0.20109273493289948, + "reward_std": 0.021647372283041477, + "rewards/accuracy_reward": 0.20109273493289948, + "step": 896 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.375000238418579, + "epoch": 0.6994152046783626, + "grad_norm": 6.366388476173305, + "kl": 2.8671875, + "learning_rate": 3.257082716926724e-07, + "loss": 0.1154, + "reward": 0.20592321455478668, + "reward_std": 0.0024056266993284225, + "rewards/accuracy_reward": 0.20592321455478668, + "step": 897 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.895833492279053, + "epoch": 0.7001949317738791, + "grad_norm": 9.449159323940716, + "kl": 3.9296875, + "learning_rate": 3.2464614240965104e-07, + "loss": 0.197, + "reward": 0.22831101715564728, + "reward_std": 0.0009021099540404975, + "rewards/accuracy_reward": 0.22831101715564728, + "step": 898 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.395833492279053, + "epoch": 0.7009746588693957, + "grad_norm": 814.5576645812628, + "kl": 27.8125, + "learning_rate": 3.235856861648358e-07, + "loss": 1.819, + "reward": 0.19169730693101883, + "reward_std": 0.04360729083418846, + "rewards/accuracy_reward": 0.19169730693101883, + "step": 899 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.750000476837158, + "epoch": 0.7017543859649122, + "grad_norm": 13.513138561797795, + "kl": 4.1796875, + "learning_rate": 3.225269108311066e-07, + "loss": 0.3896, + "reward": 0.2200138419866562, + "reward_std": 0.01727256504818797, + "rewards/accuracy_reward": 0.2200138419866562, + "step": 900 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.937500238418579, + "epoch": 0.7025341130604289, + "grad_norm": 15.221752544440637, + "kl": 6.09375, + "learning_rate": 3.214698242688638e-07, + "loss": 0.443, + "reward": 0.21599837392568588, + "reward_std": 0.03508717566728592, + "rewards/accuracy_reward": 0.21599837392568588, + "step": 901 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.604166984558105, + "epoch": 0.7033138401559454, + "grad_norm": 24.263134555368826, + "kl": 9.59375, + "learning_rate": 3.204144343259705e-07, + "loss": 0.5248, + "reward": 0.19001584500074387, + "reward_std": 0.051292162388563156, + "rewards/accuracy_reward": 0.19001584500074387, + "step": 902 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.416666984558105, + "epoch": 0.704093567251462, + "grad_norm": 7.008272689538141, + "kl": 4.2109375, + "learning_rate": 3.193607488376938e-07, + "loss": 0.1621, + "reward": 0.2144649550318718, + "reward_std": 0.01635934691876173, + "rewards/accuracy_reward": 0.2144649550318718, + "step": 903 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.812500238418579, + "epoch": 0.7048732943469785, + "grad_norm": 13.788912809415534, + "kl": 7.625, + "learning_rate": 3.1830877562664715e-07, + "loss": 0.3442, + "reward": 0.2130710855126381, + "reward_std": 0.030132445972412825, + "rewards/accuracy_reward": 0.2130710855126381, + "step": 904 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.166666984558105, + "epoch": 0.7056530214424951, + "grad_norm": 27.580211413108927, + "kl": 4.1640625, + "learning_rate": 3.172585225027318e-07, + "loss": 0.3393, + "reward": 0.1855800747871399, + "reward_std": 0.0070287775015458465, + "rewards/accuracy_reward": 0.1855800747871399, + "step": 905 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.7916669845581055, + "epoch": 0.7064327485380117, + "grad_norm": 107.79419946530524, + "kl": 15.25, + "learning_rate": 3.162099972630787e-07, + "loss": 0.7049, + "reward": 0.17611265927553177, + "reward_std": 0.03691624291241169, + "rewards/accuracy_reward": 0.17611265927553177, + "step": 906 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.604166746139526, + "epoch": 0.7072124756335283, + "grad_norm": 13.963377831608838, + "kl": 4.34375, + "learning_rate": 3.1516320769199157e-07, + "loss": 0.254, + "reward": 0.20359988510608673, + "reward_std": 0.016609287995379418, + "rewards/accuracy_reward": 0.20359988510608673, + "step": 907 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.958333492279053, + "epoch": 0.7079922027290448, + "grad_norm": 17.817515687699842, + "kl": 5.46875, + "learning_rate": 3.1411816156088786e-07, + "loss": 0.3754, + "reward": 0.22689075767993927, + "reward_std": 0.02558915875852108, + "rewards/accuracy_reward": 0.22689075767993927, + "step": 908 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.645833730697632, + "epoch": 0.7087719298245614, + "grad_norm": 10.694574590280249, + "kl": 3.484375, + "learning_rate": 3.13074866628242e-07, + "loss": 0.1257, + "reward": 0.2113136574625969, + "reward_std": 0.012055137194693089, + "rewards/accuracy_reward": 0.2113136574625969, + "step": 909 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.854166746139526, + "epoch": 0.7095516569200779, + "grad_norm": 11.213379393985633, + "kl": 5.203125, + "learning_rate": 3.1203333063952706e-07, + "loss": 0.1745, + "reward": 0.22336310148239136, + "reward_std": 0.018042195588350296, + "rewards/accuracy_reward": 0.22336310148239136, + "step": 910 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.770833492279053, + "epoch": 0.7103313840155946, + "grad_norm": 6.2183027038896315, + "kl": 4.484375, + "learning_rate": 3.109935613271579e-07, + "loss": 0.1723, + "reward": 0.2154718115925789, + "reward_std": 0.006640276871621609, + "rewards/accuracy_reward": 0.2154718115925789, + "step": 911 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.187500238418579, + "epoch": 0.7111111111111111, + "grad_norm": 17.318399561017525, + "kl": 4.8515625, + "learning_rate": 3.0995556641043353e-07, + "loss": 0.2518, + "reward": 0.19732142984867096, + "reward_std": 0.021650634706020355, + "rewards/accuracy_reward": 0.19732142984867096, + "step": 912 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.562500476837158, + "epoch": 0.7118908382066277, + "grad_norm": 38.39954013464861, + "kl": 8.15625, + "learning_rate": 3.0891935359547954e-07, + "loss": 0.3732, + "reward": 0.18085789680480957, + "reward_std": 0.024646949023008347, + "rewards/accuracy_reward": 0.18085789680480957, + "step": 913 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.6666669845581055, + "epoch": 0.7126705653021442, + "grad_norm": 20.332917727034783, + "kl": 4.6484375, + "learning_rate": 3.078849305751913e-07, + "loss": 0.2587, + "reward": 0.2028120830655098, + "reward_std": 0.005516134202480316, + "rewards/accuracy_reward": 0.2028120830655098, + "step": 914 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.6041669845581055, + "epoch": 0.7134502923976608, + "grad_norm": 2.159210519753522, + "kl": 3.6328125, + "learning_rate": 3.0685230502917644e-07, + "loss": 0.1445, + "reward": 0.25, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.25, + "step": 915 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.5416669845581055, + "epoch": 0.7142300194931774, + "grad_norm": 15.310858422287343, + "kl": 4.0, + "learning_rate": 3.0582148462369826e-07, + "loss": 0.2431, + "reward": 0.2052827551960945, + "reward_std": 0.006443641148507595, + "rewards/accuracy_reward": 0.2052827551960945, + "step": 916 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.041666746139526, + "epoch": 0.715009746588694, + "grad_norm": 20.393041611437226, + "kl": 5.9375, + "learning_rate": 3.047924770116186e-07, + "loss": 0.3657, + "reward": 0.18276400119066238, + "reward_std": 0.023592438083142042, + "rewards/accuracy_reward": 0.18276400119066238, + "step": 917 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.3125, + "epoch": 0.7157894736842105, + "grad_norm": 1496.820887608113, + "kl": 36.15625, + "learning_rate": 3.037652898323408e-07, + "loss": 3.3766, + "reward": 0.21443071961402893, + "reward_std": 0.04821654036641121, + "rewards/accuracy_reward": 0.21443071961402893, + "step": 918 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.937500238418579, + "epoch": 0.7165692007797271, + "grad_norm": 15.71162748438164, + "kl": 4.984375, + "learning_rate": 3.027399307117535e-07, + "loss": 0.3712, + "reward": 0.19430500268936157, + "reward_std": 0.02692793821915984, + "rewards/accuracy_reward": 0.19430500268936157, + "step": 919 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.250000238418579, + "epoch": 0.7173489278752436, + "grad_norm": 37.41427053580005, + "kl": 7.28125, + "learning_rate": 3.017164072621735e-07, + "loss": 0.6852, + "reward": 0.20742052048444748, + "reward_std": 0.03735269792377949, + "rewards/accuracy_reward": 0.20742052048444748, + "step": 920 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.270833492279053, + "epoch": 0.7181286549707603, + "grad_norm": 22.46056149415884, + "kl": 6.1484375, + "learning_rate": 3.006947270822898e-07, + "loss": 0.4526, + "reward": 0.1538393869996071, + "reward_std": 0.039797255769371986, + "rewards/accuracy_reward": 0.1538393869996071, + "step": 921 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.2291669845581055, + "epoch": 0.7189083820662768, + "grad_norm": 35.627087156072264, + "kl": 9.3125, + "learning_rate": 2.996748977571067e-07, + "loss": 0.3437, + "reward": 0.2000187262892723, + "reward_std": 0.05268122907727957, + "rewards/accuracy_reward": 0.2000187262892723, + "step": 922 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.7291669845581055, + "epoch": 0.7196881091617934, + "grad_norm": 20.407968793066914, + "kl": 5.921875, + "learning_rate": 2.986569268578875e-07, + "loss": 0.2892, + "reward": 0.20979664474725723, + "reward_std": 0.030791686847805977, + "rewards/accuracy_reward": 0.20979664474725723, + "step": 923 + }, + { + "clip_ratio": 0.0, + "completion_length": 14.645833730697632, + "epoch": 0.7204678362573099, + "grad_norm": 68.53982486823352, + "kl": 8.515625, + "learning_rate": 2.976408219420986e-07, + "loss": 0.7908, + "reward": 0.19621873646974564, + "reward_std": 0.041535016149282455, + "rewards/accuracy_reward": 0.19621873646974564, + "step": 924 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.000000476837158, + "epoch": 0.7212475633528265, + "grad_norm": 11.50355764746805, + "kl": 2.875, + "learning_rate": 2.966265905533534e-07, + "loss": 0.1718, + "reward": 0.18029513955116272, + "reward_std": 0.0033077369444072247, + "rewards/accuracy_reward": 0.18029513955116272, + "step": 925 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.270833492279053, + "epoch": 0.7220272904483431, + "grad_norm": 7.8761482950894015, + "kl": 3.6953125, + "learning_rate": 2.9561424022135577e-07, + "loss": 0.1141, + "reward": 0.19398538768291473, + "reward_std": 0.016037508845329285, + "rewards/accuracy_reward": 0.19398538768291473, + "step": 926 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.145833969116211, + "epoch": 0.7228070175438597, + "grad_norm": 16.11975226620772, + "kl": 3.3828125, + "learning_rate": 2.946037784618449e-07, + "loss": 0.2142, + "reward": 0.1789299175143242, + "reward_std": 0.0071819748263806105, + "rewards/accuracy_reward": 0.1789299175143242, + "step": 927 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.250000476837158, + "epoch": 0.7235867446393762, + "grad_norm": 741.3386308381091, + "kl": 21.2265625, + "learning_rate": 2.93595212776539e-07, + "loss": 0.8527, + "reward": 0.20706408470869064, + "reward_std": 0.008988404646515846, + "rewards/accuracy_reward": 0.20706408470869064, + "step": 928 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.312500238418579, + "epoch": 0.7243664717348928, + "grad_norm": 10.027931688039331, + "kl": 3.9609375, + "learning_rate": 2.9258855065307954e-07, + "loss": 0.1904, + "reward": 0.19589121639728546, + "reward_std": 0.012564446311444044, + "rewards/accuracy_reward": 0.19589121639728546, + "step": 929 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.770833492279053, + "epoch": 0.7251461988304093, + "grad_norm": 31.635156071602413, + "kl": 7.125, + "learning_rate": 2.9158379956497615e-07, + "loss": 0.4183, + "reward": 0.17945359647274017, + "reward_std": 0.04685807507485151, + "rewards/accuracy_reward": 0.17945359647274017, + "step": 930 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.583333730697632, + "epoch": 0.725925925925926, + "grad_norm": 17.605759226467647, + "kl": 5.859375, + "learning_rate": 2.9058096697155056e-07, + "loss": 0.4898, + "reward": 0.16475240886211395, + "reward_std": 0.04607238993048668, + "rewards/accuracy_reward": 0.16475240886211395, + "step": 931 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.083333492279053, + "epoch": 0.7267056530214425, + "grad_norm": 6.81837736998544, + "kl": 2.58984375, + "learning_rate": 2.8958006031788187e-07, + "loss": 0.1037, + "reward": 0.19114582240581512, + "reward_std": 0.006082022562623024, + "rewards/accuracy_reward": 0.19114582240581512, + "step": 932 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.0, + "epoch": 0.7274853801169591, + "grad_norm": 9.799916814961083, + "kl": 3.6953125, + "learning_rate": 2.885810870347503e-07, + "loss": 0.1867, + "reward": 0.21904762089252472, + "reward_std": 0.0013901765923947096, + "rewards/accuracy_reward": 0.21904762089252472, + "step": 933 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.500000238418579, + "epoch": 0.7282651072124756, + "grad_norm": 15.322958836916309, + "kl": 5.3203125, + "learning_rate": 2.875840545385834e-07, + "loss": 0.2221, + "reward": 0.20704365521669388, + "reward_std": 0.01928596943616867, + "rewards/accuracy_reward": 0.20704365521669388, + "step": 934 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.770833492279053, + "epoch": 0.7290448343079922, + "grad_norm": 62.89779119693976, + "kl": 9.28125, + "learning_rate": 2.865889702313995e-07, + "loss": 0.5436, + "reward": 0.2226661890745163, + "reward_std": 0.026209156028926373, + "rewards/accuracy_reward": 0.2226661890745163, + "step": 935 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.3125, + "epoch": 0.7298245614035088, + "grad_norm": 45.966182833692066, + "kl": 7.5546875, + "learning_rate": 2.855958415007541e-07, + "loss": 0.3375, + "reward": 0.20999066531658173, + "reward_std": 0.03812823537737131, + "rewards/accuracy_reward": 0.20999066531658173, + "step": 936 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.520833730697632, + "epoch": 0.7306042884990254, + "grad_norm": 294.342479205245, + "kl": 10.828125, + "learning_rate": 2.8460467571968394e-07, + "loss": 1.4089, + "reward": 0.18870870769023895, + "reward_std": 0.0448327511548996, + "rewards/accuracy_reward": 0.18870870769023895, + "step": 937 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.895833492279053, + "epoch": 0.7313840155945419, + "grad_norm": 11.916320863886996, + "kl": 3.7109375, + "learning_rate": 2.8361548024665293e-07, + "loss": 0.1631, + "reward": 0.20257259905338287, + "reward_std": 0.027222948148846626, + "rewards/accuracy_reward": 0.20257259905338287, + "step": 938 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.645833492279053, + "epoch": 0.7321637426900585, + "grad_norm": 8.11821760487084, + "kl": 3.5234375, + "learning_rate": 2.8262826242549735e-07, + "loss": 0.1513, + "reward": 0.22462120652198792, + "reward_std": 0.0013121606316417456, + "rewards/accuracy_reward": 0.22462120652198792, + "step": 939 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.625, + "epoch": 0.732943469785575, + "grad_norm": 4.6369447947782305, + "kl": 3.546875, + "learning_rate": 2.816430295853712e-07, + "loss": 0.1411, + "reward": 0.20333995670080185, + "reward_std": 0.007151787634938955, + "rewards/accuracy_reward": 0.20333995670080185, + "step": 940 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.895833492279053, + "epoch": 0.7337231968810917, + "grad_norm": 73.24128593498158, + "kl": 9.703125, + "learning_rate": 2.8065978904069184e-07, + "loss": 0.5183, + "reward": 0.2027934491634369, + "reward_std": 0.026852930895984173, + "rewards/accuracy_reward": 0.2027934491634369, + "step": 941 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.0, + "epoch": 0.7345029239766082, + "grad_norm": 15.39818905969451, + "kl": 4.5, + "learning_rate": 2.796785480910858e-07, + "loss": 0.3448, + "reward": 0.22633929550647736, + "reward_std": 0.007732369937002659, + "rewards/accuracy_reward": 0.22633929550647736, + "step": 942 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.312500238418579, + "epoch": 0.7352826510721248, + "grad_norm": 9.638816748497959, + "kl": 3.859375, + "learning_rate": 2.786993140213346e-07, + "loss": 0.1543, + "reward": 0.22909799963235855, + "reward_std": 0.01054774597287178, + "rewards/accuracy_reward": 0.22909799963235855, + "step": 943 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.020833492279053, + "epoch": 0.7360623781676413, + "grad_norm": 62.37680160046077, + "kl": 10.9375, + "learning_rate": 2.7772209410132044e-07, + "loss": 0.5944, + "reward": 0.23030658811330795, + "reward_std": 0.018346962984651327, + "rewards/accuracy_reward": 0.23030658811330795, + "step": 944 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.208333730697632, + "epoch": 0.7368421052631579, + "grad_norm": 40.62400287735523, + "kl": 8.75, + "learning_rate": 2.767468955859724e-07, + "loss": 0.5286, + "reward": 0.19600694626569748, + "reward_std": 0.050653524696826935, + "rewards/accuracy_reward": 0.19600694626569748, + "step": 945 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.229166746139526, + "epoch": 0.7376218323586745, + "grad_norm": 75.10386210352726, + "kl": 11.09375, + "learning_rate": 2.757737257152127e-07, + "loss": 0.6793, + "reward": 0.21377315372228622, + "reward_std": 0.038850562646985054, + "rewards/accuracy_reward": 0.21377315372228622, + "step": 946 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.770833492279053, + "epoch": 0.7384015594541911, + "grad_norm": 27.891470191757854, + "kl": 7.875, + "learning_rate": 2.7480259171390253e-07, + "loss": 0.4078, + "reward": 0.23989494889974594, + "reward_std": 0.018198708537966013, + "rewards/accuracy_reward": 0.23989494889974594, + "step": 947 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.145833492279053, + "epoch": 0.7391812865497076, + "grad_norm": 12.28104341981631, + "kl": 4.234375, + "learning_rate": 2.73833500791789e-07, + "loss": 0.2202, + "reward": 0.20513416081666946, + "reward_std": 0.00513895507901907, + "rewards/accuracy_reward": 0.20513416081666946, + "step": 948 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.8125, + "epoch": 0.7399610136452242, + "grad_norm": 121.38826739010007, + "kl": 9.578125, + "learning_rate": 2.728664601434508e-07, + "loss": 0.763, + "reward": 0.19558724761009216, + "reward_std": 0.027047929354012012, + "rewards/accuracy_reward": 0.19558724761009216, + "step": 949 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.020833492279053, + "epoch": 0.7407407407407407, + "grad_norm": 11.223203852179523, + "kl": 4.828125, + "learning_rate": 2.719014769482462e-07, + "loss": 0.2253, + "reward": 0.20562995970249176, + "reward_std": 0.03450433351099491, + "rewards/accuracy_reward": 0.20562995970249176, + "step": 950 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.5, + "epoch": 0.7415204678362574, + "grad_norm": 15.959557591894722, + "kl": 5.078125, + "learning_rate": 2.709385583702578e-07, + "loss": 0.1727, + "reward": 0.19545455276966095, + "reward_std": 0.015745919197797775, + "rewards/accuracy_reward": 0.19545455276966095, + "step": 951 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.041666746139526, + "epoch": 0.7423001949317739, + "grad_norm": 8.840969631310863, + "kl": 4.625, + "learning_rate": 2.6997771155824103e-07, + "loss": 0.2411, + "reward": 0.2261243388056755, + "reward_std": 0.014716084115207195, + "rewards/accuracy_reward": 0.2261243388056755, + "step": 952 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.8541669845581055, + "epoch": 0.7430799220272905, + "grad_norm": 8.845918098964946, + "kl": 3.7734375, + "learning_rate": 2.690189436455702e-07, + "loss": 0.2014, + "reward": 0.22633928805589676, + "reward_std": 0.007732370402663946, + "rewards/accuracy_reward": 0.22633928805589676, + "step": 953 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.708333492279053, + "epoch": 0.743859649122807, + "grad_norm": 6.947694379421619, + "kl": 2.63671875, + "learning_rate": 2.680622617501858e-07, + "loss": 0.1329, + "reward": 0.21049681305885315, + "reward_std": 0.006939306389540434, + "rewards/accuracy_reward": 0.21049681305885315, + "step": 954 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.041666746139526, + "epoch": 0.7446393762183235, + "grad_norm": 7.436401175020747, + "kl": 3.1796875, + "learning_rate": 2.671076729745417e-07, + "loss": 0.136, + "reward": 0.21041666716337204, + "reward_std": 0.0010309829376637936, + "rewards/accuracy_reward": 0.21041666716337204, + "step": 955 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.979166746139526, + "epoch": 0.7454191033138402, + "grad_norm": 4.864689427629056, + "kl": 4.359375, + "learning_rate": 2.661551844055521e-07, + "loss": 0.2003, + "reward": 0.20481150597333908, + "reward_std": 0.00874157901853323, + "rewards/accuracy_reward": 0.20481150597333908, + "step": 956 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.645833492279053, + "epoch": 0.7461988304093568, + "grad_norm": 8.507768498196715, + "kl": 3.9921875, + "learning_rate": 2.652048031145395e-07, + "loss": 0.148, + "reward": 0.22603785246610641, + "reward_std": 0.013582898303866386, + "rewards/accuracy_reward": 0.22603785246610641, + "step": 957 + }, + { + "clip_ratio": 0.0, + "completion_length": 51.187501430511475, + "epoch": 0.7469785575048733, + "grad_norm": 32.27812582134814, + "kl": 4.6484375, + "learning_rate": 2.642565361571816e-07, + "loss": 0.9699, + "reward": 0.18459384143352509, + "reward_std": 0.029080177657306194, + "rewards/accuracy_reward": 0.18459384143352509, + "step": 958 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.791666746139526, + "epoch": 0.7477582846003898, + "grad_norm": 160.75045899960304, + "kl": 10.28515625, + "learning_rate": 2.633103905734593e-07, + "loss": 0.611, + "reward": 0.21790007501840591, + "reward_std": 0.019832175225019455, + "rewards/accuracy_reward": 0.21790007501840591, + "step": 959 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.104166984558105, + "epoch": 0.7485380116959064, + "grad_norm": 20.296101176051536, + "kl": 4.078125, + "learning_rate": 2.6236637338760435e-07, + "loss": 0.419, + "reward": 0.1891876682639122, + "reward_std": 0.020885420497506857, + "rewards/accuracy_reward": 0.1891876682639122, + "step": 960 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.416666746139526, + "epoch": 0.749317738791423, + "grad_norm": 10.551977697203167, + "kl": 4.78125, + "learning_rate": 2.6142449160804716e-07, + "loss": 0.2533, + "reward": 0.22309602051973343, + "reward_std": 0.014773918315768242, + "rewards/accuracy_reward": 0.22309602051973343, + "step": 961 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.895833492279053, + "epoch": 0.7500974658869396, + "grad_norm": 19.491390891705862, + "kl": 6.625, + "learning_rate": 2.6048475222736465e-07, + "loss": 0.3948, + "reward": 0.21663805097341537, + "reward_std": 0.03417298384010792, + "rewards/accuracy_reward": 0.21663805097341537, + "step": 962 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.729166746139526, + "epoch": 0.7508771929824561, + "grad_norm": 8.890387601574608, + "kl": 3.8828125, + "learning_rate": 2.5954716222222856e-07, + "loss": 0.1911, + "reward": 0.22276785969734192, + "reward_std": 0.007732369937002659, + "rewards/accuracy_reward": 0.22276785969734192, + "step": 963 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.750000238418579, + "epoch": 0.7516569200779727, + "grad_norm": 10.851551062812241, + "kl": 4.46875, + "learning_rate": 2.586117285533536e-07, + "loss": 0.2524, + "reward": 0.19119106233119965, + "reward_std": 0.017348892986774445, + "rewards/accuracy_reward": 0.19119106233119965, + "step": 964 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.520833492279053, + "epoch": 0.7524366471734892, + "grad_norm": 18.43504329114516, + "kl": 6.328125, + "learning_rate": 2.576784581654456e-07, + "loss": 0.4014, + "reward": 0.20369762182235718, + "reward_std": 0.034923356026411057, + "rewards/accuracy_reward": 0.20369762182235718, + "step": 965 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.0, + "epoch": 0.7532163742690059, + "grad_norm": 18.8201577021743, + "kl": 4.6796875, + "learning_rate": 2.567473579871502e-07, + "loss": 0.2747, + "reward": 0.19305557012557983, + "reward_std": 0.0164988711476326, + "rewards/accuracy_reward": 0.19305557012557983, + "step": 966 + }, + { + "clip_ratio": 0.0, + "completion_length": 11.020833730697632, + "epoch": 0.7539961013645224, + "grad_norm": 11.201803955091007, + "kl": 5.5078125, + "learning_rate": 2.558184349310013e-07, + "loss": 0.2001, + "reward": 0.20014499127864838, + "reward_std": 0.013582898303866386, + "rewards/accuracy_reward": 0.20014499127864838, + "step": 967 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.833333969116211, + "epoch": 0.754775828460039, + "grad_norm": 25.199604766050193, + "kl": 4.984375, + "learning_rate": 2.5489169589336966e-07, + "loss": 0.2482, + "reward": 0.17346681654453278, + "reward_std": 0.021422578021883965, + "rewards/accuracy_reward": 0.17346681654453278, + "step": 968 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.312500238418579, + "epoch": 0.7555555555555555, + "grad_norm": 518.0749557950661, + "kl": 29.7421875, + "learning_rate": 2.5396714775441176e-07, + "loss": 1.7572, + "reward": 0.20140931755304337, + "reward_std": 0.029014225117862225, + "rewards/accuracy_reward": 0.20140931755304337, + "step": 969 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.500000476837158, + "epoch": 0.7563352826510721, + "grad_norm": 48.84057430402834, + "kl": 5.4296875, + "learning_rate": 2.5304479737801903e-07, + "loss": 0.4127, + "reward": 0.20271387696266174, + "reward_std": 0.011814319994300604, + "rewards/accuracy_reward": 0.20271387696266174, + "step": 970 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.145833492279053, + "epoch": 0.7571150097465887, + "grad_norm": 12.306816337922546, + "kl": 3.5546875, + "learning_rate": 2.521246516117662e-07, + "loss": 0.174, + "reward": 0.20942265540361404, + "reward_std": 0.009404968470335007, + "rewards/accuracy_reward": 0.20942265540361404, + "step": 971 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.375000476837158, + "epoch": 0.7578947368421053, + "grad_norm": 124.30793671989524, + "kl": 14.03125, + "learning_rate": 2.5120671728686103e-07, + "loss": 0.5753, + "reward": 0.19134657829999924, + "reward_std": 0.024854368530213833, + "rewards/accuracy_reward": 0.19134657829999924, + "step": 972 + }, + { + "clip_ratio": 0.0, + "completion_length": 49.291666746139526, + "epoch": 0.7586744639376218, + "grad_norm": 24.045483087804328, + "kl": 5.375, + "learning_rate": 2.502910012180935e-07, + "loss": 0.8198, + "reward": 0.22331849485635757, + "reward_std": 0.03475153027102351, + "rewards/accuracy_reward": 0.22331849485635757, + "step": 973 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.375, + "epoch": 0.7594541910331384, + "grad_norm": 12.934408267280064, + "kl": 5.46875, + "learning_rate": 2.4937751020378506e-07, + "loss": 0.3202, + "reward": 0.21127034723758698, + "reward_std": 0.018017299007624388, + "rewards/accuracy_reward": 0.21127034723758698, + "step": 974 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.270833492279053, + "epoch": 0.7602339181286549, + "grad_norm": 17.009590019159607, + "kl": 4.5546875, + "learning_rate": 2.484662510257382e-07, + "loss": 0.2651, + "reward": 0.20635724067687988, + "reward_std": 0.013679529540240765, + "rewards/accuracy_reward": 0.20635724067687988, + "step": 975 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.6875, + "epoch": 0.7610136452241716, + "grad_norm": 10.429697339002072, + "kl": 3.6015625, + "learning_rate": 2.475572304491863e-07, + "loss": 0.2304, + "reward": 0.22831101715564728, + "reward_std": 0.0009021099540404975, + "rewards/accuracy_reward": 0.22831101715564728, + "step": 976 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.145833492279053, + "epoch": 0.7617933723196881, + "grad_norm": 11.947555394556202, + "kl": 4.859375, + "learning_rate": 2.4665045522274315e-07, + "loss": 0.2264, + "reward": 0.20633013546466827, + "reward_std": 0.02137306472286582, + "rewards/accuracy_reward": 0.20633013546466827, + "step": 977 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.3541669845581055, + "epoch": 0.7625730994152047, + "grad_norm": 24.42401875663099, + "kl": 5.453125, + "learning_rate": 2.45745932078353e-07, + "loss": 0.3224, + "reward": 0.22212301194667816, + "reward_std": 0.01703009381890297, + "rewards/accuracy_reward": 0.22212301194667816, + "step": 978 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.020833492279053, + "epoch": 0.7633528265107212, + "grad_norm": 19.745012615084153, + "kl": 5.6328125, + "learning_rate": 2.4484366773124024e-07, + "loss": 0.5876, + "reward": 0.23024173080921173, + "reward_std": 0.03315778076648712, + "rewards/accuracy_reward": 0.23024173080921173, + "step": 979 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.375, + "epoch": 0.7641325536062378, + "grad_norm": 700.1037250130006, + "kl": 33.1875, + "learning_rate": 2.4394366887986047e-07, + "loss": 1.885, + "reward": 0.22899536788463593, + "reward_std": 0.030556529760360718, + "rewards/accuracy_reward": 0.22899536788463593, + "step": 980 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.958333492279053, + "epoch": 0.7649122807017544, + "grad_norm": 12.422299352212708, + "kl": 4.46875, + "learning_rate": 2.4304594220584965e-07, + "loss": 0.3144, + "reward": 0.213882215321064, + "reward_std": 0.025799294468015432, + "rewards/accuracy_reward": 0.213882215321064, + "step": 981 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.583333492279053, + "epoch": 0.765692007797271, + "grad_norm": 267.8492873762983, + "kl": 18.140625, + "learning_rate": 2.421504943739749e-07, + "loss": 0.9374, + "reward": 0.22915521264076233, + "reward_std": 0.02509198058396578, + "rewards/accuracy_reward": 0.22915521264076233, + "step": 982 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.437500476837158, + "epoch": 0.7664717348927875, + "grad_norm": 32.19371440790225, + "kl": 5.578125, + "learning_rate": 2.4125733203208554e-07, + "loss": 0.3481, + "reward": 0.17858626693487167, + "reward_std": 0.021845710929483175, + "rewards/accuracy_reward": 0.17858626693487167, + "step": 983 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.791666984558105, + "epoch": 0.7672514619883041, + "grad_norm": 30.963429361409688, + "kl": 6.9296875, + "learning_rate": 2.40366461811063e-07, + "loss": 0.4352, + "reward": 0.1783769130706787, + "reward_std": 0.01941643003374338, + "rewards/accuracy_reward": 0.1783769130706787, + "step": 984 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.0, + "epoch": 0.7680311890838206, + "grad_norm": 9.838602313749337, + "kl": 3.546875, + "learning_rate": 2.394778903247719e-07, + "loss": 0.194, + "reward": 0.2005864903330803, + "reward_std": 0.009700251743197441, + "rewards/accuracy_reward": 0.2005864903330803, + "step": 985 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.270833492279053, + "epoch": 0.7688109161793373, + "grad_norm": 7.563139793936384, + "kl": 3.6796875, + "learning_rate": 2.3859162417001093e-07, + "loss": 0.1569, + "reward": 0.2123677209019661, + "reward_std": 0.006838416215032339, + "rewards/accuracy_reward": 0.2123677209019661, + "step": 986 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.0, + "epoch": 0.7695906432748538, + "grad_norm": 14.499140694509943, + "kl": 4.625, + "learning_rate": 2.3770766992646406e-07, + "loss": 0.2424, + "reward": 0.2105368673801422, + "reward_std": 0.006800520233809948, + "rewards/accuracy_reward": 0.2105368673801422, + "step": 987 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.645833492279053, + "epoch": 0.7703703703703704, + "grad_norm": 14.099907936315352, + "kl": 4.2734375, + "learning_rate": 2.3682603415665133e-07, + "loss": 0.2245, + "reward": 0.20190972834825516, + "reward_std": 0.009480978362262249, + "rewards/accuracy_reward": 0.20190972834825516, + "step": 988 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.4791669845581055, + "epoch": 0.7711500974658869, + "grad_norm": 18.01395645045911, + "kl": 5.5, + "learning_rate": 2.3594672340588042e-07, + "loss": 0.3529, + "reward": 0.21834935992956161, + "reward_std": 0.01715551409870386, + "rewards/accuracy_reward": 0.21834935992956161, + "step": 989 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.645833492279053, + "epoch": 0.7719298245614035, + "grad_norm": 15.183264664856523, + "kl": 4.078125, + "learning_rate": 2.3506974420219788e-07, + "loss": 0.1894, + "reward": 0.18635330349206924, + "reward_std": 0.013662159442901611, + "rewards/accuracy_reward": 0.18635330349206924, + "step": 990 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.708333492279053, + "epoch": 0.7727095516569201, + "grad_norm": 10.698107487049688, + "kl": 4.203125, + "learning_rate": 2.341951030563408e-07, + "loss": 0.2453, + "reward": 0.20675770938396454, + "reward_std": 0.008013959042727947, + "rewards/accuracy_reward": 0.20675770938396454, + "step": 991 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.708333492279053, + "epoch": 0.7734892787524367, + "grad_norm": 10.424882318762995, + "kl": 6.5, + "learning_rate": 2.3332280646168835e-07, + "loss": 0.2424, + "reward": 0.22024955600500107, + "reward_std": 0.026445474475622177, + "rewards/accuracy_reward": 0.22024955600500107, + "step": 992 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.791666746139526, + "epoch": 0.7742690058479532, + "grad_norm": 10.250433089470693, + "kl": 3.28125, + "learning_rate": 2.3245286089421379e-07, + "loss": 0.1531, + "reward": 0.2098102867603302, + "reward_std": 0.0013121606316417456, + "rewards/accuracy_reward": 0.2098102867603302, + "step": 993 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.9791669845581055, + "epoch": 0.7750487329434698, + "grad_norm": 94.41323390877069, + "kl": 11.9140625, + "learning_rate": 2.315852728124358e-07, + "loss": 0.4808, + "reward": 0.2062269076704979, + "reward_std": 0.027422614162787795, + "rewards/accuracy_reward": 0.2062269076704979, + "step": 994 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.187500476837158, + "epoch": 0.7758284600389863, + "grad_norm": 11.481468858085083, + "kl": 6.09375, + "learning_rate": 2.307200486573718e-07, + "loss": 0.2584, + "reward": 0.2171914055943489, + "reward_std": 0.03088666219264269, + "rewards/accuracy_reward": 0.2171914055943489, + "step": 995 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.270833492279053, + "epoch": 0.776608187134503, + "grad_norm": 8.607359144191236, + "kl": 4.953125, + "learning_rate": 2.2985719485248846e-07, + "loss": 0.2727, + "reward": 0.19091825187206268, + "reward_std": 0.015199249610304832, + "rewards/accuracy_reward": 0.19091825187206268, + "step": 996 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.583333492279053, + "epoch": 0.7773879142300195, + "grad_norm": 36.365549208527334, + "kl": 4.9453125, + "learning_rate": 2.2899671780365525e-07, + "loss": 0.2074, + "reward": 0.19834958016872406, + "reward_std": 0.01723405532538891, + "rewards/accuracy_reward": 0.19834958016872406, + "step": 997 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.958333492279053, + "epoch": 0.7781676413255361, + "grad_norm": 11.162306979137217, + "kl": 5.078125, + "learning_rate": 2.2813862389909647e-07, + "loss": 0.3154, + "reward": 0.21755341440439224, + "reward_std": 0.021098894998431206, + "rewards/accuracy_reward": 0.21755341440439224, + "step": 998 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.1041669845581055, + "epoch": 0.7789473684210526, + "grad_norm": 12.039742379468745, + "kl": 3.265625, + "learning_rate": 2.2728291950934368e-07, + "loss": 0.1772, + "reward": 0.2253224328160286, + "reward_std": 0.011254892684519291, + "rewards/accuracy_reward": 0.2253224328160286, + "step": 999 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.187500238418579, + "epoch": 0.7797270955165692, + "grad_norm": 7.4709825879130625, + "kl": 5.15625, + "learning_rate": 2.2642961098718883e-07, + "loss": 0.3011, + "reward": 0.20339345186948776, + "reward_std": 0.01704784482717514, + "rewards/accuracy_reward": 0.20339345186948776, + "step": 1000 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.125, + "epoch": 0.7805068226120858, + "grad_norm": 5.1863406122968065, + "kl": 3.3359375, + "learning_rate": 2.2557870466763653e-07, + "loss": 0.1604, + "reward": 0.21778274327516556, + "reward_std": 0.006443641148507595, + "rewards/accuracy_reward": 0.21778274327516556, + "step": 1001 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.375, + "epoch": 0.7812865497076024, + "grad_norm": 9.812251880904517, + "kl": 2.73046875, + "learning_rate": 2.2473020686785755e-07, + "loss": 0.1048, + "reward": 0.2005208432674408, + "reward_std": 0.011694144457578659, + "rewards/accuracy_reward": 0.2005208432674408, + "step": 1002 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.041666984558105, + "epoch": 0.7820662768031189, + "grad_norm": 70.98439771418607, + "kl": 8.515625, + "learning_rate": 2.2388412388714154e-07, + "loss": 0.5227, + "reward": 0.18989139795303345, + "reward_std": 0.030605142936110497, + "rewards/accuracy_reward": 0.18989139795303345, + "step": 1003 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.437500476837158, + "epoch": 0.7828460038986355, + "grad_norm": 51.15610748122753, + "kl": 8.578125, + "learning_rate": 2.2304046200685054e-07, + "loss": 0.4686, + "reward": 0.1909569576382637, + "reward_std": 0.01652967883273959, + "rewards/accuracy_reward": 0.1909569576382637, + "step": 1004 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.770833492279053, + "epoch": 0.783625730994152, + "grad_norm": 189.8733122812511, + "kl": 9.15625, + "learning_rate": 2.2219922749037206e-07, + "loss": 0.5671, + "reward": 0.20651844143867493, + "reward_std": 0.024225687608122826, + "rewards/accuracy_reward": 0.20651844143867493, + "step": 1005 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.8541669845581055, + "epoch": 0.7844054580896687, + "grad_norm": 15.698955863909807, + "kl": 5.09375, + "learning_rate": 2.2136042658307302e-07, + "loss": 0.2326, + "reward": 0.2251984179019928, + "reward_std": 0.007877668365836143, + "rewards/accuracy_reward": 0.2251984179019928, + "step": 1006 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.520833730697632, + "epoch": 0.7851851851851852, + "grad_norm": 8.027174839109064, + "kl": 3.203125, + "learning_rate": 2.2052406551225272e-07, + "loss": 0.1836, + "reward": 0.18282855302095413, + "reward_std": 0.00655776634812355, + "rewards/accuracy_reward": 0.18282855302095413, + "step": 1007 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.187500238418579, + "epoch": 0.7859649122807018, + "grad_norm": 21.33752135003128, + "kl": 6.546875, + "learning_rate": 2.1969015048709733e-07, + "loss": 0.2838, + "reward": 0.22746394574642181, + "reward_std": 0.022172683267854154, + "rewards/accuracy_reward": 0.22746394574642181, + "step": 1008 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.1041669845581055, + "epoch": 0.7867446393762183, + "grad_norm": 9.143039487813896, + "kl": 3.703125, + "learning_rate": 2.1885868769863314e-07, + "loss": 0.1767, + "reward": 0.2083333283662796, + "reward_std": 0.014433758333325386, + "rewards/accuracy_reward": 0.2083333283662796, + "step": 1009 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.3125, + "epoch": 0.7875243664717348, + "grad_norm": 13.356369077202201, + "kl": 3.9140625, + "learning_rate": 2.1802968331968137e-07, + "loss": 0.3821, + "reward": 0.20141476392745972, + "reward_std": 0.016035879030823708, + "rewards/accuracy_reward": 0.20141476392745972, + "step": 1010 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.2916669845581055, + "epoch": 0.7883040935672515, + "grad_norm": 506.3640246843003, + "kl": 21.3125, + "learning_rate": 2.1720314350481142e-07, + "loss": 1.2658, + "reward": 0.17922842502593994, + "reward_std": 0.04507869854569435, + "rewards/accuracy_reward": 0.17922842502593994, + "step": 1011 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.6041669845581055, + "epoch": 0.789083820662768, + "grad_norm": 12.47273968731663, + "kl": 4.4921875, + "learning_rate": 2.163790743902959e-07, + "loss": 0.3566, + "reward": 0.23570772260427475, + "reward_std": 0.006208638660609722, + "rewards/accuracy_reward": 0.23570772260427475, + "step": 1012 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.958333492279053, + "epoch": 0.7898635477582846, + "grad_norm": 203.22345676605886, + "kl": 10.546875, + "learning_rate": 2.1555748209406476e-07, + "loss": 0.7871, + "reward": 0.20383402705192566, + "reward_std": 0.016734477481804788, + "rewards/accuracy_reward": 0.20383402705192566, + "step": 1013 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.937500476837158, + "epoch": 0.7906432748538011, + "grad_norm": 20.280019297378633, + "kl": 6.421875, + "learning_rate": 2.1473837271565986e-07, + "loss": 0.3951, + "reward": 0.21068129688501358, + "reward_std": 0.018166671507060528, + "rewards/accuracy_reward": 0.21068129688501358, + "step": 1014 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.7916669845581055, + "epoch": 0.7914230019493177, + "grad_norm": 9.917389491397316, + "kl": 4.4453125, + "learning_rate": 2.1392175233618992e-07, + "loss": 0.24, + "reward": 0.1961805671453476, + "reward_std": 0.007370158564299345, + "rewards/accuracy_reward": 0.1961805671453476, + "step": 1015 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.645833730697632, + "epoch": 0.7922027290448344, + "grad_norm": 2348.0626510959946, + "kl": 53.6328125, + "learning_rate": 2.1310762701828498e-07, + "loss": 5.0785, + "reward": 0.1970486119389534, + "reward_std": 0.03187454678118229, + "rewards/accuracy_reward": 0.1970486119389534, + "step": 1016 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.083333730697632, + "epoch": 0.7929824561403509, + "grad_norm": 150.73356857932657, + "kl": 9.6875, + "learning_rate": 2.1229600280605187e-07, + "loss": 0.8747, + "reward": 0.18567539006471634, + "reward_std": 0.031576406210660934, + "rewards/accuracy_reward": 0.18567539006471634, + "step": 1017 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.6875, + "epoch": 0.7937621832358674, + "grad_norm": 1142.1620880944467, + "kl": 41.515625, + "learning_rate": 2.114868857250288e-07, + "loss": 2.3744, + "reward": 0.16367706656455994, + "reward_std": 0.007582499412819743, + "rewards/accuracy_reward": 0.16367706656455994, + "step": 1018 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.9791669845581055, + "epoch": 0.794541910331384, + "grad_norm": 9.853267263384136, + "kl": 4.59375, + "learning_rate": 2.106802817821413e-07, + "loss": 0.1671, + "reward": 0.2074694186449051, + "reward_std": 0.031790743581950665, + "rewards/accuracy_reward": 0.2074694186449051, + "step": 1019 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.187500238418579, + "epoch": 0.7953216374269005, + "grad_norm": 11.391390259422268, + "kl": 6.015625, + "learning_rate": 2.0987619696565678e-07, + "loss": 0.2342, + "reward": 0.2012605145573616, + "reward_std": 0.020377070643007755, + "rewards/accuracy_reward": 0.2012605145573616, + "step": 1020 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.375000238418579, + "epoch": 0.7961013645224172, + "grad_norm": 3.7377393142006117, + "kl": 4.421875, + "learning_rate": 2.0907463724514096e-07, + "loss": 0.1816, + "reward": 0.2247685194015503, + "reward_std": 0.007151786237955093, + "rewards/accuracy_reward": 0.2247685194015503, + "step": 1021 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.145833492279053, + "epoch": 0.7968810916179337, + "grad_norm": 11.297806677714458, + "kl": 5.453125, + "learning_rate": 2.0827560857141275e-07, + "loss": 0.2572, + "reward": 0.22240155190229416, + "reward_std": 0.021373063325881958, + "rewards/accuracy_reward": 0.22240155190229416, + "step": 1022 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.020833730697632, + "epoch": 0.7976608187134503, + "grad_norm": 33.05835196916885, + "kl": 6.96875, + "learning_rate": 2.0747911687650077e-07, + "loss": 0.2696, + "reward": 0.20787452161312103, + "reward_std": 0.026464111171662807, + "rewards/accuracy_reward": 0.20787452161312103, + "step": 1023 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.1041669845581055, + "epoch": 0.7984405458089668, + "grad_norm": 15.751242644265972, + "kl": 4.4375, + "learning_rate": 2.066851680735986e-07, + "loss": 0.2591, + "reward": 0.21856820583343506, + "reward_std": 0.015401717275381088, + "rewards/accuracy_reward": 0.21856820583343506, + "step": 1024 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.208333492279053, + "epoch": 0.7992202729044834, + "grad_norm": 12.789644900557542, + "kl": 3.203125, + "learning_rate": 2.058937680570218e-07, + "loss": 0.1328, + "reward": 0.19362745434045792, + "reward_std": 0.022075158543884754, + "rewards/accuracy_reward": 0.19362745434045792, + "step": 1025 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.5416669845581055, + "epoch": 0.8, + "grad_norm": 16.823301316471195, + "kl": 4.515625, + "learning_rate": 2.0510492270216317e-07, + "loss": 0.3153, + "reward": 0.20570098608732224, + "reward_std": 0.024778381921350956, + "rewards/accuracy_reward": 0.20570098608732224, + "step": 1026 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.9791669845581055, + "epoch": 0.8007797270955166, + "grad_norm": 7.0176385628468765, + "kl": 5.078125, + "learning_rate": 2.0431863786544977e-07, + "loss": 0.2485, + "reward": 0.23282968252897263, + "reward_std": 0.018245520535856485, + "rewards/accuracy_reward": 0.23282968252897263, + "step": 1027 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.9166669845581055, + "epoch": 0.8015594541910331, + "grad_norm": 11.85168331368778, + "kl": 4.7890625, + "learning_rate": 2.035349193842991e-07, + "loss": 0.2387, + "reward": 0.20412935316562653, + "reward_std": 0.019685628358274698, + "rewards/accuracy_reward": 0.20412935316562653, + "step": 1028 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.854166746139526, + "epoch": 0.8023391812865497, + "grad_norm": 39.04736862176875, + "kl": 7.5625, + "learning_rate": 2.0275377307707604e-07, + "loss": 0.6151, + "reward": 0.20278416574001312, + "reward_std": 0.02558747213333845, + "rewards/accuracy_reward": 0.20278416574001312, + "step": 1029 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.958333492279053, + "epoch": 0.8031189083820662, + "grad_norm": 54.012928361862016, + "kl": 5.921875, + "learning_rate": 2.0197520474304957e-07, + "loss": 0.6474, + "reward": 0.21085228025913239, + "reward_std": 0.022825470194220543, + "rewards/accuracy_reward": 0.21085228025913239, + "step": 1030 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.6041669845581055, + "epoch": 0.8038986354775829, + "grad_norm": 17.831836069334148, + "kl": 5.859375, + "learning_rate": 2.011992201623494e-07, + "loss": 0.3242, + "reward": 0.17871718108654022, + "reward_std": 0.025753642432391644, + "rewards/accuracy_reward": 0.17871718108654022, + "step": 1031 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.083333492279053, + "epoch": 0.8046783625730994, + "grad_norm": 16.98702764958395, + "kl": 4.71875, + "learning_rate": 2.004258250959237e-07, + "loss": 0.2511, + "reward": 0.2085336670279503, + "reward_std": 0.01236457098275423, + "rewards/accuracy_reward": 0.2085336670279503, + "step": 1032 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.270833492279053, + "epoch": 0.805458089668616, + "grad_norm": 1.8858344069108528, + "kl": 3.5546875, + "learning_rate": 1.9965502528549564e-07, + "loss": 0.1398, + "reward": 0.22857142984867096, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.22857142984867096, + "step": 1033 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.916666984558105, + "epoch": 0.8062378167641325, + "grad_norm": 38.35429856673092, + "kl": 5.265625, + "learning_rate": 1.9888682645352124e-07, + "loss": 0.4413, + "reward": 0.20756207406520844, + "reward_std": 0.012499698903411627, + "rewards/accuracy_reward": 0.20756207406520844, + "step": 1034 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.708333492279053, + "epoch": 0.8070175438596491, + "grad_norm": 14.693012741694409, + "kl": 5.828125, + "learning_rate": 1.9812123430314675e-07, + "loss": 0.3358, + "reward": 0.17690972983837128, + "reward_std": 0.03676615282893181, + "rewards/accuracy_reward": 0.17690972983837128, + "step": 1035 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.500000476837158, + "epoch": 0.8077972709551657, + "grad_norm": 52.126792421583936, + "kl": 6.109375, + "learning_rate": 1.9735825451816625e-07, + "loss": 0.499, + "reward": 0.2145041972398758, + "reward_std": 0.005429095006547868, + "rewards/accuracy_reward": 0.2145041972398758, + "step": 1036 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.520833730697632, + "epoch": 0.8085769980506823, + "grad_norm": 28.50515379450771, + "kl": 7.671875, + "learning_rate": 1.9659789276297933e-07, + "loss": 0.3041, + "reward": 0.20659500360488892, + "reward_std": 0.028170164674520493, + "rewards/accuracy_reward": 0.20659500360488892, + "step": 1037 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.958333730697632, + "epoch": 0.8093567251461988, + "grad_norm": 126.35070528069173, + "kl": 10.21875, + "learning_rate": 1.9584015468254943e-07, + "loss": 0.6263, + "reward": 0.2020823061466217, + "reward_std": 0.01635452453047037, + "rewards/accuracy_reward": 0.2020823061466217, + "step": 1038 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.229166746139526, + "epoch": 0.8101364522417154, + "grad_norm": 100.24633028112018, + "kl": 8.1875, + "learning_rate": 1.9508504590236152e-07, + "loss": 0.454, + "reward": 0.21619106084108353, + "reward_std": 0.010137997101992369, + "rewards/accuracy_reward": 0.21619106084108353, + "step": 1039 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.645833492279053, + "epoch": 0.8109161793372319, + "grad_norm": 1.7762657686368508, + "kl": 2.6953125, + "learning_rate": 1.943325720283808e-07, + "loss": 0.1047, + "reward": 0.18125000596046448, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.18125000596046448, + "step": 1040 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.687500476837158, + "epoch": 0.8116959064327486, + "grad_norm": 43.31100210880956, + "kl": 6.578125, + "learning_rate": 1.9358273864701053e-07, + "loss": 0.3818, + "reward": 0.20244773477315903, + "reward_std": 0.014119701460003853, + "rewards/accuracy_reward": 0.20244773477315903, + "step": 1041 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.541666984558105, + "epoch": 0.8124756335282651, + "grad_norm": 12.221328146155246, + "kl": 3.48046875, + "learning_rate": 1.9283555132505078e-07, + "loss": 0.1876, + "reward": 0.19107519835233688, + "reward_std": 0.009265800181310624, + "rewards/accuracy_reward": 0.19107519835233688, + "step": 1042 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.895833730697632, + "epoch": 0.8132553606237817, + "grad_norm": 625.3281368873542, + "kl": 21.3515625, + "learning_rate": 1.9209101560965733e-07, + "loss": 1.2169, + "reward": 0.17777778208255768, + "reward_std": 0.04005598463118076, + "rewards/accuracy_reward": 0.17777778208255768, + "step": 1043 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.208333492279053, + "epoch": 0.8140350877192982, + "grad_norm": 3.7742028504680545, + "kl": 4.28125, + "learning_rate": 1.913491370283003e-07, + "loss": 0.1812, + "reward": 0.23549679666757584, + "reward_std": 0.006939306389540434, + "rewards/accuracy_reward": 0.23549679666757584, + "step": 1044 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.770833730697632, + "epoch": 0.8148148148148148, + "grad_norm": 34.93806399167577, + "kl": 4.40625, + "learning_rate": 1.9060992108872293e-07, + "loss": 0.282, + "reward": 0.20257117599248886, + "reward_std": 0.00905408151447773, + "rewards/accuracy_reward": 0.20257117599248886, + "step": 1045 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.3125, + "epoch": 0.8155945419103314, + "grad_norm": 17.054482394533192, + "kl": 3.8203125, + "learning_rate": 1.89873373278901e-07, + "loss": 0.2557, + "reward": 0.21153846383094788, + "reward_std": 0.0033308672718703747, + "rewards/accuracy_reward": 0.21153846383094788, + "step": 1046 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.937500238418579, + "epoch": 0.816374269005848, + "grad_norm": 25.633578744677195, + "kl": 4.5078125, + "learning_rate": 1.89139499067002e-07, + "loss": 0.4133, + "reward": 0.1970544531941414, + "reward_std": 0.02082281280308962, + "rewards/accuracy_reward": 0.1970544531941414, + "step": 1047 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.708333492279053, + "epoch": 0.8171539961013645, + "grad_norm": 4.896591068565919, + "kl": 4.5, + "learning_rate": 1.8840830390134442e-07, + "loss": 0.2521, + "reward": 0.20726492255926132, + "reward_std": 0.010129572823643684, + "rewards/accuracy_reward": 0.20726492255926132, + "step": 1048 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.187500238418579, + "epoch": 0.8179337231968811, + "grad_norm": 127.22399983848484, + "kl": 10.921875, + "learning_rate": 1.8767979321035742e-07, + "loss": 0.6176, + "reward": 0.19040203094482422, + "reward_std": 0.03167390776798129, + "rewards/accuracy_reward": 0.19040203094482422, + "step": 1049 + }, + { + "clip_ratio": 0.0, + "completion_length": 11.916666984558105, + "epoch": 0.8187134502923976, + "grad_norm": 171.92806685223522, + "kl": 12.9609375, + "learning_rate": 1.8695397240254042e-07, + "loss": 0.9516, + "reward": 0.20736607909202576, + "reward_std": 0.018282258417457342, + "rewards/accuracy_reward": 0.20736607909202576, + "step": 1050 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.687500238418579, + "epoch": 0.8194931773879143, + "grad_norm": 45.74204857070197, + "kl": 10.28125, + "learning_rate": 1.8623084686642309e-07, + "loss": 0.5615, + "reward": 0.20191581547260284, + "reward_std": 0.06121378019452095, + "rewards/accuracy_reward": 0.20191581547260284, + "step": 1051 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.104166984558105, + "epoch": 0.8202729044834308, + "grad_norm": 5.828633045565711, + "kl": 3.171875, + "learning_rate": 1.8551042197052523e-07, + "loss": 0.1471, + "reward": 0.18169643729925156, + "reward_std": 0.007298428099602461, + "rewards/accuracy_reward": 0.18169643729925156, + "step": 1052 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.833333730697632, + "epoch": 0.8210526315789474, + "grad_norm": 17.68023406395046, + "kl": 5.875, + "learning_rate": 1.8479270306331694e-07, + "loss": 0.2516, + "reward": 0.19993094354867935, + "reward_std": 0.012093579396605492, + "rewards/accuracy_reward": 0.19993094354867935, + "step": 1053 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.979166746139526, + "epoch": 0.8218323586744639, + "grad_norm": 3.6126801027387647, + "kl": 3.9609375, + "learning_rate": 1.8407769547317915e-07, + "loss": 0.2158, + "reward": 0.18511030822992325, + "reward_std": 0.007298428099602461, + "rewards/accuracy_reward": 0.18511030822992325, + "step": 1054 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.958333492279053, + "epoch": 0.8226120857699805, + "grad_norm": 12.042133738623981, + "kl": 4.0078125, + "learning_rate": 1.8336540450836343e-07, + "loss": 0.2628, + "reward": 0.19201961904764175, + "reward_std": 0.014172513037919998, + "rewards/accuracy_reward": 0.19201961904764175, + "step": 1055 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.666666746139526, + "epoch": 0.8233918128654971, + "grad_norm": 16.12702371214127, + "kl": 4.4921875, + "learning_rate": 1.8265583545695329e-07, + "loss": 0.2757, + "reward": 0.2366071492433548, + "reward_std": 0.015464740805327892, + "rewards/accuracy_reward": 0.2366071492433548, + "step": 1056 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.125000476837158, + "epoch": 0.8241715399610137, + "grad_norm": 32.75921238737116, + "kl": 6.453125, + "learning_rate": 1.8194899358682457e-07, + "loss": 0.4349, + "reward": 0.2033722996711731, + "reward_std": 0.03450218332000077, + "rewards/accuracy_reward": 0.2033722996711731, + "step": 1057 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.500000238418579, + "epoch": 0.8249512670565302, + "grad_norm": 16.961396326812746, + "kl": 2.515625, + "learning_rate": 1.812448841456062e-07, + "loss": 0.2371, + "reward": 0.19489890336990356, + "reward_std": 0.013980986550450325, + "rewards/accuracy_reward": 0.19489890336990356, + "step": 1058 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.895833492279053, + "epoch": 0.8257309941520468, + "grad_norm": 103.72792059975536, + "kl": 11.03125, + "learning_rate": 1.8054351236064162e-07, + "loss": 0.769, + "reward": 0.21145452558994293, + "reward_std": 0.030768138356506824, + "rewards/accuracy_reward": 0.21145452558994293, + "step": 1059 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.8541669845581055, + "epoch": 0.8265107212475633, + "grad_norm": 10.812579986161575, + "kl": 4.59375, + "learning_rate": 1.7984488343894982e-07, + "loss": 0.1807, + "reward": 0.2256365865468979, + "reward_std": 0.011488264426589012, + "rewards/accuracy_reward": 0.2256365865468979, + "step": 1060 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.895833492279053, + "epoch": 0.82729044834308, + "grad_norm": 18.84286940153334, + "kl": 5.15625, + "learning_rate": 1.7914900256718646e-07, + "loss": 0.2495, + "reward": 0.20378590375185013, + "reward_std": 0.009470305405557156, + "rewards/accuracy_reward": 0.20378590375185013, + "step": 1061 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.437500476837158, + "epoch": 0.8280701754385965, + "grad_norm": 14.078637171258807, + "kl": 4.421875, + "learning_rate": 1.7845587491160554e-07, + "loss": 0.1668, + "reward": 0.20159144699573517, + "reward_std": 0.018634121399372816, + "rewards/accuracy_reward": 0.20159144699573517, + "step": 1062 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.750000476837158, + "epoch": 0.8288499025341131, + "grad_norm": 77.07146085200857, + "kl": 9.71875, + "learning_rate": 1.777655056180213e-07, + "loss": 0.8015, + "reward": 0.22262488305568695, + "reward_std": 0.028617288917303085, + "rewards/accuracy_reward": 0.22262488305568695, + "step": 1063 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.729166746139526, + "epoch": 0.8296296296296296, + "grad_norm": 11.256085752051334, + "kl": 5.25, + "learning_rate": 1.770778998117694e-07, + "loss": 0.2411, + "reward": 0.2345610186457634, + "reward_std": 0.013148694764822721, + "rewards/accuracy_reward": 0.2345610186457634, + "step": 1064 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.687500476837158, + "epoch": 0.8304093567251462, + "grad_norm": 14.300891039222625, + "kl": 4.578125, + "learning_rate": 1.7639306259766951e-07, + "loss": 0.2684, + "reward": 0.204239122569561, + "reward_std": 0.016638413071632385, + "rewards/accuracy_reward": 0.204239122569561, + "step": 1065 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.750000476837158, + "epoch": 0.8311890838206628, + "grad_norm": 1993.4019754339108, + "kl": 119.390625, + "learning_rate": 1.7571099905998709e-07, + "loss": 5.1469, + "reward": 0.18536876887083054, + "reward_std": 0.024212586577050388, + "rewards/accuracy_reward": 0.18536876887083054, + "step": 1066 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.458333492279053, + "epoch": 0.8319688109161794, + "grad_norm": 131.53961724061853, + "kl": 13.90625, + "learning_rate": 1.7503171426239556e-07, + "loss": 0.9368, + "reward": 0.19176756590604782, + "reward_std": 0.046957389917224646, + "rewards/accuracy_reward": 0.19176756590604782, + "step": 1067 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.020833492279053, + "epoch": 0.8327485380116959, + "grad_norm": 16.594409607851688, + "kl": 4.234375, + "learning_rate": 1.7435521324793886e-07, + "loss": 0.2707, + "reward": 0.2057005614042282, + "reward_std": 0.02216450870037079, + "rewards/accuracy_reward": 0.2057005614042282, + "step": 1068 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.270833492279053, + "epoch": 0.8335282651072125, + "grad_norm": 5.820446340316934, + "kl": 3.0390625, + "learning_rate": 1.7368150103899424e-07, + "loss": 0.1297, + "reward": 0.21212121844291687, + "reward_std": 0.0013121606316417456, + "rewards/accuracy_reward": 0.21212121844291687, + "step": 1069 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.1875, + "epoch": 0.834307992202729, + "grad_norm": 19.98879790427782, + "kl": 5.1875, + "learning_rate": 1.7301058263723438e-07, + "loss": 0.3566, + "reward": 0.21365749835968018, + "reward_std": 0.008362155756913126, + "rewards/accuracy_reward": 0.21365749835968018, + "step": 1070 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.791666746139526, + "epoch": 0.8350877192982457, + "grad_norm": 10.243919312850661, + "kl": 4.03125, + "learning_rate": 1.7234246302359088e-07, + "loss": 0.2648, + "reward": 0.20929622650146484, + "reward_std": 0.01460318360477686, + "rewards/accuracy_reward": 0.20929622650146484, + "step": 1071 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.4791669845581055, + "epoch": 0.8358674463937622, + "grad_norm": 16.12075424043002, + "kl": 4.09375, + "learning_rate": 1.7167714715821671e-07, + "loss": 0.1907, + "reward": 0.20540675520896912, + "reward_std": 0.022147890646010637, + "rewards/accuracy_reward": 0.20540675520896912, + "step": 1072 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.416666984558105, + "epoch": 0.8366471734892788, + "grad_norm": 12.333060693824349, + "kl": 4.140625, + "learning_rate": 1.7101463998044991e-07, + "loss": 0.2911, + "reward": 0.19160164892673492, + "reward_std": 0.00890958949457854, + "rewards/accuracy_reward": 0.19160164892673492, + "step": 1073 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.937500476837158, + "epoch": 0.8374269005847953, + "grad_norm": 15.598845701690792, + "kl": 4.875, + "learning_rate": 1.7035494640877647e-07, + "loss": 0.3631, + "reward": 0.20432112365961075, + "reward_std": 0.009817011887207627, + "rewards/accuracy_reward": 0.20432112365961075, + "step": 1074 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.229166984558105, + "epoch": 0.8382066276803118, + "grad_norm": 14.73306902829391, + "kl": 3.8125, + "learning_rate": 1.6969807134079434e-07, + "loss": 0.1718, + "reward": 0.17998628318309784, + "reward_std": 0.008431690745055676, + "rewards/accuracy_reward": 0.17998628318309784, + "step": 1075 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.75, + "epoch": 0.8389863547758285, + "grad_norm": 15.513617412255428, + "kl": 5.84375, + "learning_rate": 1.6904401965317646e-07, + "loss": 0.3963, + "reward": 0.21415407955646515, + "reward_std": 0.01877352735027671, + "rewards/accuracy_reward": 0.21415407955646515, + "step": 1076 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.208333492279053, + "epoch": 0.839766081871345, + "grad_norm": 16.032939544231144, + "kl": 5.328125, + "learning_rate": 1.6839279620163505e-07, + "loss": 0.2211, + "reward": 0.21927083283662796, + "reward_std": 0.019846416427753866, + "rewards/accuracy_reward": 0.21927083283662796, + "step": 1077 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.750000238418579, + "epoch": 0.8405458089668616, + "grad_norm": 8.330629812285995, + "kl": 5.703125, + "learning_rate": 1.6774440582088524e-07, + "loss": 0.1835, + "reward": 0.21867559850215912, + "reward_std": 0.02545811329036951, + "rewards/accuracy_reward": 0.21867559850215912, + "step": 1078 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.5, + "epoch": 0.8413255360623781, + "grad_norm": 19.131966075715294, + "kl": 5.96875, + "learning_rate": 1.6709885332460926e-07, + "loss": 0.4623, + "reward": 0.17179232835769653, + "reward_std": 0.02054938394576311, + "rewards/accuracy_reward": 0.17179232835769653, + "step": 1079 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.979166984558105, + "epoch": 0.8421052631578947, + "grad_norm": 10.63528188445893, + "kl": 4.34375, + "learning_rate": 1.6645614350542097e-07, + "loss": 0.2807, + "reward": 0.19352678954601288, + "reward_std": 0.013145028613507748, + "rewards/accuracy_reward": 0.19352678954601288, + "step": 1080 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.25, + "epoch": 0.8428849902534113, + "grad_norm": 9.784262780475066, + "kl": 3.6875, + "learning_rate": 1.658162811348298e-07, + "loss": 0.1786, + "reward": 0.18511030077934265, + "reward_std": 0.026056154631078243, + "rewards/accuracy_reward": 0.18511030077934265, + "step": 1081 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.875, + "epoch": 0.8436647173489279, + "grad_norm": 173.97832607036148, + "kl": 16.3125, + "learning_rate": 1.6517927096320582e-07, + "loss": 1.0345, + "reward": 0.19607388228178024, + "reward_std": 0.03261665580794215, + "rewards/accuracy_reward": 0.19607388228178024, + "step": 1082 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.437500238418579, + "epoch": 0.8444444444444444, + "grad_norm": 16.53077320724463, + "kl": 5.2421875, + "learning_rate": 1.6454511771974406e-07, + "loss": 0.3544, + "reward": 0.20962875336408615, + "reward_std": 0.020014699548482895, + "rewards/accuracy_reward": 0.20962875336408615, + "step": 1083 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.541666984558105, + "epoch": 0.845224171539961, + "grad_norm": 42.62182815112753, + "kl": 7.53125, + "learning_rate": 1.639138261124296e-07, + "loss": 0.4622, + "reward": 0.16087768226861954, + "reward_std": 0.037379974499344826, + "rewards/accuracy_reward": 0.16087768226861954, + "step": 1084 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.4791669845581055, + "epoch": 0.8460038986354775, + "grad_norm": 30.856084535171473, + "kl": 8.53125, + "learning_rate": 1.6328540082800268e-07, + "loss": 0.497, + "reward": 0.21097032725811005, + "reward_std": 0.027320554479956627, + "rewards/accuracy_reward": 0.21097032725811005, + "step": 1085 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.958333492279053, + "epoch": 0.8467836257309942, + "grad_norm": 24.395286835136066, + "kl": 5.359375, + "learning_rate": 1.6265984653192367e-07, + "loss": 0.3167, + "reward": 0.21588580310344696, + "reward_std": 0.024897389113903046, + "rewards/accuracy_reward": 0.21588580310344696, + "step": 1086 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.479166746139526, + "epoch": 0.8475633528265107, + "grad_norm": 8.89274010583597, + "kl": 3.109375, + "learning_rate": 1.6203716786833872e-07, + "loss": 0.1553, + "reward": 0.20885418355464935, + "reward_std": 0.006443641148507595, + "rewards/accuracy_reward": 0.20885418355464935, + "step": 1087 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.666666746139526, + "epoch": 0.8483430799220273, + "grad_norm": 15.980591382919561, + "kl": 5.640625, + "learning_rate": 1.6141736946004507e-07, + "loss": 0.36, + "reward": 0.20034722983837128, + "reward_std": 0.01898013800382614, + "rewards/accuracy_reward": 0.20034722983837128, + "step": 1088 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.937500238418579, + "epoch": 0.8491228070175438, + "grad_norm": 20.49717041289614, + "kl": 7.234375, + "learning_rate": 1.608004559084568e-07, + "loss": 0.4776, + "reward": 0.21609757840633392, + "reward_std": 0.03613634407520294, + "rewards/accuracy_reward": 0.21609757840633392, + "step": 1089 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.645833492279053, + "epoch": 0.8499025341130604, + "grad_norm": 11.02603716971991, + "kl": 4.4140625, + "learning_rate": 1.6018643179357067e-07, + "loss": 0.2682, + "reward": 0.21140385419130325, + "reward_std": 0.016452504321932793, + "rewards/accuracy_reward": 0.21140385419130325, + "step": 1090 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.895833492279053, + "epoch": 0.850682261208577, + "grad_norm": 73.09758019250357, + "kl": 9.265625, + "learning_rate": 1.5957530167393224e-07, + "loss": 0.2924, + "reward": 0.1901041865348816, + "reward_std": 0.02291376329958439, + "rewards/accuracy_reward": 0.1901041865348816, + "step": 1091 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.020833492279053, + "epoch": 0.8514619883040936, + "grad_norm": 23.47442812407075, + "kl": 5.59375, + "learning_rate": 1.5896707008660167e-07, + "loss": 0.3556, + "reward": 0.2113695815205574, + "reward_std": 0.016287715174257755, + "rewards/accuracy_reward": 0.2113695815205574, + "step": 1092 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.270833492279053, + "epoch": 0.8522417153996101, + "grad_norm": 9.563165608181944, + "kl": 3.5703125, + "learning_rate": 1.5836174154712056e-07, + "loss": 0.1525, + "reward": 0.20505953580141068, + "reward_std": 0.005154912360012531, + "rewards/accuracy_reward": 0.20505953580141068, + "step": 1093 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.229166984558105, + "epoch": 0.8530214424951267, + "grad_norm": 17.736260052621304, + "kl": 6.015625, + "learning_rate": 1.577593205494778e-07, + "loss": 0.3716, + "reward": 0.1960604414343834, + "reward_std": 0.031415728852152824, + "rewards/accuracy_reward": 0.1960604414343834, + "step": 1094 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.020833492279053, + "epoch": 0.8538011695906432, + "grad_norm": 22.876046476277946, + "kl": 5.6875, + "learning_rate": 1.571598115660769e-07, + "loss": 0.4149, + "reward": 0.20456326007843018, + "reward_std": 0.021579432766884565, + "rewards/accuracy_reward": 0.20456326007843018, + "step": 1095 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.333333492279053, + "epoch": 0.8545808966861599, + "grad_norm": 12.806809681129913, + "kl": 4.296875, + "learning_rate": 1.5656321904770214e-07, + "loss": 0.2756, + "reward": 0.2205662503838539, + "reward_std": 0.015358997508883476, + "rewards/accuracy_reward": 0.2205662503838539, + "step": 1096 + }, + { + "clip_ratio": 0.0, + "completion_length": 12.895833730697632, + "epoch": 0.8553606237816764, + "grad_norm": 38.40337729864022, + "kl": 5.65625, + "learning_rate": 1.55969547423486e-07, + "loss": 0.7367, + "reward": 0.21845005452632904, + "reward_std": 0.03194329887628555, + "rewards/accuracy_reward": 0.21845005452632904, + "step": 1097 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.895833730697632, + "epoch": 0.856140350877193, + "grad_norm": 15.999375182880474, + "kl": 4.515625, + "learning_rate": 1.5537880110087604e-07, + "loss": 0.2166, + "reward": 0.2007308229804039, + "reward_std": 0.012282190844416618, + "rewards/accuracy_reward": 0.2007308229804039, + "step": 1098 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.000000238418579, + "epoch": 0.8569200779727095, + "grad_norm": 13.29523527642342, + "kl": 3.6640625, + "learning_rate": 1.547909844656022e-07, + "loss": 0.3176, + "reward": 0.21111111342906952, + "reward_std": 0.016218727454543114, + "rewards/accuracy_reward": 0.21111111342906952, + "step": 1099 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.8541669845581055, + "epoch": 0.8576998050682261, + "grad_norm": 22.085077058631693, + "kl": 7.21875, + "learning_rate": 1.5420610188164424e-07, + "loss": 0.5826, + "reward": 0.20067840069532394, + "reward_std": 0.013114954112097621, + "rewards/accuracy_reward": 0.20067840069532394, + "step": 1100 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.791666746139526, + "epoch": 0.8584795321637427, + "grad_norm": 422.52783751527136, + "kl": 29.296875, + "learning_rate": 1.5362415769119962e-07, + "loss": 1.3244, + "reward": 0.2271214872598648, + "reward_std": 0.026086248457431793, + "rewards/accuracy_reward": 0.2271214872598648, + "step": 1101 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.270833492279053, + "epoch": 0.8592592592592593, + "grad_norm": 1.7704852847080452, + "kl": 3.21875, + "learning_rate": 1.5304515621465076e-07, + "loss": 0.1288, + "reward": 0.20000001788139343, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.20000001788139343, + "step": 1102 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.645833492279053, + "epoch": 0.8600389863547758, + "grad_norm": 25.778829730750065, + "kl": 5.6015625, + "learning_rate": 1.5246910175053333e-07, + "loss": 0.3308, + "reward": 0.20887897163629532, + "reward_std": 0.019954406656324863, + "rewards/accuracy_reward": 0.20887897163629532, + "step": 1103 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.604166746139526, + "epoch": 0.8608187134502924, + "grad_norm": 19.133346968288162, + "kl": 5.546875, + "learning_rate": 1.5189599857550417e-07, + "loss": 0.2343, + "reward": 0.18292327225208282, + "reward_std": 0.029843223746865988, + "rewards/accuracy_reward": 0.18292327225208282, + "step": 1104 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.333333492279053, + "epoch": 0.8615984405458089, + "grad_norm": 15.622047407391348, + "kl": 4.1015625, + "learning_rate": 1.5132585094430973e-07, + "loss": 0.2418, + "reward": 0.2036207765340805, + "reward_std": 0.01471833884716034, + "rewards/accuracy_reward": 0.2036207765340805, + "step": 1105 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.5416669845581055, + "epoch": 0.8623781676413256, + "grad_norm": 69.2852733075212, + "kl": 7.796875, + "learning_rate": 1.507586630897542e-07, + "loss": 0.5039, + "reward": 0.21492642164230347, + "reward_std": 0.03321198932826519, + "rewards/accuracy_reward": 0.21492642164230347, + "step": 1106 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.583333492279053, + "epoch": 0.8631578947368421, + "grad_norm": 10.048310485483272, + "kl": 4.8125, + "learning_rate": 1.501944392226684e-07, + "loss": 0.2632, + "reward": 0.20047122985124588, + "reward_std": 0.019584303721785545, + "rewards/accuracy_reward": 0.20047122985124588, + "step": 1107 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.916666746139526, + "epoch": 0.8639376218323587, + "grad_norm": 8.334636581804242, + "kl": 4.421875, + "learning_rate": 1.4963318353187814e-07, + "loss": 0.1982, + "reward": 0.23336397111415863, + "reward_std": 0.011155277490615845, + "rewards/accuracy_reward": 0.23336397111415863, + "step": 1108 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.020833492279053, + "epoch": 0.8647173489278752, + "grad_norm": 14.618675105592677, + "kl": 5.40625, + "learning_rate": 1.4907490018417364e-07, + "loss": 0.3255, + "reward": 0.20911459624767303, + "reward_std": 0.024099217727780342, + "rewards/accuracy_reward": 0.20911459624767303, + "step": 1109 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.875000238418579, + "epoch": 0.8654970760233918, + "grad_norm": 12.747303584440514, + "kl": 4.6796875, + "learning_rate": 1.4851959332427808e-07, + "loss": 0.2918, + "reward": 0.1488143801689148, + "reward_std": 0.019283058121800423, + "rewards/accuracy_reward": 0.1488143801689148, + "step": 1110 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.500000238418579, + "epoch": 0.8662768031189084, + "grad_norm": 17.54463188488618, + "kl": 5.21875, + "learning_rate": 1.4796726707481705e-07, + "loss": 0.2586, + "reward": 0.21874331682920456, + "reward_std": 0.02411574963480234, + "rewards/accuracy_reward": 0.21874331682920456, + "step": 1111 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.500000238418579, + "epoch": 0.867056530214425, + "grad_norm": 23.123309948564504, + "kl": 7.09375, + "learning_rate": 1.474179255362881e-07, + "loss": 0.3821, + "reward": 0.1921483427286148, + "reward_std": 0.04064559563994408, + "rewards/accuracy_reward": 0.1921483427286148, + "step": 1112 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.437500238418579, + "epoch": 0.8678362573099415, + "grad_norm": 12.95162905582068, + "kl": 4.5625, + "learning_rate": 1.4687157278702985e-07, + "loss": 0.2659, + "reward": 0.22105217725038528, + "reward_std": 0.013675682246685028, + "rewards/accuracy_reward": 0.22105217725038528, + "step": 1113 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.645833969116211, + "epoch": 0.8686159844054581, + "grad_norm": 16.9559773382592, + "kl": 6.484375, + "learning_rate": 1.4632821288319243e-07, + "loss": 0.3578, + "reward": 0.20171932876110077, + "reward_std": 0.018030264880508184, + "rewards/accuracy_reward": 0.20171932876110077, + "step": 1114 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.395833492279053, + "epoch": 0.8693957115009746, + "grad_norm": 24.630848662618373, + "kl": 8.3125, + "learning_rate": 1.4578784985870656e-07, + "loss": 0.4284, + "reward": 0.20220322161912918, + "reward_std": 0.03783389087766409, + "rewards/accuracy_reward": 0.20220322161912918, + "step": 1115 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.583333492279053, + "epoch": 0.8701754385964913, + "grad_norm": 10.76136783138401, + "kl": 4.8125, + "learning_rate": 1.4525048772525417e-07, + "loss": 0.2265, + "reward": 0.22199518978595734, + "reward_std": 0.007469884469173849, + "rewards/accuracy_reward": 0.22199518978595734, + "step": 1116 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.791666746139526, + "epoch": 0.8709551656920078, + "grad_norm": 462.75876021620957, + "kl": 75.1875, + "learning_rate": 1.447161304722383e-07, + "loss": 1.5123, + "reward": 0.16037777066230774, + "reward_std": 0.03151809982955456, + "rewards/accuracy_reward": 0.16037777066230774, + "step": 1117 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.541666746139526, + "epoch": 0.8717348927875244, + "grad_norm": 343.9876563620424, + "kl": 14.2421875, + "learning_rate": 1.4418478206675365e-07, + "loss": 0.8938, + "reward": 0.20913627743721008, + "reward_std": 0.011012413073331118, + "rewards/accuracy_reward": 0.20913627743721008, + "step": 1118 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.645833492279053, + "epoch": 0.8725146198830409, + "grad_norm": 1925.6196485974585, + "kl": 30.875, + "learning_rate": 1.4365644645355727e-07, + "loss": 3.2486, + "reward": 0.18649181723594666, + "reward_std": 0.046888433396816254, + "rewards/accuracy_reward": 0.18649181723594666, + "step": 1119 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.666666984558105, + "epoch": 0.8732943469785575, + "grad_norm": 140.65767714461947, + "kl": 13.5625, + "learning_rate": 1.4313112755503874e-07, + "loss": 1.049, + "reward": 0.2146035134792328, + "reward_std": 0.02879335079342127, + "rewards/accuracy_reward": 0.2146035134792328, + "step": 1120 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.500000476837158, + "epoch": 0.8740740740740741, + "grad_norm": 59.719873758277735, + "kl": 8.0, + "learning_rate": 1.4260882927119172e-07, + "loss": 0.5343, + "reward": 0.18853729963302612, + "reward_std": 0.026253076270222664, + "rewards/accuracy_reward": 0.18853729963302612, + "step": 1121 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.895833969116211, + "epoch": 0.8748538011695907, + "grad_norm": 8.89181466582034, + "kl": 3.4140625, + "learning_rate": 1.4208955547958455e-07, + "loss": 0.1375, + "reward": 0.19229472428560257, + "reward_std": 0.005041201948188245, + "rewards/accuracy_reward": 0.19229472428560257, + "step": 1122 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.062500476837158, + "epoch": 0.8756335282651072, + "grad_norm": 11.445772771655074, + "kl": 3.7578125, + "learning_rate": 1.4157331003533152e-07, + "loss": 0.2011, + "reward": 0.21140196174383163, + "reward_std": 0.013236417435109615, + "rewards/accuracy_reward": 0.21140196174383163, + "step": 1123 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.5, + "epoch": 0.8764132553606238, + "grad_norm": 10.780250433067875, + "kl": 3.5546875, + "learning_rate": 1.410600967710644e-07, + "loss": 0.3108, + "reward": 0.2260010838508606, + "reward_std": 0.008903942070901394, + "rewards/accuracy_reward": 0.2260010838508606, + "step": 1124 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.833333492279053, + "epoch": 0.8771929824561403, + "grad_norm": 97.82024019220468, + "kl": 12.125, + "learning_rate": 1.4054991949690399e-07, + "loss": 0.6938, + "reward": 0.1811150163412094, + "reward_std": 0.030483581125736237, + "rewards/accuracy_reward": 0.1811150163412094, + "step": 1125 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.7916669845581055, + "epoch": 0.877972709551657, + "grad_norm": 424.8255646396545, + "kl": 15.640625, + "learning_rate": 1.400427820004316e-07, + "loss": 1.2221, + "reward": 0.1847241073846817, + "reward_std": 0.017680260352790356, + "rewards/accuracy_reward": 0.1847241073846817, + "step": 1126 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.2291669845581055, + "epoch": 0.8787524366471735, + "grad_norm": 13.177723315748436, + "kl": 3.65625, + "learning_rate": 1.3953868804666098e-07, + "loss": 0.1647, + "reward": 0.21326472610235214, + "reward_std": 0.008053896948695183, + "rewards/accuracy_reward": 0.21326472610235214, + "step": 1127 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.312500238418579, + "epoch": 0.87953216374269, + "grad_norm": 15.35623347914603, + "kl": 3.9921875, + "learning_rate": 1.3903764137801087e-07, + "loss": 0.228, + "reward": 0.2282852679491043, + "reward_std": 0.02157183224335313, + "rewards/accuracy_reward": 0.2282852679491043, + "step": 1128 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.895833492279053, + "epoch": 0.8803118908382066, + "grad_norm": 716.5300574091425, + "kl": 29.03125, + "learning_rate": 1.3853964571427647e-07, + "loss": 2.0881, + "reward": 0.20357157289981842, + "reward_std": 0.036397796124219894, + "rewards/accuracy_reward": 0.20357157289981842, + "step": 1129 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.437500476837158, + "epoch": 0.8810916179337231, + "grad_norm": 12.857147794185964, + "kl": 5.3671875, + "learning_rate": 1.3804470475260233e-07, + "loss": 0.2394, + "reward": 0.20706189423799515, + "reward_std": 0.04022690188139677, + "rewards/accuracy_reward": 0.20706189423799515, + "step": 1130 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.750000238418579, + "epoch": 0.8818713450292398, + "grad_norm": 9.32015323076926, + "kl": 5.09375, + "learning_rate": 1.3755282216745453e-07, + "loss": 0.2462, + "reward": 0.21319444477558136, + "reward_std": 0.02095174416899681, + "rewards/accuracy_reward": 0.21319444477558136, + "step": 1131 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.270833969116211, + "epoch": 0.8826510721247564, + "grad_norm": 5.787760223589242, + "kl": 4.1328125, + "learning_rate": 1.3706400161059386e-07, + "loss": 0.1816, + "reward": 0.20977095514535904, + "reward_std": 0.007338681723922491, + "rewards/accuracy_reward": 0.20977095514535904, + "step": 1132 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.250000238418579, + "epoch": 0.8834307992202729, + "grad_norm": 5.163187287093115, + "kl": 4.125, + "learning_rate": 1.3657824671104836e-07, + "loss": 0.2036, + "reward": 0.22553323954343796, + "reward_std": 0.007182053290307522, + "rewards/accuracy_reward": 0.22553323954343796, + "step": 1133 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.979166746139526, + "epoch": 0.8842105263157894, + "grad_norm": 247.3670389819012, + "kl": 18.0625, + "learning_rate": 1.360955610750864e-07, + "loss": 0.9605, + "reward": 0.19943124800920486, + "reward_std": 0.01915727835148573, + "rewards/accuracy_reward": 0.19943124800920486, + "step": 1134 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.0, + "epoch": 0.884990253411306, + "grad_norm": 30.307461728398913, + "kl": 6.265625, + "learning_rate": 1.3561594828619013e-07, + "loss": 0.4557, + "reward": 0.2269803136587143, + "reward_std": 0.01649410603567958, + "rewards/accuracy_reward": 0.2269803136587143, + "step": 1135 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.791666746139526, + "epoch": 0.8857699805068227, + "grad_norm": 74.67816621501852, + "kl": 11.40625, + "learning_rate": 1.3513941190502867e-07, + "loss": 0.9551, + "reward": 0.1939118579030037, + "reward_std": 0.04064749926328659, + "rewards/accuracy_reward": 0.1939118579030037, + "step": 1136 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.354166984558105, + "epoch": 0.8865497076023392, + "grad_norm": 29.435556153930253, + "kl": 7.5, + "learning_rate": 1.346659554694317e-07, + "loss": 0.4278, + "reward": 0.1762477159500122, + "reward_std": 0.022466310299932957, + "rewards/accuracy_reward": 0.1762477159500122, + "step": 1137 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.8125, + "epoch": 0.8873294346978557, + "grad_norm": 8.002436086170635, + "kl": 3.28125, + "learning_rate": 1.3419558249436325e-07, + "loss": 0.1409, + "reward": 0.20936384797096252, + "reward_std": 0.01086391182616353, + "rewards/accuracy_reward": 0.20936384797096252, + "step": 1138 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.458333492279053, + "epoch": 0.8881091617933723, + "grad_norm": 10.483695744828937, + "kl": 4.890625, + "learning_rate": 1.3372829647189556e-07, + "loss": 0.3631, + "reward": 0.1949465423822403, + "reward_std": 0.022146458737552166, + "rewards/accuracy_reward": 0.1949465423822403, + "step": 1139 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.9791669845581055, + "epoch": 0.8888888888888888, + "grad_norm": 71.98654901799392, + "kl": 8.09375, + "learning_rate": 1.3326410087118333e-07, + "loss": 0.5113, + "reward": 0.22195132076740265, + "reward_std": 0.020509536378085613, + "rewards/accuracy_reward": 0.22195132076740265, + "step": 1140 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.270833492279053, + "epoch": 0.8896686159844055, + "grad_norm": 6.163650790043578, + "kl": 3.8125, + "learning_rate": 1.328029991384376e-07, + "loss": 0.1678, + "reward": 0.23054317384958267, + "reward_std": 0.005541531601920724, + "rewards/accuracy_reward": 0.23054317384958267, + "step": 1141 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.250000238418579, + "epoch": 0.890448343079922, + "grad_norm": 20.036961689839035, + "kl": 5.78125, + "learning_rate": 1.3234499469690053e-07, + "loss": 0.3493, + "reward": 0.18308180570602417, + "reward_std": 0.026062607765197754, + "rewards/accuracy_reward": 0.18308180570602417, + "step": 1142 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.3541669845581055, + "epoch": 0.8912280701754386, + "grad_norm": 972.6753125383146, + "kl": 16.234375, + "learning_rate": 1.3189009094681976e-07, + "loss": 1.9322, + "reward": 0.22068586945533752, + "reward_std": 0.02204093337059021, + "rewards/accuracy_reward": 0.22068586945533752, + "step": 1143 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.875000476837158, + "epoch": 0.8920077972709551, + "grad_norm": 10684.773364023755, + "kl": 399.078125, + "learning_rate": 1.3143829126542334e-07, + "loss": 29.1504, + "reward": 0.20947421342134476, + "reward_std": 0.015208574943244457, + "rewards/accuracy_reward": 0.20947421342134476, + "step": 1144 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.041666984558105, + "epoch": 0.8927875243664717, + "grad_norm": 26.873300554003084, + "kl": 6.109375, + "learning_rate": 1.309895990068944e-07, + "loss": 0.364, + "reward": 0.21848131716251373, + "reward_std": 0.02330506592988968, + "rewards/accuracy_reward": 0.21848131716251373, + "step": 1145 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.270833492279053, + "epoch": 0.8935672514619883, + "grad_norm": 7.956893906005812, + "kl": 4.734375, + "learning_rate": 1.3054401750234664e-07, + "loss": 0.2887, + "reward": 0.1941876858472824, + "reward_std": 0.01563416514545679, + "rewards/accuracy_reward": 0.1941876858472824, + "step": 1146 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.437500476837158, + "epoch": 0.8943469785575049, + "grad_norm": 65.8008291993919, + "kl": 8.171875, + "learning_rate": 1.3010155005979906e-07, + "loss": 0.6336, + "reward": 0.22762896120548248, + "reward_std": 0.026936329435557127, + "rewards/accuracy_reward": 0.22762896120548248, + "step": 1147 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.854166984558105, + "epoch": 0.8951267056530214, + "grad_norm": 112.0515083490587, + "kl": 9.40625, + "learning_rate": 1.29662199964152e-07, + "loss": 0.4971, + "reward": 0.16739416122436523, + "reward_std": 0.03169836103916168, + "rewards/accuracy_reward": 0.16739416122436523, + "step": 1148 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.7291669845581055, + "epoch": 0.895906432748538, + "grad_norm": 517.3697121252072, + "kl": 17.640625, + "learning_rate": 1.2922597047716234e-07, + "loss": 1.4871, + "reward": 0.19385723769664764, + "reward_std": 0.015208422672003508, + "rewards/accuracy_reward": 0.19385723769664764, + "step": 1149 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.4375, + "epoch": 0.8966861598440545, + "grad_norm": 11.284650736010855, + "kl": 4.859375, + "learning_rate": 1.2879286483741935e-07, + "loss": 0.322, + "reward": 0.19349761307239532, + "reward_std": 0.02272272203117609, + "rewards/accuracy_reward": 0.19349761307239532, + "step": 1150 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.625, + "epoch": 0.8974658869395712, + "grad_norm": 8.082345545362394, + "kl": 3.7109375, + "learning_rate": 1.2836288626032073e-07, + "loss": 0.2926, + "reward": 0.2061813324689865, + "reward_std": 0.0033308672718703747, + "rewards/accuracy_reward": 0.2061813324689865, + "step": 1151 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.041666984558105, + "epoch": 0.8982456140350877, + "grad_norm": 31.012880045472915, + "kl": 5.625, + "learning_rate": 1.2793603793804885e-07, + "loss": 0.451, + "reward": 0.17605771869421005, + "reward_std": 0.033883729949593544, + "rewards/accuracy_reward": 0.17605771869421005, + "step": 1152 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.645833730697632, + "epoch": 0.8990253411306043, + "grad_norm": 20.032442402711478, + "kl": 5.375, + "learning_rate": 1.275123230395468e-07, + "loss": 0.2227, + "reward": 0.21672210842370987, + "reward_std": 0.012559764087200165, + "rewards/accuracy_reward": 0.21672210842370987, + "step": 1153 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.583333492279053, + "epoch": 0.8998050682261208, + "grad_norm": 9.47799606863283, + "kl": 5.546875, + "learning_rate": 1.270917447104949e-07, + "loss": 0.2963, + "reward": 0.21750182658433914, + "reward_std": 0.015390753746032715, + "rewards/accuracy_reward": 0.21750182658433914, + "step": 1154 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.187500238418579, + "epoch": 0.9005847953216374, + "grad_norm": 4.135377287109526, + "kl": 3.0625, + "learning_rate": 1.266743060732875e-07, + "loss": 0.1235, + "reward": 0.19702380895614624, + "reward_std": 0.0010309829376637936, + "rewards/accuracy_reward": 0.19702380895614624, + "step": 1155 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.000000238418579, + "epoch": 0.901364522417154, + "grad_norm": 12.315522826394604, + "kl": 5.390625, + "learning_rate": 1.262600102270098e-07, + "loss": 0.3911, + "reward": 0.20960648357868195, + "reward_std": 0.019279979169368744, + "rewards/accuracy_reward": 0.20960648357868195, + "step": 1156 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.729166746139526, + "epoch": 0.9021442495126706, + "grad_norm": 75.26008963979227, + "kl": 8.3046875, + "learning_rate": 1.2584886024741467e-07, + "loss": 0.3112, + "reward": 0.21346726268529892, + "reward_std": 0.006082022562623024, + "rewards/accuracy_reward": 0.21346726268529892, + "step": 1157 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.1875, + "epoch": 0.9029239766081871, + "grad_norm": 13.550818569612096, + "kl": 3.8125, + "learning_rate": 1.2544085918689992e-07, + "loss": 0.158, + "reward": 0.18850447237491608, + "reward_std": 0.010908886790275574, + "rewards/accuracy_reward": 0.18850447237491608, + "step": 1158 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.250000238418579, + "epoch": 0.9037037037037037, + "grad_norm": 4.852081209370184, + "kl": 3.59375, + "learning_rate": 1.250360100744857e-07, + "loss": 0.1377, + "reward": 0.2282407358288765, + "reward_std": 0.005406242795288563, + "rewards/accuracy_reward": 0.2282407358288765, + "step": 1159 + }, + { + "clip_ratio": 0.0, + "completion_length": 31.58333396911621, + "epoch": 0.9044834307992202, + "grad_norm": 18.573257939301595, + "kl": 7.171875, + "learning_rate": 1.2463431591579183e-07, + "loss": 0.5888, + "reward": 0.21907051652669907, + "reward_std": 0.030235057114623487, + "rewards/accuracy_reward": 0.21907051652669907, + "step": 1160 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.541666746139526, + "epoch": 0.9052631578947369, + "grad_norm": 33.315793730608064, + "kl": 6.8125, + "learning_rate": 1.2423577969301564e-07, + "loss": 0.4517, + "reward": 0.20457153022289276, + "reward_std": 0.02641383372247219, + "rewards/accuracy_reward": 0.20457153022289276, + "step": 1161 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.520833969116211, + "epoch": 0.9060428849902534, + "grad_norm": 115.1980286593509, + "kl": 6.796875, + "learning_rate": 1.238404043649098e-07, + "loss": 0.448, + "reward": 0.2078033685684204, + "reward_std": 0.022235992830246687, + "rewards/accuracy_reward": 0.2078033685684204, + "step": 1162 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.166666984558105, + "epoch": 0.90682261208577, + "grad_norm": 9.745138487856355, + "kl": 3.4765625, + "learning_rate": 1.234481928667604e-07, + "loss": 0.145, + "reward": 0.17665661126375198, + "reward_std": 0.009434967301785946, + "rewards/accuracy_reward": 0.17665661126375198, + "step": 1163 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.1041669845581055, + "epoch": 0.9076023391812865, + "grad_norm": 40.65086542764804, + "kl": 5.765625, + "learning_rate": 1.2305914811036495e-07, + "loss": 0.2547, + "reward": 0.20849844068288803, + "reward_std": 0.019543007481843233, + "rewards/accuracy_reward": 0.20849844068288803, + "step": 1164 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.895833492279053, + "epoch": 0.9083820662768031, + "grad_norm": 11.226648393999609, + "kl": 4.25, + "learning_rate": 1.226732729840111e-07, + "loss": 0.2436, + "reward": 0.208762988448143, + "reward_std": 0.01885221805423498, + "rewards/accuracy_reward": 0.208762988448143, + "step": 1165 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.625000238418579, + "epoch": 0.9091617933723197, + "grad_norm": 1.8076732133449984, + "kl": 2.5625, + "learning_rate": 1.222905703524548e-07, + "loss": 0.0966, + "reward": 0.21250000596046448, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.21250000596046448, + "step": 1166 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.187500238418579, + "epoch": 0.9099415204678363, + "grad_norm": 27.695314065525512, + "kl": 7.84375, + "learning_rate": 1.2191104305689945e-07, + "loss": 0.4291, + "reward": 0.19272439181804657, + "reward_std": 0.027663799934089184, + "rewards/accuracy_reward": 0.19272439181804657, + "step": 1167 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.770833492279053, + "epoch": 0.9107212475633528, + "grad_norm": 5.968095678215035, + "kl": 3.3046875, + "learning_rate": 1.2153469391497442e-07, + "loss": 0.1415, + "reward": 0.20635723322629929, + "reward_std": 0.0009021099540404975, + "rewards/accuracy_reward": 0.20635723322629929, + "step": 1168 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.1666669845581055, + "epoch": 0.9115009746588694, + "grad_norm": 11.761933997975017, + "kl": 4.1328125, + "learning_rate": 1.2116152572071445e-07, + "loss": 0.2332, + "reward": 0.22108261287212372, + "reward_std": 0.008909666910767555, + "rewards/accuracy_reward": 0.22108261287212372, + "step": 1169 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.937500238418579, + "epoch": 0.9122807017543859, + "grad_norm": 52.160939766803395, + "kl": 6.078125, + "learning_rate": 1.2079154124453864e-07, + "loss": 0.4229, + "reward": 0.2112390249967575, + "reward_std": 0.019674470648169518, + "rewards/accuracy_reward": 0.2112390249967575, + "step": 1170 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.416666746139526, + "epoch": 0.9130604288499026, + "grad_norm": 16.42590522237208, + "kl": 5.5625, + "learning_rate": 1.2042474323323011e-07, + "loss": 0.3279, + "reward": 0.21971727162599564, + "reward_std": 0.013451378792524338, + "rewards/accuracy_reward": 0.21971727162599564, + "step": 1171 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.208333492279053, + "epoch": 0.9138401559454191, + "grad_norm": 83.65911752529338, + "kl": 9.15625, + "learning_rate": 1.2006113440991547e-07, + "loss": 0.5471, + "reward": 0.20092131197452545, + "reward_std": 0.025659140199422836, + "rewards/accuracy_reward": 0.20092131197452545, + "step": 1172 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.0625, + "epoch": 0.9146198830409357, + "grad_norm": 14.190439321257367, + "kl": 5.59375, + "learning_rate": 1.197007174740447e-07, + "loss": 0.3338, + "reward": 0.21384231001138687, + "reward_std": 0.011795071884989738, + "rewards/accuracy_reward": 0.21384231001138687, + "step": 1173 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.4166669845581055, + "epoch": 0.9153996101364522, + "grad_norm": 34609.638390187094, + "kl": 1020.625, + "learning_rate": 1.1934349510137093e-07, + "loss": 73.4034, + "reward": 0.18986543267965317, + "reward_std": 0.02204020880162716, + "rewards/accuracy_reward": 0.18986543267965317, + "step": 1174 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.916666746139526, + "epoch": 0.9161793372319688, + "grad_norm": 10.355660028530062, + "kl": 3.0, + "learning_rate": 1.1898946994393078e-07, + "loss": 0.1932, + "reward": 0.1994047835469246, + "reward_std": 0.014433756470680237, + "rewards/accuracy_reward": 0.1994047835469246, + "step": 1175 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.3125, + "epoch": 0.9169590643274854, + "grad_norm": 29.439951587205623, + "kl": 6.6484375, + "learning_rate": 1.1863864463002456e-07, + "loss": 0.3758, + "reward": 0.18477682024240494, + "reward_std": 0.01697483379393816, + "rewards/accuracy_reward": 0.18477682024240494, + "step": 1176 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.520833492279053, + "epoch": 0.917738791423002, + "grad_norm": 18.86348570769248, + "kl": 6.609375, + "learning_rate": 1.1829102176419677e-07, + "loss": 0.3719, + "reward": 0.21543899178504944, + "reward_std": 0.030653451569378376, + "rewards/accuracy_reward": 0.21543899178504944, + "step": 1177 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.312500238418579, + "epoch": 0.9185185185185185, + "grad_norm": 12.709368015183454, + "kl": 5.03125, + "learning_rate": 1.179466039272168e-07, + "loss": 0.2233, + "reward": 0.21021388471126556, + "reward_std": 0.02950027398765087, + "rewards/accuracy_reward": 0.21021388471126556, + "step": 1178 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.145833492279053, + "epoch": 0.9192982456140351, + "grad_norm": 8.751122325128433, + "kl": 3.12890625, + "learning_rate": 1.1760539367605973e-07, + "loss": 0.1993, + "reward": 0.19363171607255936, + "reward_std": 0.012781540863215923, + "rewards/accuracy_reward": 0.19363171607255936, + "step": 1179 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.375000238418579, + "epoch": 0.9200779727095516, + "grad_norm": 18.81870982814433, + "kl": 4.7421875, + "learning_rate": 1.1726739354388738e-07, + "loss": 0.3269, + "reward": 0.21153492480516434, + "reward_std": 0.01997565571218729, + "rewards/accuracy_reward": 0.21153492480516434, + "step": 1180 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.500000238418579, + "epoch": 0.9208576998050683, + "grad_norm": 18.514301584667013, + "kl": 6.4375, + "learning_rate": 1.1693260604002953e-07, + "loss": 0.3957, + "reward": 0.20284295082092285, + "reward_std": 0.01831751875579357, + "rewards/accuracy_reward": 0.20284295082092285, + "step": 1181 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.041666984558105, + "epoch": 0.9216374269005848, + "grad_norm": 16.032920708448362, + "kl": 5.28125, + "learning_rate": 1.1660103364996511e-07, + "loss": 0.4031, + "reward": 0.20760168880224228, + "reward_std": 0.027552330866456032, + "rewards/accuracy_reward": 0.20760168880224228, + "step": 1182 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.958333492279053, + "epoch": 0.9224171539961014, + "grad_norm": 16.156417531867408, + "kl": 8.46875, + "learning_rate": 1.1627267883530406e-07, + "loss": 0.4259, + "reward": 0.20990155637264252, + "reward_std": 0.0494983084499836, + "rewards/accuracy_reward": 0.20990155637264252, + "step": 1183 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.291666746139526, + "epoch": 0.9231968810916179, + "grad_norm": 6.488863215628915, + "kl": 3.359375, + "learning_rate": 1.1594754403376874e-07, + "loss": 0.1805, + "reward": 0.1920343115925789, + "reward_std": 0.005943312309682369, + "rewards/accuracy_reward": 0.1920343115925789, + "step": 1184 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.125, + "epoch": 0.9239766081871345, + "grad_norm": 14.413775158356152, + "kl": 5.875, + "learning_rate": 1.1562563165917601e-07, + "loss": 0.2764, + "reward": 0.19793479144573212, + "reward_std": 0.042699236422777176, + "rewards/accuracy_reward": 0.19793479144573212, + "step": 1185 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.104166984558105, + "epoch": 0.9247563352826511, + "grad_norm": 3.5160482172980037, + "kl": 3.65625, + "learning_rate": 1.1530694410141929e-07, + "loss": 0.1318, + "reward": 0.19533730298280716, + "reward_std": 0.00583947729319334, + "rewards/accuracy_reward": 0.19533730298280716, + "step": 1186 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.1875, + "epoch": 0.9255360623781677, + "grad_norm": 15.227634775719885, + "kl": 4.984375, + "learning_rate": 1.1499148372645093e-07, + "loss": 0.2668, + "reward": 0.20154379308223724, + "reward_std": 0.02526291087269783, + "rewards/accuracy_reward": 0.20154379308223724, + "step": 1187 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.166666746139526, + "epoch": 0.9263157894736842, + "grad_norm": 12.846189521542879, + "kl": 3.1171875, + "learning_rate": 1.1467925287626427e-07, + "loss": 0.1805, + "reward": 0.19768907874822617, + "reward_std": 0.013208545744419098, + "rewards/accuracy_reward": 0.19768907874822617, + "step": 1188 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.4375, + "epoch": 0.9270955165692008, + "grad_norm": 5.698184317439598, + "kl": 4.09375, + "learning_rate": 1.1437025386887663e-07, + "loss": 0.2474, + "reward": 0.21061508357524872, + "reward_std": 0.007805786095559597, + "rewards/accuracy_reward": 0.21061508357524872, + "step": 1189 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.583333492279053, + "epoch": 0.9278752436647173, + "grad_norm": 8.350626761169142, + "kl": 3.8359375, + "learning_rate": 1.1406448899831195e-07, + "loss": 0.1786, + "reward": 0.2227737009525299, + "reward_std": 0.00771215558052063, + "rewards/accuracy_reward": 0.2227737009525299, + "step": 1190 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.916666984558105, + "epoch": 0.928654970760234, + "grad_norm": 22.456403327207482, + "kl": 3.9375, + "learning_rate": 1.137619605345838e-07, + "loss": 0.2947, + "reward": 0.18116123229265213, + "reward_std": 0.01267927698791027, + "rewards/accuracy_reward": 0.18116123229265213, + "step": 1191 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.520833492279053, + "epoch": 0.9294346978557505, + "grad_norm": 258.7369868877813, + "kl": 15.25, + "learning_rate": 1.1346267072367852e-07, + "loss": 1.0728, + "reward": 0.20177220553159714, + "reward_std": 0.025494796689599752, + "rewards/accuracy_reward": 0.20177220553159714, + "step": 1192 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.5625, + "epoch": 0.930214424951267, + "grad_norm": 31.576097396456294, + "kl": 4.8671875, + "learning_rate": 1.1316662178753838e-07, + "loss": 0.3667, + "reward": 0.19582483172416687, + "reward_std": 0.028432806953787804, + "rewards/accuracy_reward": 0.19582483172416687, + "step": 1193 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.854166746139526, + "epoch": 0.9309941520467836, + "grad_norm": 7.601727572238798, + "kl": 2.609375, + "learning_rate": 1.1287381592404547e-07, + "loss": 0.0806, + "reward": 0.1994047686457634, + "reward_std": 0.014433756470680237, + "rewards/accuracy_reward": 0.1994047686457634, + "step": 1194 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.708333492279053, + "epoch": 0.9317738791423001, + "grad_norm": 12.930760197596525, + "kl": 2.3359375, + "learning_rate": 1.12584255307005e-07, + "loss": 0.1788, + "reward": 0.17796655744314194, + "reward_std": 0.006866272073239088, + "rewards/accuracy_reward": 0.17796655744314194, + "step": 1195 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.9166669845581055, + "epoch": 0.9325536062378168, + "grad_norm": 12.495804845981928, + "kl": 4.28125, + "learning_rate": 1.1229794208612933e-07, + "loss": 0.2171, + "reward": 0.22080034762620926, + "reward_std": 0.006542775314301252, + "rewards/accuracy_reward": 0.22080034762620926, + "step": 1196 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.9375, + "epoch": 0.9333333333333333, + "grad_norm": 8.945474450309012, + "kl": 4.078125, + "learning_rate": 1.1201487838702199e-07, + "loss": 0.1545, + "reward": 0.22458858788013458, + "reward_std": 0.013796973042190075, + "rewards/accuracy_reward": 0.22458858788013458, + "step": 1197 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.458333492279053, + "epoch": 0.9341130604288499, + "grad_norm": 16.513812977339807, + "kl": 5.703125, + "learning_rate": 1.1173506631116191e-07, + "loss": 0.3528, + "reward": 0.19142332673072815, + "reward_std": 0.016939722350798547, + "rewards/accuracy_reward": 0.19142332673072815, + "step": 1198 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.000000476837158, + "epoch": 0.9348927875243664, + "grad_norm": 20.319445965948496, + "kl": 5.53125, + "learning_rate": 1.1145850793588782e-07, + "loss": 0.4997, + "reward": 0.23345467448234558, + "reward_std": 0.011634243186563253, + "rewards/accuracy_reward": 0.23345467448234558, + "step": 1199 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.500000476837158, + "epoch": 0.935672514619883, + "grad_norm": 25.01932496061275, + "kl": 5.921875, + "learning_rate": 1.1118520531438278e-07, + "loss": 0.3437, + "reward": 0.1736689805984497, + "reward_std": 0.02426753845065832, + "rewards/accuracy_reward": 0.1736689805984497, + "step": 1200 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.833333730697632, + "epoch": 0.9364522417153996, + "grad_norm": 3.786562131627424, + "kl": 3.3671875, + "learning_rate": 1.1091516047565902e-07, + "loss": 0.1425, + "reward": 0.22369793057441711, + "reward_std": 0.004510548897087574, + "rewards/accuracy_reward": 0.22369793057441711, + "step": 1201 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.770833492279053, + "epoch": 0.9372319688109162, + "grad_norm": 481.3115022134051, + "kl": 22.78125, + "learning_rate": 1.1064837542454282e-07, + "loss": 1.4153, + "reward": 0.18739942461252213, + "reward_std": 0.03562930226325989, + "rewards/accuracy_reward": 0.18739942461252213, + "step": 1202 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.916666746139526, + "epoch": 0.9380116959064327, + "grad_norm": 11.11658018073384, + "kl": 4.8125, + "learning_rate": 1.1038485214165961e-07, + "loss": 0.2915, + "reward": 0.2019917070865631, + "reward_std": 0.03573805186897516, + "rewards/accuracy_reward": 0.2019917070865631, + "step": 1203 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.645833492279053, + "epoch": 0.9387914230019493, + "grad_norm": 603.6540781376382, + "kl": 20.65625, + "learning_rate": 1.1012459258341926e-07, + "loss": 2.0731, + "reward": 0.18863649666309357, + "reward_std": 0.0340578593313694, + "rewards/accuracy_reward": 0.18863649666309357, + "step": 1204 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.458333492279053, + "epoch": 0.9395711500974658, + "grad_norm": 615.0637870484035, + "kl": 23.28125, + "learning_rate": 1.0986759868200167e-07, + "loss": 1.3454, + "reward": 0.20687806606292725, + "reward_std": 0.02090786350890994, + "rewards/accuracy_reward": 0.20687806606292725, + "step": 1205 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.625000238418579, + "epoch": 0.9403508771929825, + "grad_norm": 36.43824596412844, + "kl": 9.6875, + "learning_rate": 1.096138723453423e-07, + "loss": 0.5498, + "reward": 0.18897167593240738, + "reward_std": 0.025384733453392982, + "rewards/accuracy_reward": 0.18897167593240738, + "step": 1206 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.4375, + "epoch": 0.941130604288499, + "grad_norm": 33.525121208377314, + "kl": 8.546875, + "learning_rate": 1.0936341545711805e-07, + "loss": 0.4616, + "reward": 0.20922619104385376, + "reward_std": 0.03994132578372955, + "rewards/accuracy_reward": 0.20922619104385376, + "step": 1207 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.354166984558105, + "epoch": 0.9419103313840156, + "grad_norm": 902.3143949112615, + "kl": 29.734375, + "learning_rate": 1.0911622987673322e-07, + "loss": 2.8071, + "reward": 0.21465390920639038, + "reward_std": 0.02480224333703518, + "rewards/accuracy_reward": 0.21465390920639038, + "step": 1208 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.166666984558105, + "epoch": 0.9426900584795321, + "grad_norm": 85.01216407765074, + "kl": 6.296875, + "learning_rate": 1.0887231743930586e-07, + "loss": 0.3739, + "reward": 0.1962299346923828, + "reward_std": 0.019409867003560066, + "rewards/accuracy_reward": 0.1962299346923828, + "step": 1209 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.5416669845581055, + "epoch": 0.9434697855750487, + "grad_norm": 26.91519146651911, + "kl": 3.140625, + "learning_rate": 1.0863167995565398e-07, + "loss": 0.1239, + "reward": 0.2066294327378273, + "reward_std": 0.0058517600409686565, + "rewards/accuracy_reward": 0.2066294327378273, + "step": 1210 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.666666984558105, + "epoch": 0.9442495126705653, + "grad_norm": 28.40528646739898, + "kl": 4.09375, + "learning_rate": 1.083943192122822e-07, + "loss": 0.4532, + "reward": 0.1771974265575409, + "reward_std": 0.026410337537527084, + "rewards/accuracy_reward": 0.1771974265575409, + "step": 1211 + }, + { + "clip_ratio": 0.0, + "completion_length": 12.145833492279053, + "epoch": 0.9450292397660819, + "grad_norm": 95.29599006350061, + "kl": 9.0390625, + "learning_rate": 1.0816023697136834e-07, + "loss": 0.2776, + "reward": 0.15802696347236633, + "reward_std": 0.014931716956198215, + "rewards/accuracy_reward": 0.15802696347236633, + "step": 1212 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.270833969116211, + "epoch": 0.9458089668615984, + "grad_norm": 1438.3356344164663, + "kl": 43.328125, + "learning_rate": 1.079294349707507e-07, + "loss": 2.8694, + "reward": 0.18272939324378967, + "reward_std": 0.026362543925642967, + "rewards/accuracy_reward": 0.18272939324378967, + "step": 1213 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.562500238418579, + "epoch": 0.946588693957115, + "grad_norm": 16.099709958046574, + "kl": 7.765625, + "learning_rate": 1.0770191492391467e-07, + "loss": 0.5786, + "reward": 0.21423230320215225, + "reward_std": 0.038808904588222504, + "rewards/accuracy_reward": 0.21423230320215225, + "step": 1214 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.0, + "epoch": 0.9473684210526315, + "grad_norm": 81.7876986375748, + "kl": 8.5, + "learning_rate": 1.0747767851998042e-07, + "loss": 0.6992, + "reward": 0.22134177386760712, + "reward_std": 0.019505458883941174, + "rewards/accuracy_reward": 0.22134177386760712, + "step": 1215 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.583333492279053, + "epoch": 0.9481481481481482, + "grad_norm": 9.954648623766223, + "kl": 3.3984375, + "learning_rate": 1.072567274236901e-07, + "loss": 0.1362, + "reward": 0.21450893580913544, + "reward_std": 0.005412658676505089, + "rewards/accuracy_reward": 0.21450893580913544, + "step": 1216 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.145833492279053, + "epoch": 0.9489278752436647, + "grad_norm": 93.48502279620097, + "kl": 14.296875, + "learning_rate": 1.0703906327539558e-07, + "loss": 0.7091, + "reward": 0.21654657274484634, + "reward_std": 0.033344101160764694, + "rewards/accuracy_reward": 0.21654657274484634, + "step": 1217 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.416666746139526, + "epoch": 0.9497076023391813, + "grad_norm": 20.659318990927677, + "kl": 5.046875, + "learning_rate": 1.068246876910463e-07, + "loss": 0.28, + "reward": 0.2123677358031273, + "reward_std": 0.016037508845329285, + "rewards/accuracy_reward": 0.2123677358031273, + "step": 1218 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.083333492279053, + "epoch": 0.9504873294346978, + "grad_norm": 164.37267681489453, + "kl": 8.796875, + "learning_rate": 1.0661360226217719e-07, + "loss": 0.6351, + "reward": 0.2137548327445984, + "reward_std": 0.011381349992007017, + "rewards/accuracy_reward": 0.2137548327445984, + "step": 1219 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.375000238418579, + "epoch": 0.9512670565302144, + "grad_norm": 19.27620049583751, + "kl": 3.4765625, + "learning_rate": 1.0640580855589704e-07, + "loss": 0.4161, + "reward": 0.2089814841747284, + "reward_std": 0.012188505847007036, + "rewards/accuracy_reward": 0.2089814841747284, + "step": 1220 + }, + { + "clip_ratio": 0.0, + "completion_length": 15.479167461395264, + "epoch": 0.952046783625731, + "grad_norm": 7.730954997286787, + "kl": 1.640625, + "learning_rate": 1.0620130811487648e-07, + "loss": 0.106, + "reward": 0.14662089943885803, + "reward_std": 0.0024267282569780946, + "rewards/accuracy_reward": 0.14662089943885803, + "step": 1221 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.333333730697632, + "epoch": 0.9528265107212476, + "grad_norm": 32.25352365358629, + "kl": 4.9375, + "learning_rate": 1.0600010245733696e-07, + "loss": 0.3226, + "reward": 0.2036026269197464, + "reward_std": 0.014297558926045895, + "rewards/accuracy_reward": 0.2036026269197464, + "step": 1222 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.604166984558105, + "epoch": 0.9536062378167641, + "grad_norm": 13.334690298168347, + "kl": 3.6796875, + "learning_rate": 1.0580219307703931e-07, + "loss": 0.2758, + "reward": 0.21032074093818665, + "reward_std": 0.0075492357718758285, + "rewards/accuracy_reward": 0.21032074093818665, + "step": 1223 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.458333492279053, + "epoch": 0.9543859649122807, + "grad_norm": 1280.042720248613, + "kl": 42.4375, + "learning_rate": 1.0560758144327249e-07, + "loss": 2.3375, + "reward": 0.1932620033621788, + "reward_std": 0.019000491127371788, + "rewards/accuracy_reward": 0.1932620033621788, + "step": 1224 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.125, + "epoch": 0.9551656920077972, + "grad_norm": 3.092564535732343, + "kl": 4.1640625, + "learning_rate": 1.05416269000843e-07, + "loss": 0.1704, + "reward": 0.22726935893297195, + "reward_std": 0.004510548897087574, + "rewards/accuracy_reward": 0.22726935893297195, + "step": 1225 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.229166746139526, + "epoch": 0.9559454191033139, + "grad_norm": 18.288901438928676, + "kl": 4.625, + "learning_rate": 1.0522825717006393e-07, + "loss": 0.3177, + "reward": 0.2161763682961464, + "reward_std": 0.0243800962343812, + "rewards/accuracy_reward": 0.2161763682961464, + "step": 1226 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.416666984558105, + "epoch": 0.9567251461988304, + "grad_norm": 68.73787223708729, + "kl": 8.75, + "learning_rate": 1.0504354734674423e-07, + "loss": 1.0187, + "reward": 0.19006869941949844, + "reward_std": 0.0508610624819994, + "rewards/accuracy_reward": 0.19006869941949844, + "step": 1227 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.645833492279053, + "epoch": 0.957504873294347, + "grad_norm": 17.623840564827777, + "kl": 5.3203125, + "learning_rate": 1.04862140902179e-07, + "loss": 0.3469, + "reward": 0.19324291497468948, + "reward_std": 0.015531366690993309, + "rewards/accuracy_reward": 0.19324291497468948, + "step": 1228 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.0, + "epoch": 0.9582846003898635, + "grad_norm": 11.588559130133975, + "kl": 3.65625, + "learning_rate": 1.0468403918313855e-07, + "loss": 0.1767, + "reward": 0.21558122336864471, + "reward_std": 0.0016980890650302172, + "rewards/accuracy_reward": 0.21558122336864471, + "step": 1229 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.75, + "epoch": 0.9590643274853801, + "grad_norm": 12.342223267124131, + "kl": 3.9921875, + "learning_rate": 1.0450924351185888e-07, + "loss": 0.1872, + "reward": 0.22793931514024734, + "reward_std": 0.008539648144505918, + "rewards/accuracy_reward": 0.22793931514024734, + "step": 1230 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.145833492279053, + "epoch": 0.9598440545808967, + "grad_norm": 61.24098797262069, + "kl": 4.6875, + "learning_rate": 1.0433775518603174e-07, + "loss": 0.2307, + "reward": 0.2035786584019661, + "reward_std": 0.009253811556845903, + "rewards/accuracy_reward": 0.2035786584019661, + "step": 1231 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.583333492279053, + "epoch": 0.9606237816764133, + "grad_norm": 13.356084363838436, + "kl": 4.5625, + "learning_rate": 1.04169575478795e-07, + "loss": 0.3373, + "reward": 0.20350747555494308, + "reward_std": 0.020974313840270042, + "rewards/accuracy_reward": 0.20350747555494308, + "step": 1232 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.5, + "epoch": 0.9614035087719298, + "grad_norm": 8.445724184250526, + "kl": 4.03125, + "learning_rate": 1.0400470563872306e-07, + "loss": 0.1708, + "reward": 0.17264822870492935, + "reward_std": 0.0009540432947687805, + "rewards/accuracy_reward": 0.17264822870492935, + "step": 1233 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.979166746139526, + "epoch": 0.9621832358674464, + "grad_norm": 18.51814365364438, + "kl": 4.15625, + "learning_rate": 1.0384314688981785e-07, + "loss": 0.2084, + "reward": 0.21725810319185257, + "reward_std": 0.007227339199744165, + "rewards/accuracy_reward": 0.21725810319185257, + "step": 1234 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.083333730697632, + "epoch": 0.9629629629629629, + "grad_norm": 8.989129470598803, + "kl": 5.375, + "learning_rate": 1.0368490043149947e-07, + "loss": 0.341, + "reward": 0.20741870999336243, + "reward_std": 0.01566131366416812, + "rewards/accuracy_reward": 0.20741870999336243, + "step": 1235 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.6041669845581055, + "epoch": 0.9637426900584796, + "grad_norm": 19.988993760877054, + "kl": 5.453125, + "learning_rate": 1.0352996743859747e-07, + "loss": 0.3337, + "reward": 0.2078678384423256, + "reward_std": 0.02216923190280795, + "rewards/accuracy_reward": 0.2078678384423256, + "step": 1236 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.291666746139526, + "epoch": 0.9645224171539961, + "grad_norm": 1.6801326524284173, + "kl": 3.53125, + "learning_rate": 1.0337834906134214e-07, + "loss": 0.141, + "reward": 0.22857142984867096, + "reward_std": 0.0, + "rewards/accuracy_reward": 0.22857142984867096, + "step": 1237 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.270833492279053, + "epoch": 0.9653021442495127, + "grad_norm": 13.702818498569883, + "kl": 3.5390625, + "learning_rate": 1.0323004642535574e-07, + "loss": 0.1354, + "reward": 0.22127975523471832, + "reward_std": 0.012887283228337765, + "rewards/accuracy_reward": 0.22127975523471832, + "step": 1238 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.1041669845581055, + "epoch": 0.9660818713450292, + "grad_norm": 15.791125628059532, + "kl": 4.765625, + "learning_rate": 1.0308506063164444e-07, + "loss": 0.2684, + "reward": 0.2126072272658348, + "reward_std": 0.018195314332842827, + "rewards/accuracy_reward": 0.2126072272658348, + "step": 1239 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.979166746139526, + "epoch": 0.9668615984405458, + "grad_norm": 62443.014210519956, + "kl": 1961.703125, + "learning_rate": 1.0294339275658995e-07, + "loss": 140.2023, + "reward": 0.1819750964641571, + "reward_std": 0.024145943112671375, + "rewards/accuracy_reward": 0.1819750964641571, + "step": 1240 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.062500476837158, + "epoch": 0.9676413255360624, + "grad_norm": 18.503225726011376, + "kl": 6.875, + "learning_rate": 1.0280504385194162e-07, + "loss": 0.2852, + "reward": 0.20905672013759613, + "reward_std": 0.03467312082648277, + "rewards/accuracy_reward": 0.20905672013759613, + "step": 1241 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.3541669845581055, + "epoch": 0.968421052631579, + "grad_norm": 74.83209982632016, + "kl": 10.09375, + "learning_rate": 1.0267001494480849e-07, + "loss": 0.5172, + "reward": 0.2077132984995842, + "reward_std": 0.03274921886622906, + "rewards/accuracy_reward": 0.2077132984995842, + "step": 1242 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.750000238418579, + "epoch": 0.9692007797270955, + "grad_norm": 6954.26387137655, + "kl": 121.25, + "learning_rate": 1.0253830703765194e-07, + "loss": 11.2978, + "reward": 0.18995098769664764, + "reward_std": 0.029535386711359024, + "rewards/accuracy_reward": 0.18995098769664764, + "step": 1243 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.125, + "epoch": 0.969980506822612, + "grad_norm": 6.886992709405694, + "kl": 3.265625, + "learning_rate": 1.0240992110827795e-07, + "loss": 0.1332, + "reward": 0.21111111342906952, + "reward_std": 0.0048112524673342705, + "rewards/accuracy_reward": 0.21111111342906952, + "step": 1244 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.520833492279053, + "epoch": 0.9707602339181286, + "grad_norm": 26.88384821686633, + "kl": 6.109375, + "learning_rate": 1.0228485810983002e-07, + "loss": 0.3363, + "reward": 0.1592872068285942, + "reward_std": 0.034972164779901505, + "rewards/accuracy_reward": 0.1592872068285942, + "step": 1245 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.750000238418579, + "epoch": 0.9715399610136453, + "grad_norm": 6.611852730437946, + "kl": 3.71875, + "learning_rate": 1.0216311897078211e-07, + "loss": 0.2004, + "reward": 0.21078432351350784, + "reward_std": 0.005943312309682369, + "rewards/accuracy_reward": 0.21078432351350784, + "step": 1246 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.562500238418579, + "epoch": 0.9723196881091618, + "grad_norm": 287.6797822502787, + "kl": 18.25, + "learning_rate": 1.0204470459493156e-07, + "loss": 1.0408, + "reward": 0.21247275918722153, + "reward_std": 0.037593014538288116, + "rewards/accuracy_reward": 0.21247275918722153, + "step": 1247 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.458333492279053, + "epoch": 0.9730994152046784, + "grad_norm": 12.864392783785085, + "kl": 4.953125, + "learning_rate": 1.0192961586139258e-07, + "loss": 0.2233, + "reward": 0.19879601150751114, + "reward_std": 0.023160387645475566, + "rewards/accuracy_reward": 0.19879601150751114, + "step": 1248 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.354166746139526, + "epoch": 0.9738791423001949, + "grad_norm": 32.808840714369744, + "kl": 7.171875, + "learning_rate": 1.0181785362458968e-07, + "loss": 0.4135, + "reward": 0.22201142460107803, + "reward_std": 0.02556948084384203, + "rewards/accuracy_reward": 0.22201142460107803, + "step": 1249 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.708333492279053, + "epoch": 0.9746588693957114, + "grad_norm": 39.84092094840707, + "kl": 6.65625, + "learning_rate": 1.0170941871425117e-07, + "loss": 0.3967, + "reward": 0.21897321939468384, + "reward_std": 0.01583898440003395, + "rewards/accuracy_reward": 0.21897321939468384, + "step": 1250 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.916666746139526, + "epoch": 0.9754385964912281, + "grad_norm": 10.262711165620495, + "kl": 2.3125, + "learning_rate": 1.0160431193540322e-07, + "loss": 0.0968, + "reward": 0.203311026096344, + "reward_std": 0.0009021099540404975, + "rewards/accuracy_reward": 0.203311026096344, + "step": 1251 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.145833492279053, + "epoch": 0.9762183235867447, + "grad_norm": 8.736925306283432, + "kl": 3.1171875, + "learning_rate": 1.0150253406836379e-07, + "loss": 0.1789, + "reward": 0.1967427283525467, + "reward_std": 0.002004689071327448, + "rewards/accuracy_reward": 0.1967427283525467, + "step": 1252 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.083333492279053, + "epoch": 0.9769980506822612, + "grad_norm": 23.63281770606272, + "kl": 8.15625, + "learning_rate": 1.0140408586873678e-07, + "loss": 0.511, + "reward": 0.19002976268529892, + "reward_std": 0.03935716859996319, + "rewards/accuracy_reward": 0.19002976268529892, + "step": 1253 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.6041669845581055, + "epoch": 0.9777777777777777, + "grad_norm": 15.597506038748369, + "kl": 5.7734375, + "learning_rate": 1.0130896806740653e-07, + "loss": 0.1966, + "reward": 0.2057291641831398, + "reward_std": 0.023454854264855385, + "rewards/accuracy_reward": 0.2057291641831398, + "step": 1254 + }, + { + "clip_ratio": 0.0, + "completion_length": 9.562500238418579, + "epoch": 0.9785575048732943, + "grad_norm": 139.57860657873223, + "kl": 12.015625, + "learning_rate": 1.0121718137053227e-07, + "loss": 0.5991, + "reward": 0.1736689880490303, + "reward_std": 0.02626141719520092, + "rewards/accuracy_reward": 0.1736689880490303, + "step": 1255 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.5625, + "epoch": 0.979337231968811, + "grad_norm": 4.295624653767669, + "kl": 3.796875, + "learning_rate": 1.0112872645954301e-07, + "loss": 0.1564, + "reward": 0.199768528342247, + "reward_std": 0.007151786237955093, + "rewards/accuracy_reward": 0.199768528342247, + "step": 1256 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.020833492279053, + "epoch": 0.9801169590643275, + "grad_norm": 12.883196542470925, + "kl": 4.0625, + "learning_rate": 1.0104360399113245e-07, + "loss": 0.2487, + "reward": 0.22572863847017288, + "reward_std": 0.013378364033997059, + "rewards/accuracy_reward": 0.22572863847017288, + "step": 1257 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.958333492279053, + "epoch": 0.980896686159844, + "grad_norm": 8.154645974639108, + "kl": 4.765625, + "learning_rate": 1.009618145972539e-07, + "loss": 0.2565, + "reward": 0.20494791120290756, + "reward_std": 0.013742412440478802, + "rewards/accuracy_reward": 0.20494791120290756, + "step": 1258 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.229166746139526, + "epoch": 0.9816764132553606, + "grad_norm": 161.61828964264342, + "kl": 9.03125, + "learning_rate": 1.0088335888511596e-07, + "loss": 0.4625, + "reward": 0.18164489418268204, + "reward_std": 0.028004980646073818, + "rewards/accuracy_reward": 0.18164489418268204, + "step": 1259 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.270833492279053, + "epoch": 0.9824561403508771, + "grad_norm": 231.5601376793861, + "kl": 13.078125, + "learning_rate": 1.0080823743717767e-07, + "loss": 0.6933, + "reward": 0.23341548442840576, + "reward_std": 0.021758271381258965, + "rewards/accuracy_reward": 0.23341548442840576, + "step": 1260 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.833333492279053, + "epoch": 0.9832358674463938, + "grad_norm": 21.823511268124925, + "kl": 8.234375, + "learning_rate": 1.0073645081114439e-07, + "loss": 0.5074, + "reward": 0.1992010995745659, + "reward_std": 0.03368851449340582, + "rewards/accuracy_reward": 0.1992010995745659, + "step": 1261 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.062500238418579, + "epoch": 0.9840155945419103, + "grad_norm": 11.9517005931049, + "kl": 5.703125, + "learning_rate": 1.0066799953996367e-07, + "loss": 0.3365, + "reward": 0.20743610709905624, + "reward_std": 0.016178769059479237, + "rewards/accuracy_reward": 0.20743610709905624, + "step": 1262 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.1666669845581055, + "epoch": 0.9847953216374269, + "grad_norm": 8.902749826657654, + "kl": 4.953125, + "learning_rate": 1.0060288413182106e-07, + "loss": 0.2441, + "reward": 0.22991492599248886, + "reward_std": 0.005898410454392433, + "rewards/accuracy_reward": 0.22991492599248886, + "step": 1263 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.895833492279053, + "epoch": 0.9855750487329434, + "grad_norm": 69.05943044232757, + "kl": 6.59375, + "learning_rate": 1.0054110507013654e-07, + "loss": 0.8897, + "reward": 0.21852556616067886, + "reward_std": 0.014509346336126328, + "rewards/accuracy_reward": 0.21852556616067886, + "step": 1264 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.458333492279053, + "epoch": 0.98635477582846, + "grad_norm": 12.546071757736438, + "kl": 5.0, + "learning_rate": 1.0048266281356104e-07, + "loss": 0.3133, + "reward": 0.19441090524196625, + "reward_std": 0.023479340597987175, + "rewards/accuracy_reward": 0.19441090524196625, + "step": 1265 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.895833730697632, + "epoch": 0.9871345029239766, + "grad_norm": 28.319472096410852, + "kl": 7.28125, + "learning_rate": 1.0042755779597266e-07, + "loss": 0.6178, + "reward": 0.20146438479423523, + "reward_std": 0.0390668660402298, + "rewards/accuracy_reward": 0.20146438479423523, + "step": 1266 + }, + { + "clip_ratio": 0.0, + "completion_length": 12.645833492279053, + "epoch": 0.9879142300194932, + "grad_norm": 12.912836876293984, + "kl": 3.0, + "learning_rate": 1.0037579042647388e-07, + "loss": 0.1384, + "reward": 0.16597173362970352, + "reward_std": 0.006705150939524174, + "rewards/accuracy_reward": 0.16597173362970352, + "step": 1267 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.375000238418579, + "epoch": 0.9886939571150097, + "grad_norm": 11.633395365255938, + "kl": 5.1953125, + "learning_rate": 1.0032736108938814e-07, + "loss": 0.3637, + "reward": 0.2040233239531517, + "reward_std": 0.01249195309355855, + "rewards/accuracy_reward": 0.2040233239531517, + "step": 1268 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.437500476837158, + "epoch": 0.9894736842105263, + "grad_norm": 45.452321688826196, + "kl": 7.8125, + "learning_rate": 1.0028227014425727e-07, + "loss": 0.615, + "reward": 0.19626323133707047, + "reward_std": 0.032283578999340534, + "rewards/accuracy_reward": 0.19626323133707047, + "step": 1269 + }, + { + "clip_ratio": 0.0, + "completion_length": 10.270833492279053, + "epoch": 0.9902534113060428, + "grad_norm": 37.94943283136805, + "kl": 7.46875, + "learning_rate": 1.0024051792583864e-07, + "loss": 0.3525, + "reward": 0.1793772429227829, + "reward_std": 0.022501050494611263, + "rewards/accuracy_reward": 0.1793772429227829, + "step": 1270 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.729166746139526, + "epoch": 0.9910331384015595, + "grad_norm": 7.035641756101844, + "kl": 2.9921875, + "learning_rate": 1.0020210474410287e-07, + "loss": 0.1701, + "reward": 0.2020089402794838, + "reward_std": 0.005412658676505089, + "rewards/accuracy_reward": 0.2020089402794838, + "step": 1271 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.208333492279053, + "epoch": 0.991812865497076, + "grad_norm": 56.37035675158799, + "kl": 6.875, + "learning_rate": 1.0016703088423122e-07, + "loss": 0.3265, + "reward": 0.18746256083250046, + "reward_std": 0.02559092827141285, + "rewards/accuracy_reward": 0.18746256083250046, + "step": 1272 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.0, + "epoch": 0.9925925925925926, + "grad_norm": 8.637001339240673, + "kl": 3.4296875, + "learning_rate": 1.0013529660661385e-07, + "loss": 0.1913, + "reward": 0.19130875170230865, + "reward_std": 0.01228992361575365, + "rewards/accuracy_reward": 0.19130875170230865, + "step": 1273 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.8541669845581055, + "epoch": 0.9933723196881091, + "grad_norm": 39.20174231712636, + "kl": 4.8046875, + "learning_rate": 1.0010690214684763e-07, + "loss": 0.2005, + "reward": 0.20710954070091248, + "reward_std": 0.006324126850813627, + "rewards/accuracy_reward": 0.20710954070091248, + "step": 1274 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.083333492279053, + "epoch": 0.9941520467836257, + "grad_norm": 19.651620932828706, + "kl": 5.140625, + "learning_rate": 1.0008184771573434e-07, + "loss": 0.2307, + "reward": 0.20983947068452835, + "reward_std": 0.02158815204165876, + "rewards/accuracy_reward": 0.20983947068452835, + "step": 1275 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.3541669845581055, + "epoch": 0.9949317738791423, + "grad_norm": 31.11847289386265, + "kl": 4.875, + "learning_rate": 1.0006013349927938e-07, + "loss": 0.2856, + "reward": 0.20519594103097916, + "reward_std": 0.0067443439620547, + "rewards/accuracy_reward": 0.20519594103097916, + "step": 1276 + }, + { + "clip_ratio": 0.0, + "completion_length": 14.479166984558105, + "epoch": 0.9957115009746589, + "grad_norm": 157.28736066483953, + "kl": 12.125, + "learning_rate": 1.0004175965869012e-07, + "loss": 0.7557, + "reward": 0.1874784380197525, + "reward_std": 0.014421529602259398, + "rewards/accuracy_reward": 0.1874784380197525, + "step": 1277 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.208333492279053, + "epoch": 0.9964912280701754, + "grad_norm": 21.41478823472146, + "kl": 7.296875, + "learning_rate": 1.0002672633037497e-07, + "loss": 0.4059, + "reward": 0.22284622490406036, + "reward_std": 0.015839688014239073, + "rewards/accuracy_reward": 0.22284622490406036, + "step": 1278 + }, + { + "clip_ratio": 0.0, + "completion_length": 8.104166984558105, + "epoch": 0.997270955165692, + "grad_norm": 6.684971065452833, + "kl": 4.1484375, + "learning_rate": 1.0001503362594198e-07, + "loss": 0.1739, + "reward": 0.20092594623565674, + "reward_std": 0.006838416215032339, + "rewards/accuracy_reward": 0.20092594623565674, + "step": 1279 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.958333492279053, + "epoch": 0.9980506822612085, + "grad_norm": 9.472871495229432, + "kl": 3.5546875, + "learning_rate": 1.0000668163219843e-07, + "loss": 0.1427, + "reward": 0.2108631134033203, + "reward_std": 0.018042195588350296, + "rewards/accuracy_reward": 0.2108631134033203, + "step": 1280 + }, + { + "clip_ratio": 0.0, + "completion_length": 7.750000238418579, + "epoch": 0.9988304093567252, + "grad_norm": 11.175917789351356, + "kl": 4.171875, + "learning_rate": 1.000016704111499e-07, + "loss": 0.2831, + "reward": 0.19787301868200302, + "reward_std": 0.023838849738240242, + "rewards/accuracy_reward": 0.19787301868200302, + "step": 1281 + }, + { + "clip_ratio": 0.0, + "completion_length": 6.833333253860474, + "epoch": 0.9996101364522417, + "grad_norm": 13.260794823295758, + "kl": 3.9375, + "learning_rate": 1e-07, + "loss": 0.2462, + "reward": 0.2180059626698494, + "reward_std": 0.018042195588350296, + "rewards/accuracy_reward": 0.2180059626698494, + "step": 1282 + }, + { + "epoch": 0.9996101364522417, + "step": 1282, + "total_flos": 0.0, + "train_loss": 0.6022085310079814, + "train_runtime": 10443.6197, + "train_samples_per_second": 0.491, + "train_steps_per_second": 0.123 + } + ], + "logging_steps": 1, + "max_steps": 1282, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 200, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +}