| { |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.2547121752419766, |
| "eval_steps": 500, |
| "global_step": 500, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 321.578125, |
| "epoch": 0.0005094243504839531, |
| "grad_norm": 21.497011168465292, |
| "kl": 0.0, |
| "learning_rate": 9.997452878247579e-07, |
| "loss": -0.0, |
| "reward": -0.492842435836792, |
| "reward_std": 0.7784243226051331, |
| "rewards/accuracy_reward": -0.4125000238418579, |
| "rewards/cosine_rewards": -0.08018936403095722, |
| "rewards/format_reward": 0.0, |
| "rewards/repetition_rewards": -0.0001530575300421333, |
| "step": 1 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 211.796875, |
| "epoch": 0.0010188487009679063, |
| "grad_norm": 8.570878529351686, |
| "kl": 0.00115203857421875, |
| "learning_rate": 9.99490575649516e-07, |
| "loss": 0.0, |
| "reward": -0.2021125927567482, |
| "reward_std": 0.686398446559906, |
| "rewards/accuracy_reward": -0.18437501601874828, |
| "rewards/cosine_rewards": -0.01752197090536356, |
| "rewards/format_reward": 0.0, |
| "rewards/repetition_rewards": -0.00021561131143243983, |
| "step": 2 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 242.640625, |
| "epoch": 0.0015282730514518594, |
| "grad_norm": 7.698910727869972, |
| "kl": 0.0014190673828125, |
| "learning_rate": 9.99235863474274e-07, |
| "loss": 0.0001, |
| "reward": -0.6304773092269897, |
| "reward_std": 0.5950716435909271, |
| "rewards/accuracy_reward": -0.6093750298023224, |
| "rewards/cosine_rewards": -0.03664374351501465, |
| "rewards/format_reward": 0.015625, |
| "rewards/repetition_rewards": -8.355615136679262e-05, |
| "step": 3 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 192.765625, |
| "epoch": 0.0020376974019358125, |
| "grad_norm": 8.264023776311538, |
| "kl": 0.00258636474609375, |
| "learning_rate": 9.98981151299032e-07, |
| "loss": 0.0001, |
| "reward": -0.4020528346300125, |
| "reward_std": 0.7227448225021362, |
| "rewards/accuracy_reward": -0.38750001788139343, |
| "rewards/cosine_rewards": -0.014348747674375772, |
| "rewards/format_reward": 0.0, |
| "rewards/repetition_rewards": -0.00020408956333994865, |
| "step": 4 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 199.953125, |
| "epoch": 0.0025471217524197657, |
| "grad_norm": 9.41735274952485, |
| "kl": 0.00286865234375, |
| "learning_rate": 9.9872643912379e-07, |
| "loss": 0.0001, |
| "reward": -0.45950669050216675, |
| "reward_std": 0.6219092607498169, |
| "rewards/accuracy_reward": -0.4343750476837158, |
| "rewards/cosine_rewards": -0.02503613755106926, |
| "rewards/format_reward": 0.0, |
| "rewards/repetition_rewards": -9.553764903103001e-05, |
| "step": 5 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 197.9375, |
| "epoch": 0.003056546102903719, |
| "grad_norm": 12.944765767909546, |
| "kl": 0.008697509765625, |
| "learning_rate": 9.984717269485481e-07, |
| "loss": 0.0003, |
| "reward": -0.42242346704006195, |
| "reward_std": 0.6794147342443466, |
| "rewards/accuracy_reward": -0.40937504172325134, |
| "rewards/cosine_rewards": -0.028209966607391834, |
| "rewards/format_reward": 0.015625, |
| "rewards/repetition_rewards": -0.0004634863289538771, |
| "step": 6 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 131.859375, |
| "epoch": 0.003565970453387672, |
| "grad_norm": 10.259430825273313, |
| "kl": 0.013763427734375, |
| "learning_rate": 9.98217014773306e-07, |
| "loss": 0.0005, |
| "reward": -0.33318234980106354, |
| "reward_std": 0.7437820434570312, |
| "rewards/accuracy_reward": -0.35625000298023224, |
| "rewards/cosine_rewards": -0.00804880098439753, |
| "rewards/format_reward": 0.03125, |
| "rewards/repetition_rewards": -0.00013354701513890177, |
| "step": 7 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 138.0625, |
| "epoch": 0.004075394803871625, |
| "grad_norm": 8.664940595308508, |
| "kl": 0.01800537109375, |
| "learning_rate": 9.979623025980642e-07, |
| "loss": 0.0007, |
| "reward": -0.3353596553206444, |
| "reward_std": 0.7424190640449524, |
| "rewards/accuracy_reward": -0.32500002533197403, |
| "rewards/cosine_rewards": -0.010187382809817791, |
| "rewards/format_reward": 0.0, |
| "rewards/repetition_rewards": -0.00017226976342499256, |
| "step": 8 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 130.609375, |
| "epoch": 0.004584819154355578, |
| "grad_norm": 12.906962146752678, |
| "kl": 0.013641357421875, |
| "learning_rate": 9.977075904228221e-07, |
| "loss": 0.0005, |
| "reward": -0.5576262176036835, |
| "reward_std": 0.38936011493206024, |
| "rewards/accuracy_reward": -0.546875, |
| "rewards/cosine_rewards": -0.010545612312853336, |
| "rewards/format_reward": 0.0, |
| "rewards/repetition_rewards": -0.00020559210679493845, |
| "step": 9 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 118.375, |
| "epoch": 0.005094243504839531, |
| "grad_norm": 12.675664846435772, |
| "kl": 0.014129638671875, |
| "learning_rate": 9.974528782475803e-07, |
| "loss": 0.0006, |
| "reward": -0.5825353264808655, |
| "reward_std": 0.32141495356336236, |
| "rewards/accuracy_reward": -0.5750000178813934, |
| "rewards/cosine_rewards": -0.0075353041756898165, |
| "rewards/format_reward": 0.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 10 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 116.5625, |
| "epoch": 0.0056036678553234845, |
| "grad_norm": 83.14378275688269, |
| "kl": 0.011932373046875, |
| "learning_rate": 9.971981660723382e-07, |
| "loss": 0.0005, |
| "reward": -0.4973638355731964, |
| "reward_std": 0.6479763090610504, |
| "rewards/accuracy_reward": -0.4906250536441803, |
| "rewards/cosine_rewards": -0.006738818949088454, |
| "rewards/format_reward": 0.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 11 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 122.5, |
| "epoch": 0.006113092205807438, |
| "grad_norm": 10.015051037156322, |
| "kl": 0.01776123046875, |
| "learning_rate": 9.969434538970963e-07, |
| "loss": 0.0007, |
| "reward": -0.5842953324317932, |
| "reward_std": 0.3923248201608658, |
| "rewards/accuracy_reward": -0.5750000029802322, |
| "rewards/cosine_rewards": -0.009295305702835321, |
| "rewards/format_reward": 0.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 12 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.984375, |
| "epoch": 0.006622516556291391, |
| "grad_norm": 11.394446741932766, |
| "kl": 0.018157958984375, |
| "learning_rate": 9.966887417218542e-07, |
| "loss": 0.0007, |
| "reward": -0.5545713007450104, |
| "reward_std": 0.5603736639022827, |
| "rewards/accuracy_reward": -0.5468750298023224, |
| "rewards/cosine_rewards": -0.007696274435147643, |
| "rewards/format_reward": 0.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 13 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.8125, |
| "epoch": 0.007131940906775344, |
| "grad_norm": 11.774338615514537, |
| "kl": 0.017730712890625, |
| "learning_rate": 9.964340295466124e-07, |
| "loss": 0.0007, |
| "reward": -0.24103393778204918, |
| "reward_std": 0.770084798336029, |
| "rewards/accuracy_reward": -0.23750002682209015, |
| "rewards/cosine_rewards": -0.0035339330206625164, |
| "rewards/format_reward": 0.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 14 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 99.5, |
| "epoch": 0.007641365257259297, |
| "grad_norm": 12.461454822945, |
| "kl": 0.01995849609375, |
| "learning_rate": 9.961793173713703e-07, |
| "loss": 0.0008, |
| "reward": -0.7055607736110687, |
| "reward_std": 0.2303236834704876, |
| "rewards/accuracy_reward": -0.7156250178813934, |
| "rewards/cosine_rewards": -0.005560769001021981, |
| "rewards/format_reward": 0.015625, |
| "rewards/repetition_rewards": 0.0, |
| "step": 15 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 101.53125, |
| "epoch": 0.00815078960774325, |
| "grad_norm": 16.951183982865736, |
| "kl": 0.0206298828125, |
| "learning_rate": 9.959246051961282e-07, |
| "loss": 0.0008, |
| "reward": -0.3540929928421974, |
| "reward_std": 0.7245323657989502, |
| "rewards/accuracy_reward": -0.3500000238418579, |
| "rewards/cosine_rewards": -0.004092983668670058, |
| "rewards/format_reward": 0.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 16 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 96.578125, |
| "epoch": 0.008660213958227204, |
| "grad_norm": 9.458837096460513, |
| "kl": 0.025634765625, |
| "learning_rate": 9.956698930208864e-07, |
| "loss": 0.001, |
| "reward": -0.36599001288414, |
| "reward_std": 0.6569808125495911, |
| "rewards/accuracy_reward": -0.37812502682209015, |
| "rewards/cosine_rewards": -0.003489995375275612, |
| "rewards/format_reward": 0.015625, |
| "rewards/repetition_rewards": 0.0, |
| "step": 17 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 96.625, |
| "epoch": 0.009169638308711156, |
| "grad_norm": 11.937426620152417, |
| "kl": 0.02801513671875, |
| "learning_rate": 9.954151808456443e-07, |
| "loss": 0.0011, |
| "reward": -0.40710097551345825, |
| "reward_std": 0.7412720322608948, |
| "rewards/accuracy_reward": -0.43437501788139343, |
| "rewards/cosine_rewards": -0.003975986503064632, |
| "rewards/format_reward": 0.03125, |
| "rewards/repetition_rewards": 0.0, |
| "step": 18 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 97.21875, |
| "epoch": 0.00967906265919511, |
| "grad_norm": 12.317962197180934, |
| "kl": 0.03466796875, |
| "learning_rate": 9.951604686704024e-07, |
| "loss": 0.0014, |
| "reward": -0.25013431906700134, |
| "reward_std": 0.7123757898807526, |
| "rewards/accuracy_reward": -0.32500001788139343, |
| "rewards/cosine_rewards": -0.003259307239204645, |
| "rewards/format_reward": 0.078125, |
| "rewards/repetition_rewards": 0.0, |
| "step": 19 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 97.109375, |
| "epoch": 0.010188487009679063, |
| "grad_norm": 24.27751022595849, |
| "kl": 0.037109375, |
| "learning_rate": 9.949057564951603e-07, |
| "loss": 0.0015, |
| "reward": -0.2632312625646591, |
| "reward_std": 0.6930468529462814, |
| "rewards/accuracy_reward": -0.4624999985098839, |
| "rewards/cosine_rewards": -0.003856247873045504, |
| "rewards/format_reward": 0.203125, |
| "rewards/repetition_rewards": 0.0, |
| "step": 20 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.03125, |
| "epoch": 0.010697911360163017, |
| "grad_norm": 12.508736780907405, |
| "kl": 0.053955078125, |
| "learning_rate": 9.946510443199185e-07, |
| "loss": 0.0022, |
| "reward": -0.010567170567810535, |
| "reward_std": 0.7874742448329926, |
| "rewards/accuracy_reward": -0.4125000238418579, |
| "rewards/cosine_rewards": -0.004317150334827602, |
| "rewards/format_reward": 0.40625, |
| "rewards/repetition_rewards": 0.0, |
| "step": 21 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.390625, |
| "epoch": 0.011207335710646969, |
| "grad_norm": 10.983519481785477, |
| "kl": 0.073974609375, |
| "learning_rate": 9.943963321446764e-07, |
| "loss": 0.003, |
| "reward": 0.5529356598854065, |
| "reward_std": 0.9540310502052307, |
| "rewards/accuracy_reward": -0.2093750163912773, |
| "rewards/cosine_rewards": -0.003314302652142942, |
| "rewards/format_reward": 0.765625, |
| "rewards/repetition_rewards": 0.0, |
| "step": 22 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.796875, |
| "epoch": 0.011716760061130923, |
| "grad_norm": 59.13650095831239, |
| "kl": 0.084716796875, |
| "learning_rate": 9.941416199694345e-07, |
| "loss": 0.0034, |
| "reward": 0.49799469113349915, |
| "reward_std": 0.6547213792800903, |
| "rewards/accuracy_reward": -0.43437501788139343, |
| "rewards/cosine_rewards": -0.005130313569679856, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 23 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 114.34375, |
| "epoch": 0.012226184411614875, |
| "grad_norm": 33.85321217925331, |
| "kl": 0.078369140625, |
| "learning_rate": 9.938869077941925e-07, |
| "loss": 0.0031, |
| "reward": 0.5440552532672882, |
| "reward_std": 0.4689805209636688, |
| "rewards/accuracy_reward": -0.43437501788139343, |
| "rewards/cosine_rewards": -0.005944762844592333, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 24 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 99.078125, |
| "epoch": 0.01273560876209883, |
| "grad_norm": 27.614415529764607, |
| "kl": 0.23876953125, |
| "learning_rate": 9.936321956189506e-07, |
| "loss": 0.0096, |
| "reward": 0.319291889667511, |
| "reward_std": 0.2991320895962417, |
| "rewards/accuracy_reward": -0.659375011920929, |
| "rewards/cosine_rewards": -0.005708091426640749, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 25 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 21.09375, |
| "epoch": 0.013245033112582781, |
| "grad_norm": 78.27860464199816, |
| "kl": 0.810546875, |
| "learning_rate": 9.933774834437085e-07, |
| "loss": 0.0324, |
| "reward": 0.758573591709137, |
| "reward_std": 0.8151377141475677, |
| "rewards/accuracy_reward": -0.24062500894069672, |
| "rewards/cosine_rewards": -0.0008013773494894849, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 26 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 18.75, |
| "epoch": 0.013754457463066735, |
| "grad_norm": 15.792726008582374, |
| "kl": 0.876953125, |
| "learning_rate": 9.931227712684667e-07, |
| "loss": 0.0351, |
| "reward": 0.5177058726549149, |
| "reward_std": 0.6054319739341736, |
| "rewards/accuracy_reward": -0.46562501788139343, |
| "rewards/cosine_rewards": -0.000942649960052222, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -0.000101461038866546, |
| "step": 27 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 15.15625, |
| "epoch": 0.014263881813550688, |
| "grad_norm": 28.538085950991302, |
| "kl": 0.853515625, |
| "learning_rate": 9.928680590932246e-07, |
| "loss": 0.0342, |
| "reward": 0.30591557919979095, |
| "reward_std": 0.3449897766113281, |
| "rewards/accuracy_reward": -0.6625000238418579, |
| "rewards/cosine_rewards": -0.00033439824983361177, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": 0.0, |
| "step": 28 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 13.1875, |
| "epoch": 0.014773306164034642, |
| "grad_norm": 19.672166251005525, |
| "kl": 0.939453125, |
| "learning_rate": 9.926133469179825e-07, |
| "loss": 0.0375, |
| "reward": 0.4091247171163559, |
| "reward_std": 0.46140581369400024, |
| "rewards/accuracy_reward": -0.5750000476837158, |
| "rewards/cosine_rewards": -0.0002502501738490537, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 29 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 16.171875, |
| "epoch": 0.015282730514518594, |
| "grad_norm": 35.96869326683099, |
| "kl": 1.416015625, |
| "learning_rate": 9.923586347427406e-07, |
| "loss": 0.0566, |
| "reward": 0.5554585456848145, |
| "reward_std": 0.7011753022670746, |
| "rewards/accuracy_reward": -0.3812499940395355, |
| "rewards/cosine_rewards": -0.0007914370798971504, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 30 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 16.15625, |
| "epoch": 0.015792154865002548, |
| "grad_norm": 24.745191247130563, |
| "kl": 1.01171875, |
| "learning_rate": 9.921039225674986e-07, |
| "loss": 0.0405, |
| "reward": 0.6306657046079636, |
| "reward_std": 0.7620185613632202, |
| "rewards/accuracy_reward": -0.32187502086162567, |
| "rewards/cosine_rewards": -0.0005842609098181129, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": 0.0, |
| "step": 31 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 39.140625, |
| "epoch": 0.0163015792154865, |
| "grad_norm": 12.709104159306316, |
| "kl": 0.7265625, |
| "learning_rate": 9.918492103922567e-07, |
| "loss": 0.0291, |
| "reward": 0.38606902956962585, |
| "reward_std": 0.8792209327220917, |
| "rewards/accuracy_reward": -0.39375001192092896, |
| "rewards/cosine_rewards": -0.001430943259038031, |
| "rewards/format_reward": 0.78125, |
| "rewards/repetition_rewards": 0.0, |
| "step": 32 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 15.03125, |
| "epoch": 0.016811003565970453, |
| "grad_norm": 17.976804747397264, |
| "kl": 0.904296875, |
| "learning_rate": 9.915944982170146e-07, |
| "loss": 0.0361, |
| "reward": 0.4996982365846634, |
| "reward_std": 0.7649624943733215, |
| "rewards/accuracy_reward": -0.4687500149011612, |
| "rewards/cosine_rewards": -0.00030174180574249476, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": 0.0, |
| "step": 33 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 22.609375, |
| "epoch": 0.017320427916454408, |
| "grad_norm": 39.58286880123024, |
| "kl": 0.8828125, |
| "learning_rate": 9.913397860417728e-07, |
| "loss": 0.0353, |
| "reward": 0.4207390695810318, |
| "reward_std": 0.8459209501743317, |
| "rewards/accuracy_reward": -0.4375000298023224, |
| "rewards/cosine_rewards": -0.0011358977280906402, |
| "rewards/format_reward": 0.859375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 34 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 13.328125, |
| "epoch": 0.01782985226693836, |
| "grad_norm": 19.17123986238676, |
| "kl": 0.955078125, |
| "learning_rate": 9.910850738665307e-07, |
| "loss": 0.0383, |
| "reward": 0.474868506193161, |
| "reward_std": 0.6449769139289856, |
| "rewards/accuracy_reward": -0.49375005066394806, |
| "rewards/cosine_rewards": -0.0001314536166319158, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": 0.0, |
| "step": 35 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 24.328125, |
| "epoch": 0.018339276617422313, |
| "grad_norm": 23.5658227799872, |
| "kl": 0.95703125, |
| "learning_rate": 9.908303616912888e-07, |
| "loss": 0.0382, |
| "reward": 0.4700201153755188, |
| "reward_std": 0.7454200983047485, |
| "rewards/accuracy_reward": -0.41875001788139343, |
| "rewards/cosine_rewards": -0.0018548529915278777, |
| "rewards/format_reward": 0.890625, |
| "rewards/repetition_rewards": 0.0, |
| "step": 36 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 13.921875, |
| "epoch": 0.018848700967906265, |
| "grad_norm": 11.872294898362206, |
| "kl": 1.001953125, |
| "learning_rate": 9.905756495160467e-07, |
| "loss": 0.0401, |
| "reward": 0.5029261708259583, |
| "reward_std": 0.7742039263248444, |
| "rewards/accuracy_reward": -0.4343750327825546, |
| "rewards/cosine_rewards": -0.0001988118929148186, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 37 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 16.40625, |
| "epoch": 0.01935812531839022, |
| "grad_norm": 14.39070050703297, |
| "kl": 0.978515625, |
| "learning_rate": 9.903209373408049e-07, |
| "loss": 0.0391, |
| "reward": 0.4616774320602417, |
| "reward_std": 0.7915183901786804, |
| "rewards/accuracy_reward": -0.4125000238418579, |
| "rewards/cosine_rewards": -0.000822544090624433, |
| "rewards/format_reward": 0.875, |
| "rewards/repetition_rewards": 0.0, |
| "step": 38 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 22.1875, |
| "epoch": 0.019867549668874173, |
| "grad_norm": 9.423514240680602, |
| "kl": 0.9375, |
| "learning_rate": 9.900662251655628e-07, |
| "loss": 0.0376, |
| "reward": 0.5430571883916855, |
| "reward_std": 0.5502887666225433, |
| "rewards/accuracy_reward": -0.4062500447034836, |
| "rewards/cosine_rewards": -0.003600762978749117, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": -0.00021701389050576836, |
| "step": 39 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 12.4375, |
| "epoch": 0.020376974019358125, |
| "grad_norm": 25.806979588800377, |
| "kl": 0.9140625, |
| "learning_rate": 9.89811512990321e-07, |
| "loss": 0.0366, |
| "reward": 0.503069132566452, |
| "reward_std": 0.6060213148593903, |
| "rewards/accuracy_reward": -0.4656250327825546, |
| "rewards/cosine_rewards": -5.582944686466362e-05, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": 0.0, |
| "step": 40 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 12.46875, |
| "epoch": 0.020886398369842078, |
| "grad_norm": 20.104239930601235, |
| "kl": 0.9296875, |
| "learning_rate": 9.895568008150789e-07, |
| "loss": 0.0372, |
| "reward": 0.631201758980751, |
| "reward_std": 0.7148115336894989, |
| "rewards/accuracy_reward": -0.35312502086162567, |
| "rewards/cosine_rewards": -4.822800292458851e-05, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 41 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 12.828125, |
| "epoch": 0.021395822720326033, |
| "grad_norm": 7.720832433302504, |
| "kl": 0.841796875, |
| "learning_rate": 9.89302088639837e-07, |
| "loss": 0.0336, |
| "reward": 0.5936954319477081, |
| "reward_std": 0.4961870163679123, |
| "rewards/accuracy_reward": -0.4062500298023224, |
| "rewards/cosine_rewards": -5.458852319861762e-05, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 42 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 12.984375, |
| "epoch": 0.021905247070809986, |
| "grad_norm": 9.831243087089065, |
| "kl": 0.76953125, |
| "learning_rate": 9.89047376464595e-07, |
| "loss": 0.0308, |
| "reward": 0.6499472558498383, |
| "reward_std": 0.7755721807479858, |
| "rewards/accuracy_reward": -0.3500000238418579, |
| "rewards/cosine_rewards": -5.273178430797998e-05, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 43 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 13.0, |
| "epoch": 0.022414671421293938, |
| "grad_norm": 15.405185965961632, |
| "kl": 0.79296875, |
| "learning_rate": 9.88792664289353e-07, |
| "loss": 0.0318, |
| "reward": 0.8749629557132721, |
| "reward_std": 0.8532125055789948, |
| "rewards/accuracy_reward": -0.1250000149011612, |
| "rewards/cosine_rewards": -3.706023017002735e-05, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 44 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 13.0, |
| "epoch": 0.02292409577177789, |
| "grad_norm": 73.99173140679622, |
| "kl": 0.814453125, |
| "learning_rate": 9.88537952114111e-07, |
| "loss": 0.0326, |
| "reward": 0.8468359708786011, |
| "reward_std": 0.6202812939882278, |
| "rewards/accuracy_reward": -0.15312501415610313, |
| "rewards/cosine_rewards": -3.904559889633674e-05, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 45 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 13.09375, |
| "epoch": 0.023433520122261846, |
| "grad_norm": 57.01467559353994, |
| "kl": 0.802734375, |
| "learning_rate": 9.882832399388691e-07, |
| "loss": 0.0321, |
| "reward": 0.7187013626098633, |
| "reward_std": 0.7317405939102173, |
| "rewards/accuracy_reward": -0.26562502793967724, |
| "rewards/cosine_rewards": -4.8641444664099254e-05, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 46 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 14.8125, |
| "epoch": 0.023942944472745798, |
| "grad_norm": 49.94239079179156, |
| "kl": 0.8125, |
| "learning_rate": 9.88028527763627e-07, |
| "loss": 0.0325, |
| "reward": 0.7904289066791534, |
| "reward_std": 0.6411640644073486, |
| "rewards/accuracy_reward": -0.2093750163912773, |
| "rewards/cosine_rewards": -0.00019609702576417476, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 47 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 13.984375, |
| "epoch": 0.02445236882322975, |
| "grad_norm": 29.421172213478044, |
| "kl": 0.8046875, |
| "learning_rate": 9.877738155883852e-07, |
| "loss": 0.0322, |
| "reward": 0.7342777252197266, |
| "reward_std": 0.3429698422551155, |
| "rewards/accuracy_reward": -0.2656249850988388, |
| "rewards/cosine_rewards": -9.731029422255233e-05, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 48 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 13.0, |
| "epoch": 0.024961793173713703, |
| "grad_norm": 30.98665699286148, |
| "kl": 0.86328125, |
| "learning_rate": 9.87519103413143e-07, |
| "loss": 0.0346, |
| "reward": 1.0437248945236206, |
| "reward_std": 0.6164620369672775, |
| "rewards/accuracy_reward": 0.04374997317790985, |
| "rewards/cosine_rewards": -2.5148013037323835e-05, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 49 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 13.0, |
| "epoch": 0.02547121752419766, |
| "grad_norm": 25.805195350433394, |
| "kl": 0.787109375, |
| "learning_rate": 9.872643912379012e-07, |
| "loss": 0.0315, |
| "reward": 0.6499470472335815, |
| "reward_std": 0.4753982424736023, |
| "rewards/accuracy_reward": -0.3500000238418579, |
| "rewards/cosine_rewards": -5.2943185437470675e-05, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 50 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 13.0, |
| "epoch": 0.02598064187468161, |
| "grad_norm": 60.37753917289171, |
| "kl": 0.865234375, |
| "learning_rate": 9.870096790626592e-07, |
| "loss": 0.0347, |
| "reward": 1.0999788641929626, |
| "reward_std": 0.716822475194931, |
| "rewards/accuracy_reward": 0.09999999590218067, |
| "rewards/cosine_rewards": -2.117727399308933e-05, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 51 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 13.0, |
| "epoch": 0.026490066225165563, |
| "grad_norm": 60.29300770886218, |
| "kl": 0.859375, |
| "learning_rate": 9.867549668874173e-07, |
| "loss": 0.0343, |
| "reward": 1.3249947428703308, |
| "reward_std": 0.6325759440660477, |
| "rewards/accuracy_reward": 0.32499997690320015, |
| "rewards/cosine_rewards": -5.294318725646008e-06, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 52 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 14.859375, |
| "epoch": 0.026999490575649515, |
| "grad_norm": 33.49731491963465, |
| "kl": 0.96484375, |
| "learning_rate": 9.865002547121752e-07, |
| "loss": 0.0386, |
| "reward": 0.6497911810874939, |
| "reward_std": 0.23335448652505875, |
| "rewards/accuracy_reward": -0.3500000163912773, |
| "rewards/cosine_rewards": -0.00020882973694824614, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 53 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 13.0, |
| "epoch": 0.02750891492613347, |
| "grad_norm": 21.106152145400298, |
| "kl": 0.85546875, |
| "learning_rate": 9.862455425369333e-07, |
| "loss": 0.0342, |
| "reward": 1.3812487125396729, |
| "reward_std": 0.26327238231897354, |
| "rewards/accuracy_reward": 0.3812499940395355, |
| "rewards/cosine_rewards": -1.3235799087851774e-06, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 54 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 13.0, |
| "epoch": 0.028018339276617423, |
| "grad_norm": 50.8468456202969, |
| "kl": 0.767578125, |
| "learning_rate": 9.859908303616913e-07, |
| "loss": 0.0307, |
| "reward": 1.493756651878357, |
| "reward_std": 0.3182205259799957, |
| "rewards/accuracy_reward": 0.4937499910593033, |
| "rewards/cosine_rewards": 6.617898179683834e-06, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 55 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 13.0, |
| "epoch": 0.028527763627101375, |
| "grad_norm": 45.22229728431362, |
| "kl": 0.833984375, |
| "learning_rate": 9.857361181864494e-07, |
| "loss": 0.0334, |
| "reward": 0.9030899405479431, |
| "reward_std": 0.2386654019355774, |
| "rewards/accuracy_reward": -0.09687501937150955, |
| "rewards/cosine_rewards": -3.507485962472856e-05, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 56 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 13.0, |
| "epoch": 0.029037187977585328, |
| "grad_norm": 251.48941881136554, |
| "kl": 0.828125, |
| "learning_rate": 9.854814060112073e-07, |
| "loss": 0.0331, |
| "reward": 1.5781376361846924, |
| "reward_std": 0.3039933070540428, |
| "rewards/accuracy_reward": 0.5781249701976776, |
| "rewards/cosine_rewards": 1.2574006632348755e-05, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 57 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 14.453125, |
| "epoch": 0.029546612328069283, |
| "grad_norm": 36.77436472817121, |
| "kl": 0.939453125, |
| "learning_rate": 9.852266938359653e-07, |
| "loss": 0.0376, |
| "reward": 1.334273636341095, |
| "reward_std": 0.34448733925819397, |
| "rewards/accuracy_reward": 0.34999997913837433, |
| "rewards/cosine_rewards": -0.00010142281280423049, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 58 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 14.4375, |
| "epoch": 0.030056036678553236, |
| "grad_norm": 46.54068675299219, |
| "kl": 0.89453125, |
| "learning_rate": 9.849719816607234e-07, |
| "loss": 0.0358, |
| "reward": 0.9967500269412994, |
| "reward_std": 0.4488208740949631, |
| "rewards/accuracy_reward": 0.012499993667006493, |
| "rewards/cosine_rewards": -0.00012501747096393956, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 59 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 13.0, |
| "epoch": 0.030565461029037188, |
| "grad_norm": 63.09725081890949, |
| "kl": 0.837890625, |
| "learning_rate": 9.847172694854813e-07, |
| "loss": 0.0335, |
| "reward": 0.9874708652496338, |
| "reward_std": 0.33707569539546967, |
| "rewards/accuracy_reward": -0.012500002980232239, |
| "rewards/cosine_rewards": -2.9118752991053043e-05, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 60 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 13.0, |
| "epoch": 0.03107488537952114, |
| "grad_norm": 105.45023488529014, |
| "kl": 0.8359375, |
| "learning_rate": 9.844625573102394e-07, |
| "loss": 0.0334, |
| "reward": 1.1281058490276337, |
| "reward_std": 0.3039932996034622, |
| "rewards/accuracy_reward": 0.12812498584389687, |
| "rewards/cosine_rewards": -1.919190435728524e-05, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 61 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 14.640625, |
| "epoch": 0.031584309730005096, |
| "grad_norm": 85.12929156788996, |
| "kl": 0.84375, |
| "learning_rate": 9.842078451349974e-07, |
| "loss": 0.0337, |
| "reward": 0.7748350501060486, |
| "reward_std": 0.5448895841836929, |
| "rewards/accuracy_reward": -0.2093750238418579, |
| "rewards/cosine_rewards": -0.00016493651855853386, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 62 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 19.078125, |
| "epoch": 0.032093734080489045, |
| "grad_norm": 8.426167428667783, |
| "kl": 0.814453125, |
| "learning_rate": 9.839531329597555e-07, |
| "loss": 0.0326, |
| "reward": 0.874523401260376, |
| "reward_std": 0.0010828198865056038, |
| "rewards/accuracy_reward": -0.1250000149011612, |
| "rewards/cosine_rewards": -0.0004766158472193638, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 63 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 19.34375, |
| "epoch": 0.032603158430973, |
| "grad_norm": 86.47647424288853, |
| "kl": 0.810546875, |
| "learning_rate": 9.836984207845134e-07, |
| "loss": 0.0324, |
| "reward": 1.6625866889953613, |
| "reward_std": 0.19662056118249893, |
| "rewards/accuracy_reward": 0.6625000238418579, |
| "rewards/cosine_rewards": 8.658922160975635e-05, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 64 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 17.6875, |
| "epoch": 0.033112582781456956, |
| "grad_norm": 59.22678939682069, |
| "kl": 0.86328125, |
| "learning_rate": 9.834437086092716e-07, |
| "loss": 0.0345, |
| "reward": 0.915763258934021, |
| "reward_std": 0.082692209049128, |
| "rewards/accuracy_reward": -0.06875000894069672, |
| "rewards/cosine_rewards": 0.00013823993504047394, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 65 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 24.671875, |
| "epoch": 0.033622007131940905, |
| "grad_norm": 170.96252285475958, |
| "kl": 0.7734375, |
| "learning_rate": 9.831889964340295e-07, |
| "loss": 0.0309, |
| "reward": 1.2965829372406006, |
| "reward_std": 0.32569222897291183, |
| "rewards/accuracy_reward": 0.2968750074505806, |
| "rewards/cosine_rewards": -0.0002921203849837184, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 66 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 18.890625, |
| "epoch": 0.03413143148242486, |
| "grad_norm": 302.02752287161115, |
| "kl": 0.84765625, |
| "learning_rate": 9.829342842587876e-07, |
| "loss": 0.0339, |
| "reward": 1.2968038320541382, |
| "reward_std": 0.27610647678375244, |
| "rewards/accuracy_reward": 0.296875, |
| "rewards/cosine_rewards": -7.123823161236942e-05, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 67 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 22.984375, |
| "epoch": 0.034640855832908816, |
| "grad_norm": 617.6786375620823, |
| "kl": 0.77734375, |
| "learning_rate": 9.826795720835456e-07, |
| "loss": 0.0311, |
| "reward": 1.4656760096549988, |
| "reward_std": 0.2886117473244667, |
| "rewards/accuracy_reward": 0.46562498807907104, |
| "rewards/cosine_rewards": 5.1008202717639506e-05, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 68 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 24.34375, |
| "epoch": 0.035150280183392765, |
| "grad_norm": 29.79960642054238, |
| "kl": 0.728515625, |
| "learning_rate": 9.824248599083037e-07, |
| "loss": 0.0292, |
| "reward": 1.309334635734558, |
| "reward_std": 0.20424916595220566, |
| "rewards/accuracy_reward": 0.32500000298023224, |
| "rewards/cosine_rewards": -4.041045031044632e-05, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 69 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 23.90625, |
| "epoch": 0.03565970453387672, |
| "grad_norm": 91.00871807242451, |
| "kl": 0.744140625, |
| "learning_rate": 9.821701477330616e-07, |
| "loss": 0.0298, |
| "reward": 1.2686043679714203, |
| "reward_std": 0.10558865318307653, |
| "rewards/accuracy_reward": 0.26874998211860657, |
| "rewards/cosine_rewards": -0.00014566810568794608, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 70 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 36.328125, |
| "epoch": 0.03616912888436067, |
| "grad_norm": 159.05100875041754, |
| "kl": 0.765625, |
| "learning_rate": 9.819154355578195e-07, |
| "loss": 0.0306, |
| "reward": 1.2812767028808594, |
| "reward_std": 0.6231541335582733, |
| "rewards/accuracy_reward": 0.296875, |
| "rewards/cosine_rewards": 2.6669338694773614e-05, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 71 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 38.4375, |
| "epoch": 0.036678553234844626, |
| "grad_norm": 97.83490373613579, |
| "kl": 0.666015625, |
| "learning_rate": 9.816607233825777e-07, |
| "loss": 0.0266, |
| "reward": 1.647216558456421, |
| "reward_std": 0.32463081181049347, |
| "rewards/accuracy_reward": 0.6625000089406967, |
| "rewards/cosine_rewards": 0.00034145097015425563, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 72 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 40.046875, |
| "epoch": 0.03718797758532858, |
| "grad_norm": 88.5872574021467, |
| "kl": 0.630859375, |
| "learning_rate": 9.814060112073356e-07, |
| "loss": 0.0253, |
| "reward": 1.7599374055862427, |
| "reward_std": 0.3454015702009201, |
| "rewards/accuracy_reward": 0.7750000059604645, |
| "rewards/cosine_rewards": 0.0005623315373668447, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 73 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 60.140625, |
| "epoch": 0.03769740193581253, |
| "grad_norm": 16.49274288281998, |
| "kl": 0.501953125, |
| "learning_rate": 9.811512990320937e-07, |
| "loss": 0.0201, |
| "reward": 1.7728378772735596, |
| "reward_std": 0.2738931328058243, |
| "rewards/accuracy_reward": 0.8031250238418579, |
| "rewards/cosine_rewards": 0.0009629083215259016, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": 0.0, |
| "step": 74 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 100.9375, |
| "epoch": 0.038206826286296486, |
| "grad_norm": 18.283090697254373, |
| "kl": 0.18359375, |
| "learning_rate": 9.808965868568517e-07, |
| "loss": 0.0074, |
| "reward": 1.437682330608368, |
| "reward_std": 0.19817885756492615, |
| "rewards/accuracy_reward": 0.4375000149011612, |
| "rewards/cosine_rewards": 0.00018233060836791992, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 75 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.34375, |
| "epoch": 0.03871625063678044, |
| "grad_norm": 21.656183371701722, |
| "kl": 0.13330078125, |
| "learning_rate": 9.806418746816098e-07, |
| "loss": 0.0054, |
| "reward": 1.2524056434631348, |
| "reward_std": 0.14976192265748978, |
| "rewards/accuracy_reward": 0.26874999701976776, |
| "rewards/cosine_rewards": -0.0007193188357632607, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 76 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.5, |
| "epoch": 0.03922567498726439, |
| "grad_norm": 12.450187143986858, |
| "kl": 0.1328125, |
| "learning_rate": 9.803871625063677e-07, |
| "loss": 0.0053, |
| "reward": 1.535181999206543, |
| "reward_std": 0.04510992762516253, |
| "rewards/accuracy_reward": 0.5499999970197678, |
| "rewards/cosine_rewards": 0.0008070359472185373, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 77 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.96875, |
| "epoch": 0.039735099337748346, |
| "grad_norm": 96.22924405795379, |
| "kl": 0.12744140625, |
| "learning_rate": 9.801324503311258e-07, |
| "loss": 0.0051, |
| "reward": 1.4221445322036743, |
| "reward_std": 0.5387175530195236, |
| "rewards/accuracy_reward": 0.4374999850988388, |
| "rewards/cosine_rewards": 0.0002695363436941989, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 78 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 117.46875, |
| "epoch": 0.040244523688232295, |
| "grad_norm": 22.356744283314942, |
| "kl": 0.12451171875, |
| "learning_rate": 9.798777381558838e-07, |
| "loss": 0.005, |
| "reward": 0.9280500411987305, |
| "reward_std": 0.3032594621181488, |
| "rewards/accuracy_reward": -0.06875001452863216, |
| "rewards/cosine_rewards": -0.003199932281859219, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 79 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.09375, |
| "epoch": 0.04075394803871625, |
| "grad_norm": 11.587196398677966, |
| "kl": 0.12353515625, |
| "learning_rate": 9.79623025980642e-07, |
| "loss": 0.0049, |
| "reward": 1.0698014497756958, |
| "reward_std": 0.306557297706604, |
| "rewards/accuracy_reward": 0.07187498360872269, |
| "rewards/cosine_rewards": -0.0020735373545903713, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 80 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 119.359375, |
| "epoch": 0.041263372389200206, |
| "grad_norm": 15.552459183086345, |
| "kl": 0.115234375, |
| "learning_rate": 9.793683138053998e-07, |
| "loss": 0.0046, |
| "reward": 1.902881920337677, |
| "reward_std": 0.2866080105304718, |
| "rewards/accuracy_reward": 0.9156250059604645, |
| "rewards/cosine_rewards": 0.0028820185689255595, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 81 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 121.28125, |
| "epoch": 0.041772796739684155, |
| "grad_norm": 21.56424441435487, |
| "kl": 0.110107421875, |
| "learning_rate": 9.79113601630158e-07, |
| "loss": 0.0044, |
| "reward": 1.2677271366119385, |
| "reward_std": 0.10610348492627963, |
| "rewards/accuracy_reward": 0.26874999701976776, |
| "rewards/cosine_rewards": -0.0010228125611320138, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 82 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 115.484375, |
| "epoch": 0.04228222109016811, |
| "grad_norm": 11.018514876954287, |
| "kl": 0.125244140625, |
| "learning_rate": 9.788588894549159e-07, |
| "loss": 0.005, |
| "reward": 1.2675296068191528, |
| "reward_std": 0.16161296842619777, |
| "rewards/accuracy_reward": 0.26874999701976776, |
| "rewards/cosine_rewards": -0.0012203185469843447, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 83 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 119.078125, |
| "epoch": 0.04279164544065207, |
| "grad_norm": 20.603407343348504, |
| "kl": 0.1083984375, |
| "learning_rate": 9.78604177279674e-07, |
| "loss": 0.0043, |
| "reward": 1.1548139452934265, |
| "reward_std": 0.537171483039856, |
| "rewards/accuracy_reward": 0.1562499925494194, |
| "rewards/cosine_rewards": -0.0014360386412590742, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 84 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 122.640625, |
| "epoch": 0.043301069791136015, |
| "grad_norm": 60.83530697951784, |
| "kl": 0.18359375, |
| "learning_rate": 9.78349465104432e-07, |
| "loss": 0.0073, |
| "reward": 1.5197246074676514, |
| "reward_std": 0.5150813460350037, |
| "rewards/accuracy_reward": 0.518750011920929, |
| "rewards/cosine_rewards": 0.000974582158960402, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 85 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 121.015625, |
| "epoch": 0.04381049414161997, |
| "grad_norm": 13.721540678774238, |
| "kl": 0.12451171875, |
| "learning_rate": 9.780947529291899e-07, |
| "loss": 0.005, |
| "reward": 1.1832407712936401, |
| "reward_std": 0.18641822785139084, |
| "rewards/accuracy_reward": 0.18437500298023224, |
| "rewards/cosine_rewards": -0.001134182559326291, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 86 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 119.578125, |
| "epoch": 0.04431991849210392, |
| "grad_norm": 215.40977191184217, |
| "kl": 0.115478515625, |
| "learning_rate": 9.77840040753948e-07, |
| "loss": 0.0046, |
| "reward": 1.2237018644809723, |
| "reward_std": 0.23010382801294327, |
| "rewards/accuracy_reward": 0.24062499776482582, |
| "rewards/cosine_rewards": -0.0012981001054868102, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 87 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 127.125, |
| "epoch": 0.044829342842587876, |
| "grad_norm": 11.570945117677702, |
| "kl": 0.110595703125, |
| "learning_rate": 9.77585328578706e-07, |
| "loss": 0.0044, |
| "reward": 1.5510605573654175, |
| "reward_std": 0.0015471973456442356, |
| "rewards/accuracy_reward": 0.550000011920929, |
| "rewards/cosine_rewards": 0.001060541602782905, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 88 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 129.953125, |
| "epoch": 0.04533876719307183, |
| "grad_norm": 9.451343704964001, |
| "kl": 0.104248046875, |
| "learning_rate": 9.77330616403464e-07, |
| "loss": 0.0042, |
| "reward": 1.5073344111442566, |
| "reward_std": 0.35981758683919907, |
| "rewards/accuracy_reward": 0.5218750052154064, |
| "rewards/cosine_rewards": 0.0010844313073903322, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 89 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 129.71875, |
| "epoch": 0.04584819154355578, |
| "grad_norm": 16.032910799390205, |
| "kl": 0.094970703125, |
| "learning_rate": 9.77075904228222e-07, |
| "loss": 0.0038, |
| "reward": 1.919905662536621, |
| "reward_std": 0.24129686888772994, |
| "rewards/accuracy_reward": 0.9156250059604645, |
| "rewards/cosine_rewards": 0.0042806623969227076, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 90 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 130.390625, |
| "epoch": 0.046357615894039736, |
| "grad_norm": 21.086614900693085, |
| "kl": 0.101806640625, |
| "learning_rate": 9.768211920529801e-07, |
| "loss": 0.0041, |
| "reward": 1.5919697284698486, |
| "reward_std": 0.2428576573729515, |
| "rewards/accuracy_reward": 0.6062500029802322, |
| "rewards/cosine_rewards": 0.0013447333476506174, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 91 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 142.734375, |
| "epoch": 0.04686704024452369, |
| "grad_norm": 8.093804874468816, |
| "kl": 0.095458984375, |
| "learning_rate": 9.76566479877738e-07, |
| "loss": 0.0038, |
| "reward": 1.6935226917266846, |
| "reward_std": 0.1869470328092575, |
| "rewards/accuracy_reward": 0.690625011920929, |
| "rewards/cosine_rewards": 0.0028977063193451613, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 92 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 143.203125, |
| "epoch": 0.04737646459500764, |
| "grad_norm": 10.624707519768712, |
| "kl": 0.099609375, |
| "learning_rate": 9.763117677024962e-07, |
| "loss": 0.004, |
| "reward": 1.4030739068984985, |
| "reward_std": 0.3680836334824562, |
| "rewards/accuracy_reward": 0.43437500298023224, |
| "rewards/cosine_rewards": -5.1158247515559196e-05, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": 0.0, |
| "step": 93 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 150.90625, |
| "epoch": 0.047885888945491596, |
| "grad_norm": 25.156466743552734, |
| "kl": 0.101318359375, |
| "learning_rate": 9.760570555272541e-07, |
| "loss": 0.0041, |
| "reward": 1.5800000429153442, |
| "reward_std": 0.5084549486637115, |
| "rewards/accuracy_reward": 0.578125, |
| "rewards/cosine_rewards": 0.0018750545859802514, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 94 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 148.46875, |
| "epoch": 0.048395313295975545, |
| "grad_norm": 12.684719084813777, |
| "kl": 0.10205078125, |
| "learning_rate": 9.758023433520122e-07, |
| "loss": 0.0041, |
| "reward": 1.5234779119491577, |
| "reward_std": 0.18682076036930084, |
| "rewards/accuracy_reward": 0.5218749940395355, |
| "rewards/cosine_rewards": 0.0016028713434934616, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 95 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 161.109375, |
| "epoch": 0.0489047376464595, |
| "grad_norm": 13.854458043447748, |
| "kl": 0.108154296875, |
| "learning_rate": 9.755476311767702e-07, |
| "loss": 0.0043, |
| "reward": 1.6655999422073364, |
| "reward_std": 0.4286635220050812, |
| "rewards/accuracy_reward": 0.6624999791383743, |
| "rewards/cosine_rewards": 0.0030998505535535514, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 96 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 167.609375, |
| "epoch": 0.049414161996943456, |
| "grad_norm": 15.632405914543359, |
| "kl": 0.098388671875, |
| "learning_rate": 9.752929190015283e-07, |
| "loss": 0.0039, |
| "reward": 1.1535860896110535, |
| "reward_std": 0.36188751459121704, |
| "rewards/accuracy_reward": 0.1562499888241291, |
| "rewards/cosine_rewards": -0.002663849270902574, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 97 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 168.453125, |
| "epoch": 0.049923586347427405, |
| "grad_norm": 8.747829969093605, |
| "kl": 0.112060546875, |
| "learning_rate": 9.750382068262862e-07, |
| "loss": 0.0045, |
| "reward": 1.3531205654144287, |
| "reward_std": 0.18947682529687881, |
| "rewards/accuracy_reward": 0.3531249985098839, |
| "rewards/cosine_rewards": -4.528439603745937e-06, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 98 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 168.71875, |
| "epoch": 0.05043301069791136, |
| "grad_norm": 12.880266278774922, |
| "kl": 0.112060546875, |
| "learning_rate": 9.747834946510442e-07, |
| "loss": 0.0045, |
| "reward": 1.619386613368988, |
| "reward_std": 0.5798123776912689, |
| "rewards/accuracy_reward": 0.6624999940395355, |
| "rewards/cosine_rewards": 0.0037616335321217775, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": 0.0, |
| "step": 99 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 173.859375, |
| "epoch": 0.05094243504839532, |
| "grad_norm": 18.637994264229288, |
| "kl": 0.109619140625, |
| "learning_rate": 9.745287824758023e-07, |
| "loss": 0.0044, |
| "reward": 1.448248565196991, |
| "reward_std": 0.4085986465215683, |
| "rewards/accuracy_reward": 0.4624999687075615, |
| "rewards/cosine_rewards": 0.0013735336251556873, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 100 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 175.375, |
| "epoch": 0.051451859398879266, |
| "grad_norm": 26.647397602965935, |
| "kl": 0.110107421875, |
| "learning_rate": 9.742740703005602e-07, |
| "loss": 0.0044, |
| "reward": 1.0668614506721497, |
| "reward_std": 0.35026729106903076, |
| "rewards/accuracy_reward": 0.07187499292194843, |
| "rewards/cosine_rewards": -0.004894306650385261, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": -0.00011927480954909697, |
| "step": 101 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 174.0, |
| "epoch": 0.05196128374936322, |
| "grad_norm": 12.612763516820905, |
| "kl": 0.112060546875, |
| "learning_rate": 9.740193581253183e-07, |
| "loss": 0.0045, |
| "reward": 1.4354371428489685, |
| "reward_std": 0.20804932340979576, |
| "rewards/accuracy_reward": 0.46562499552965164, |
| "rewards/cosine_rewards": 0.0010621265973895788, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": 0.0, |
| "step": 102 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 176.984375, |
| "epoch": 0.05247070809984717, |
| "grad_norm": 19.00540852037756, |
| "kl": 0.116455078125, |
| "learning_rate": 9.737646459500763e-07, |
| "loss": 0.0047, |
| "reward": 1.0997494161128998, |
| "reward_std": 0.5674505531787872, |
| "rewards/accuracy_reward": 0.1499999761581421, |
| "rewards/cosine_rewards": -0.003250634763389826, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": -0.0001250000059371814, |
| "step": 103 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 187.3125, |
| "epoch": 0.052980132450331126, |
| "grad_norm": 8.993773014446798, |
| "kl": 0.115478515625, |
| "learning_rate": 9.735099337748344e-07, |
| "loss": 0.0046, |
| "reward": 1.547185480594635, |
| "reward_std": 0.5772347450256348, |
| "rewards/accuracy_reward": 0.5750000029802322, |
| "rewards/cosine_rewards": 0.0034354651579633355, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": 0.0, |
| "step": 104 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 182.09375, |
| "epoch": 0.05348955680081508, |
| "grad_norm": 21.83379704072219, |
| "kl": 0.11279296875, |
| "learning_rate": 9.732552215995923e-07, |
| "loss": 0.0045, |
| "reward": 0.9665651321411133, |
| "reward_std": 0.19240357726812363, |
| "rewards/accuracy_reward": -0.012500010430812836, |
| "rewards/cosine_rewards": -0.005309856729581952, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 105 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 184.296875, |
| "epoch": 0.05399898115129903, |
| "grad_norm": 15.74936299058057, |
| "kl": 0.1240234375, |
| "learning_rate": 9.730005094243505e-07, |
| "loss": 0.005, |
| "reward": 0.8526512682437897, |
| "reward_std": 0.45641621947288513, |
| "rewards/accuracy_reward": -0.1250000149011612, |
| "rewards/cosine_rewards": -0.006723731989040971, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 106 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 178.203125, |
| "epoch": 0.054508405501782986, |
| "grad_norm": 7.621263779056561, |
| "kl": 0.116455078125, |
| "learning_rate": 9.727457972491084e-07, |
| "loss": 0.0047, |
| "reward": 1.4213617444038391, |
| "reward_std": 0.42073580622673035, |
| "rewards/accuracy_reward": 0.4375, |
| "rewards/cosine_rewards": -0.0005132523947395384, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 107 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 180.234375, |
| "epoch": 0.05501782985226694, |
| "grad_norm": 14.098307573524853, |
| "kl": 0.119140625, |
| "learning_rate": 9.724910850738665e-07, |
| "loss": 0.0048, |
| "reward": 1.1232723593711853, |
| "reward_std": 0.45567604154348373, |
| "rewards/accuracy_reward": 0.12812499329447746, |
| "rewards/cosine_rewards": -0.004852580255828798, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 108 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 183.328125, |
| "epoch": 0.05552725420275089, |
| "grad_norm": 10.937546247684887, |
| "kl": 0.18994140625, |
| "learning_rate": 9.722363728986245e-07, |
| "loss": 0.0076, |
| "reward": 1.8381596803665161, |
| "reward_std": 0.28626738488674164, |
| "rewards/accuracy_reward": 0.831250011920929, |
| "rewards/cosine_rewards": 0.006909639807417989, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 109 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 174.375, |
| "epoch": 0.056036678553234846, |
| "grad_norm": 16.445027328915916, |
| "kl": 0.11181640625, |
| "learning_rate": 9.719816607233826e-07, |
| "loss": 0.0045, |
| "reward": 1.2096136808395386, |
| "reward_std": 0.36066293716430664, |
| "rewards/accuracy_reward": 0.21249999105930328, |
| "rewards/cosine_rewards": -0.002886334084905684, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 110 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 168.765625, |
| "epoch": 0.056546102903718795, |
| "grad_norm": 18.874204475299454, |
| "kl": 0.106689453125, |
| "learning_rate": 9.717269485481405e-07, |
| "loss": 0.0043, |
| "reward": 1.3519207835197449, |
| "reward_std": 0.08376272046007216, |
| "rewards/accuracy_reward": 0.3531250059604645, |
| "rewards/cosine_rewards": -0.0012042350135743618, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 111 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 157.796875, |
| "epoch": 0.05705552725420275, |
| "grad_norm": 9.104959106458736, |
| "kl": 0.121337890625, |
| "learning_rate": 9.714722363728986e-07, |
| "loss": 0.0049, |
| "reward": 1.381228744983673, |
| "reward_std": 0.16323383897542953, |
| "rewards/accuracy_reward": 0.3812500238418579, |
| "rewards/cosine_rewards": -2.1282234229147434e-05, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 112 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 162.171875, |
| "epoch": 0.05756495160468671, |
| "grad_norm": 13.28611466805594, |
| "kl": 0.10888671875, |
| "learning_rate": 9.712175241976566e-07, |
| "loss": 0.0044, |
| "reward": 1.3366525173187256, |
| "reward_std": 0.28694501193240285, |
| "rewards/accuracy_reward": 0.3531249985098839, |
| "rewards/cosine_rewards": -0.0008475282229483128, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 113 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 159.4375, |
| "epoch": 0.058074375955170655, |
| "grad_norm": 20.0350592953945, |
| "kl": 0.107666015625, |
| "learning_rate": 9.709628120224145e-07, |
| "loss": 0.0043, |
| "reward": 1.4107850790023804, |
| "reward_std": 0.18804995715618134, |
| "rewards/accuracy_reward": 0.40937501937150955, |
| "rewards/cosine_rewards": 0.001410042867064476, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 114 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 159.6875, |
| "epoch": 0.05858380030565461, |
| "grad_norm": 9.526099983425397, |
| "kl": 0.10595703125, |
| "learning_rate": 9.707080998471726e-07, |
| "loss": 0.0042, |
| "reward": 1.4221826791763306, |
| "reward_std": 0.2918977811932564, |
| "rewards/accuracy_reward": 0.4374999925494194, |
| "rewards/cosine_rewards": 0.00030758429784327745, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 115 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 157.984375, |
| "epoch": 0.05909322465613857, |
| "grad_norm": 14.046929728525855, |
| "kl": 0.11572265625, |
| "learning_rate": 9.704533876719306e-07, |
| "loss": 0.0046, |
| "reward": 1.2389479279518127, |
| "reward_std": 0.4534989148378372, |
| "rewards/accuracy_reward": 0.24062498658895493, |
| "rewards/cosine_rewards": -0.00167706364300102, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 116 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 167.390625, |
| "epoch": 0.059602649006622516, |
| "grad_norm": 12.733865198375515, |
| "kl": 0.105224609375, |
| "learning_rate": 9.701986754966887e-07, |
| "loss": 0.0042, |
| "reward": 1.0669120252132416, |
| "reward_std": 0.319850392639637, |
| "rewards/accuracy_reward": 0.07187498360872269, |
| "rewards/cosine_rewards": -0.004963014740496874, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 117 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 156.140625, |
| "epoch": 0.06011207335710647, |
| "grad_norm": 11.670513012812588, |
| "kl": 0.093505859375, |
| "learning_rate": 9.699439633214466e-07, |
| "loss": 0.0038, |
| "reward": 1.665140986442566, |
| "reward_std": 0.12403370253741741, |
| "rewards/accuracy_reward": 0.6624999940395355, |
| "rewards/cosine_rewards": 0.0026409668498672545, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 118 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 155.828125, |
| "epoch": 0.06062149770759042, |
| "grad_norm": 28.993909689663674, |
| "kl": 0.1015625, |
| "learning_rate": 9.696892511462047e-07, |
| "loss": 0.0041, |
| "reward": 1.0673952102661133, |
| "reward_std": 0.30907338857650757, |
| "rewards/accuracy_reward": 0.07187499105930328, |
| "rewards/cosine_rewards": -0.004479756113141775, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 119 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 154.40625, |
| "epoch": 0.061130922058074376, |
| "grad_norm": 27.685530130702478, |
| "kl": 0.104736328125, |
| "learning_rate": 9.694345389709627e-07, |
| "loss": 0.0042, |
| "reward": 1.4373126029968262, |
| "reward_std": 0.21315501490607858, |
| "rewards/accuracy_reward": 0.4375, |
| "rewards/cosine_rewards": -0.0001873411238193512, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 120 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 152.0625, |
| "epoch": 0.06164034640855833, |
| "grad_norm": 12.6569610895899, |
| "kl": 0.121826171875, |
| "learning_rate": 9.691798267957208e-07, |
| "loss": 0.0049, |
| "reward": 1.3240773677825928, |
| "reward_std": 0.26775629818439484, |
| "rewards/accuracy_reward": 0.32499999552965164, |
| "rewards/cosine_rewards": -0.0009226472466252744, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 121 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 151.78125, |
| "epoch": 0.06214977075904228, |
| "grad_norm": 11.557058687556623, |
| "kl": 0.103759765625, |
| "learning_rate": 9.689251146204787e-07, |
| "loss": 0.0041, |
| "reward": 1.7503631114959717, |
| "reward_std": 0.08287379238754511, |
| "rewards/accuracy_reward": 0.7468750178813934, |
| "rewards/cosine_rewards": 0.0034881452447734773, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 122 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 158.578125, |
| "epoch": 0.06265919510952624, |
| "grad_norm": 19.676894670897823, |
| "kl": 0.1025390625, |
| "learning_rate": 9.686704024452369e-07, |
| "loss": 0.0041, |
| "reward": 1.3520426154136658, |
| "reward_std": 0.24371477961540222, |
| "rewards/accuracy_reward": 0.3531249761581421, |
| "rewards/cosine_rewards": -0.0010823981137946248, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 123 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 167.484375, |
| "epoch": 0.06316861946001019, |
| "grad_norm": 7.7934253879172894, |
| "kl": 0.10107421875, |
| "learning_rate": 9.684156902699948e-07, |
| "loss": 0.004, |
| "reward": 1.4387494623661041, |
| "reward_std": 0.26880691200494766, |
| "rewards/accuracy_reward": 0.4374999888241291, |
| "rewards/cosine_rewards": 0.0015281732194125652, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": -0.000278731546131894, |
| "step": 124 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 160.671875, |
| "epoch": 0.06367804381049415, |
| "grad_norm": 14.376740722468174, |
| "kl": 0.1064453125, |
| "learning_rate": 9.68160978094753e-07, |
| "loss": 0.0043, |
| "reward": 1.2107464671134949, |
| "reward_std": 0.20096861571073532, |
| "rewards/accuracy_reward": 0.21249999292194843, |
| "rewards/cosine_rewards": -0.0017535560764372349, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 125 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 158.25, |
| "epoch": 0.06418746816097809, |
| "grad_norm": 24.20355073697258, |
| "kl": 0.10693359375, |
| "learning_rate": 9.679062659195109e-07, |
| "loss": 0.0043, |
| "reward": 1.09614896774292, |
| "reward_std": 0.4781967103481293, |
| "rewards/accuracy_reward": 0.09999998658895493, |
| "rewards/cosine_rewards": -0.0038510175654664636, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 126 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 165.203125, |
| "epoch": 0.06469689251146205, |
| "grad_norm": 8.56510891629326, |
| "kl": 0.111083984375, |
| "learning_rate": 9.676515537442688e-07, |
| "loss": 0.0044, |
| "reward": 1.5521512031555176, |
| "reward_std": 0.46633191406726837, |
| "rewards/accuracy_reward": 0.550000011920929, |
| "rewards/cosine_rewards": 0.002151212247554213, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 127 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 164.421875, |
| "epoch": 0.065206316861946, |
| "grad_norm": 13.379306876395301, |
| "kl": 0.120849609375, |
| "learning_rate": 9.67396841569027e-07, |
| "loss": 0.0048, |
| "reward": 1.722820222377777, |
| "reward_std": 0.32213538885116577, |
| "rewards/accuracy_reward": 0.71875, |
| "rewards/cosine_rewards": 0.004070190014317632, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 128 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 173.703125, |
| "epoch": 0.06571574121242996, |
| "grad_norm": 153.40445361752842, |
| "kl": 0.1162109375, |
| "learning_rate": 9.67142129393785e-07, |
| "loss": 0.0047, |
| "reward": 1.2643532752990723, |
| "reward_std": 0.23417328391224146, |
| "rewards/accuracy_reward": 0.2656250037252903, |
| "rewards/cosine_rewards": -0.0012717264471575618, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 129 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 168.5625, |
| "epoch": 0.06622516556291391, |
| "grad_norm": 9.275290295194138, |
| "kl": 0.10302734375, |
| "learning_rate": 9.66887417218543e-07, |
| "loss": 0.0041, |
| "reward": 1.0960015654563904, |
| "reward_std": 0.21426187455654144, |
| "rewards/accuracy_reward": 0.09999999403953552, |
| "rewards/cosine_rewards": -0.003998432832304388, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 130 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 176.265625, |
| "epoch": 0.06673458991339785, |
| "grad_norm": 19.81840956306445, |
| "kl": 0.105224609375, |
| "learning_rate": 9.66632705043301e-07, |
| "loss": 0.0042, |
| "reward": 1.6365814805030823, |
| "reward_std": 0.20723329484462738, |
| "rewards/accuracy_reward": 0.6343750059604645, |
| "rewards/cosine_rewards": 0.0022064344957470894, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 131 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 174.234375, |
| "epoch": 0.06724401426388181, |
| "grad_norm": 8.864924483945437, |
| "kl": 0.108642578125, |
| "learning_rate": 9.66377992868059e-07, |
| "loss": 0.0044, |
| "reward": 1.3077268600463867, |
| "reward_std": 0.3278057724237442, |
| "rewards/accuracy_reward": 0.32500000670552254, |
| "rewards/cosine_rewards": -0.0012397709069773555, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -0.00040839536814019084, |
| "step": 132 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 175.6875, |
| "epoch": 0.06775343861436577, |
| "grad_norm": 14.31694516925494, |
| "kl": 0.112060546875, |
| "learning_rate": 9.661232806928172e-07, |
| "loss": 0.0045, |
| "reward": 1.3927981853485107, |
| "reward_std": 0.3101032227277756, |
| "rewards/accuracy_reward": 0.40937499701976776, |
| "rewards/cosine_rewards": -0.0007292817026609555, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -0.00022258506942307577, |
| "step": 133 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 174.515625, |
| "epoch": 0.06826286296484972, |
| "grad_norm": 13.206286933876525, |
| "kl": 0.110107421875, |
| "learning_rate": 9.65868568517575e-07, |
| "loss": 0.0044, |
| "reward": 1.4952284097671509, |
| "reward_std": 0.16513758851215243, |
| "rewards/accuracy_reward": 0.4937499910593033, |
| "rewards/cosine_rewards": 0.0014784452505409718, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 134 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 188.15625, |
| "epoch": 0.06877228731533368, |
| "grad_norm": 8.196723017225267, |
| "kl": 0.110107421875, |
| "learning_rate": 9.656138563423332e-07, |
| "loss": 0.0044, |
| "reward": 1.3523318767547607, |
| "reward_std": 0.19119784235954285, |
| "rewards/accuracy_reward": 0.3531249985098839, |
| "rewards/cosine_rewards": -0.0007931197178550065, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 135 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 184.375, |
| "epoch": 0.06928171166581763, |
| "grad_norm": 7.96588213615428, |
| "kl": 0.10400390625, |
| "learning_rate": 9.653591441670911e-07, |
| "loss": 0.0042, |
| "reward": 1.3809208273887634, |
| "reward_std": 0.16492938250303268, |
| "rewards/accuracy_reward": 0.3812500238418579, |
| "rewards/cosine_rewards": -0.00032906350679695606, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 136 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 179.78125, |
| "epoch": 0.06979113601630157, |
| "grad_norm": 15.45343619628501, |
| "kl": 0.1142578125, |
| "learning_rate": 9.651044319918493e-07, |
| "loss": 0.0046, |
| "reward": 1.5364066362380981, |
| "reward_std": 0.3309681713581085, |
| "rewards/accuracy_reward": 0.550000011920929, |
| "rewards/cosine_rewards": 0.002031611278653145, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 137 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 191.484375, |
| "epoch": 0.07030056036678553, |
| "grad_norm": 10.658123621710919, |
| "kl": 0.1142578125, |
| "learning_rate": 9.648497198166072e-07, |
| "loss": 0.0046, |
| "reward": 1.5228744149208069, |
| "reward_std": 0.08533496968448162, |
| "rewards/accuracy_reward": 0.5218750089406967, |
| "rewards/cosine_rewards": 0.0011002181563526392, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": -0.00010080645006382838, |
| "step": 138 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 205.734375, |
| "epoch": 0.07080998471726949, |
| "grad_norm": 13.847520067525743, |
| "kl": 0.118408203125, |
| "learning_rate": 9.645950076413653e-07, |
| "loss": 0.0047, |
| "reward": 0.6919489502906799, |
| "reward_std": 0.29361478984355927, |
| "rewards/accuracy_reward": -0.29375001788139343, |
| "rewards/cosine_rewards": -0.014301038347184658, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 139 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 187.6875, |
| "epoch": 0.07131940906775344, |
| "grad_norm": 32.08146525993247, |
| "kl": 0.115234375, |
| "learning_rate": 9.643402954661233e-07, |
| "loss": 0.0046, |
| "reward": 1.3814507126808167, |
| "reward_std": 0.10938079445622861, |
| "rewards/accuracy_reward": 0.3812499940395355, |
| "rewards/cosine_rewards": 0.00031310925260186195, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": -0.00011241007450735196, |
| "step": 140 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 201.890625, |
| "epoch": 0.0718288334182374, |
| "grad_norm": 13.580365765026054, |
| "kl": 0.12158203125, |
| "learning_rate": 9.640855832908814e-07, |
| "loss": 0.0049, |
| "reward": 1.2904618978500366, |
| "reward_std": 0.09482555650174618, |
| "rewards/accuracy_reward": 0.296875, |
| "rewards/cosine_rewards": -0.006186658749356866, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": -0.00022644927958026528, |
| "step": 141 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 196.5, |
| "epoch": 0.07233825776872134, |
| "grad_norm": 24.285612348709225, |
| "kl": 0.113525390625, |
| "learning_rate": 9.638308711156393e-07, |
| "loss": 0.0045, |
| "reward": 1.4387189745903015, |
| "reward_std": 0.30863603949546814, |
| "rewards/accuracy_reward": 0.4374999888241291, |
| "rewards/cosine_rewards": 0.0012189627159386873, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 142 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 197.5, |
| "epoch": 0.0728476821192053, |
| "grad_norm": 22.094925488371874, |
| "kl": 0.11669921875, |
| "learning_rate": 9.635761589403972e-07, |
| "loss": 0.0047, |
| "reward": 1.495344638824463, |
| "reward_std": 0.46879828721284866, |
| "rewards/accuracy_reward": 0.4937499761581421, |
| "rewards/cosine_rewards": 0.0015945886261761189, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 143 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 195.125, |
| "epoch": 0.07335710646968925, |
| "grad_norm": 7.954348731599845, |
| "kl": 0.1259765625, |
| "learning_rate": 9.633214467651554e-07, |
| "loss": 0.005, |
| "reward": 1.5794875025749207, |
| "reward_std": 0.2703954949975014, |
| "rewards/accuracy_reward": 0.6062500178813934, |
| "rewards/cosine_rewards": 0.0044874417362734675, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": 0.0, |
| "step": 144 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 203.234375, |
| "epoch": 0.0738665308201732, |
| "grad_norm": 10.240352920236468, |
| "kl": 0.1240234375, |
| "learning_rate": 9.630667345899133e-07, |
| "loss": 0.005, |
| "reward": 1.323024868965149, |
| "reward_std": 0.3642221838235855, |
| "rewards/accuracy_reward": 0.32499999552965164, |
| "rewards/cosine_rewards": -0.0019751336076296866, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 145 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 210.96875, |
| "epoch": 0.07437595517065716, |
| "grad_norm": 7.757169620409243, |
| "kl": 0.1318359375, |
| "learning_rate": 9.628120224146714e-07, |
| "loss": 0.0053, |
| "reward": 1.4797114729881287, |
| "reward_std": 0.40076301991939545, |
| "rewards/accuracy_reward": 0.4937500078231096, |
| "rewards/cosine_rewards": 0.001799287972971797, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -0.00021284000831656158, |
| "step": 146 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 203.015625, |
| "epoch": 0.0748853795211411, |
| "grad_norm": 9.791138031684504, |
| "kl": 0.1123046875, |
| "learning_rate": 9.625573102394294e-07, |
| "loss": 0.0045, |
| "reward": 1.5510019659996033, |
| "reward_std": 0.27960680425167084, |
| "rewards/accuracy_reward": 0.5781249850988388, |
| "rewards/cosine_rewards": 0.0043075907160528, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -0.0001806358341127634, |
| "step": 147 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 211.6875, |
| "epoch": 0.07539480387162506, |
| "grad_norm": 7.852116556950569, |
| "kl": 0.12060546875, |
| "learning_rate": 9.623025980641875e-07, |
| "loss": 0.0048, |
| "reward": 1.2943141460418701, |
| "reward_std": 0.43064263463020325, |
| "rewards/accuracy_reward": 0.2968749701976776, |
| "rewards/cosine_rewards": -0.0023121244739741087, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": -0.0002487746678525582, |
| "step": 148 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 220.8125, |
| "epoch": 0.07590422822210902, |
| "grad_norm": 19.19937094751421, |
| "kl": 0.1240234375, |
| "learning_rate": 9.620478858889454e-07, |
| "loss": 0.005, |
| "reward": 1.8400413990020752, |
| "reward_std": 0.39075249433517456, |
| "rewards/accuracy_reward": 0.8593749701976776, |
| "rewards/cosine_rewards": 0.011916308663785458, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": 0.0, |
| "step": 149 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 219.890625, |
| "epoch": 0.07641365257259297, |
| "grad_norm": 17.532821778348946, |
| "kl": 0.1376953125, |
| "learning_rate": 9.617931737137036e-07, |
| "loss": 0.0055, |
| "reward": 1.5498095750808716, |
| "reward_std": 0.29365313798189163, |
| "rewards/accuracy_reward": 0.5781250298023224, |
| "rewards/cosine_rewards": 0.0030243303044699132, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -8.979885024018586e-05, |
| "step": 150 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 215.3125, |
| "epoch": 0.07692307692307693, |
| "grad_norm": 8.653969191960044, |
| "kl": 0.120361328125, |
| "learning_rate": 9.615384615384615e-07, |
| "loss": 0.0048, |
| "reward": 1.2203205227851868, |
| "reward_std": 0.5703159868717194, |
| "rewards/accuracy_reward": 0.24062499403953552, |
| "rewards/cosine_rewards": -0.004613903176505119, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -6.565126386703923e-05, |
| "step": 151 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 214.484375, |
| "epoch": 0.07743250127356088, |
| "grad_norm": 56.921468540439704, |
| "kl": 0.119873046875, |
| "learning_rate": 9.612837493632196e-07, |
| "loss": 0.0048, |
| "reward": 1.2310086488723755, |
| "reward_std": 0.41925153136253357, |
| "rewards/accuracy_reward": 0.2656249925494194, |
| "rewards/cosine_rewards": -0.00321156473364681, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -0.00015470296784769744, |
| "step": 152 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 219.0, |
| "epoch": 0.07794192562404482, |
| "grad_norm": 9.400079372239615, |
| "kl": 0.107666015625, |
| "learning_rate": 9.610290371879775e-07, |
| "loss": 0.0043, |
| "reward": 1.6124141216278076, |
| "reward_std": 0.487982913851738, |
| "rewards/accuracy_reward": 0.606249988079071, |
| "rewards/cosine_rewards": 0.006320342654362321, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": -0.00015624999650754035, |
| "step": 153 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 214.90625, |
| "epoch": 0.07845134997452878, |
| "grad_norm": 16.320123460893743, |
| "kl": 0.125732421875, |
| "learning_rate": 9.607743250127357e-07, |
| "loss": 0.005, |
| "reward": 1.5100122094154358, |
| "reward_std": 0.4279818534851074, |
| "rewards/accuracy_reward": 0.5218749791383743, |
| "rewards/cosine_rewards": 0.0037621970986947417, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 154 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 226.125, |
| "epoch": 0.07896077432501274, |
| "grad_norm": 12.345482711993489, |
| "kl": 0.163330078125, |
| "learning_rate": 9.605196128374936e-07, |
| "loss": 0.0065, |
| "reward": 0.9318991005420685, |
| "reward_std": 0.23740804940462112, |
| "rewards/accuracy_reward": -0.04062497615814209, |
| "rewards/cosine_rewards": -0.011850890005007386, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 155 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 223.671875, |
| "epoch": 0.07947019867549669, |
| "grad_norm": 6.991107442023172, |
| "kl": 0.1240234375, |
| "learning_rate": 9.602649006622515e-07, |
| "loss": 0.005, |
| "reward": 0.9476701319217682, |
| "reward_std": 0.33865927904844284, |
| "rewards/accuracy_reward": -0.04062502086162567, |
| "rewards/cosine_rewards": -0.011704806645866483, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 156 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 239.46875, |
| "epoch": 0.07997962302598065, |
| "grad_norm": 11.299778697394407, |
| "kl": 0.117919921875, |
| "learning_rate": 9.600101884870097e-07, |
| "loss": 0.0047, |
| "reward": 1.3649136424064636, |
| "reward_std": 0.42557042837142944, |
| "rewards/accuracy_reward": 0.3812499940395355, |
| "rewards/cosine_rewards": -0.0007113651372492313, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 157 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 231.015625, |
| "epoch": 0.08048904737646459, |
| "grad_norm": 16.322542283478924, |
| "kl": 0.12255859375, |
| "learning_rate": 9.597554763117676e-07, |
| "loss": 0.0049, |
| "reward": 1.3941306471824646, |
| "reward_std": 0.4155275672674179, |
| "rewards/accuracy_reward": 0.40937498211860657, |
| "rewards/cosine_rewards": 0.000547687232028693, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -0.00016711230273358524, |
| "step": 158 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 213.3125, |
| "epoch": 0.08099847172694855, |
| "grad_norm": 6.393101239111367, |
| "kl": 0.11865234375, |
| "learning_rate": 9.595007641365257e-07, |
| "loss": 0.0047, |
| "reward": 1.2930153012275696, |
| "reward_std": 0.2976529533043504, |
| "rewards/accuracy_reward": 0.296875, |
| "rewards/cosine_rewards": -0.0038597104139626026, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 159 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 215.125, |
| "epoch": 0.0815078960774325, |
| "grad_norm": 15.2053157125375, |
| "kl": 0.119140625, |
| "learning_rate": 9.592460519612836e-07, |
| "loss": 0.0048, |
| "reward": 1.2102863192558289, |
| "reward_std": 0.43368688225746155, |
| "rewards/accuracy_reward": 0.2124999836087227, |
| "rewards/cosine_rewards": -0.0022136420011520386, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 160 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 209.109375, |
| "epoch": 0.08201732042791646, |
| "grad_norm": 8.230345044429809, |
| "kl": 0.11376953125, |
| "learning_rate": 9.589913397860418e-07, |
| "loss": 0.0045, |
| "reward": 1.5247125625610352, |
| "reward_std": 0.313697911798954, |
| "rewards/accuracy_reward": 0.5218749791383743, |
| "rewards/cosine_rewards": 0.0028375727706588805, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 161 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 208.421875, |
| "epoch": 0.08252674477840041, |
| "grad_norm": 7.440050437747031, |
| "kl": 0.132568359375, |
| "learning_rate": 9.587366276107997e-07, |
| "loss": 0.0053, |
| "reward": 1.4958758354187012, |
| "reward_std": 0.2720055654644966, |
| "rewards/accuracy_reward": 0.4937499761581421, |
| "rewards/cosine_rewards": 0.002125886792782694, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 162 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 201.125, |
| "epoch": 0.08303616912888435, |
| "grad_norm": 44.81125981696038, |
| "kl": 0.119384765625, |
| "learning_rate": 9.584819154355578e-07, |
| "loss": 0.0048, |
| "reward": 1.5242316722869873, |
| "reward_std": 0.6014019548892975, |
| "rewards/accuracy_reward": 0.5218749940395355, |
| "rewards/cosine_rewards": 0.0023566827294416726, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 163 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 207.046875, |
| "epoch": 0.08354559347936831, |
| "grad_norm": 13.052733655899614, |
| "kl": 0.119140625, |
| "learning_rate": 9.582272032603158e-07, |
| "loss": 0.0048, |
| "reward": 1.6689130067825317, |
| "reward_std": 0.2884200101252645, |
| "rewards/accuracy_reward": 0.6624999940395355, |
| "rewards/cosine_rewards": 0.0064131125109270215, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 164 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 204.53125, |
| "epoch": 0.08405501782985227, |
| "grad_norm": 45.85990240379708, |
| "kl": 0.455078125, |
| "learning_rate": 9.57972491085074e-07, |
| "loss": 0.0181, |
| "reward": 1.72576242685318, |
| "reward_std": 0.48727013170719147, |
| "rewards/accuracy_reward": 0.7187499701976776, |
| "rewards/cosine_rewards": 0.007012464571744204, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 165 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 203.890625, |
| "epoch": 0.08456444218033622, |
| "grad_norm": 62.93710363981597, |
| "kl": 0.117431640625, |
| "learning_rate": 9.577177789098318e-07, |
| "loss": 0.0047, |
| "reward": 0.9781621694564819, |
| "reward_std": 0.20786645263433456, |
| "rewards/accuracy_reward": -0.012500008568167686, |
| "rewards/cosine_rewards": -0.009250549599528313, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": -8.729050023248419e-05, |
| "step": 166 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 201.109375, |
| "epoch": 0.08507386653082018, |
| "grad_norm": 11.046432411232663, |
| "kl": 0.13134765625, |
| "learning_rate": 9.5746306673459e-07, |
| "loss": 0.0052, |
| "reward": 1.3762089014053345, |
| "reward_std": 0.32985249161720276, |
| "rewards/accuracy_reward": 0.37812500819563866, |
| "rewards/cosine_rewards": -0.0019161199452355504, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 167 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 201.578125, |
| "epoch": 0.08558329088130413, |
| "grad_norm": 5.950394983602238, |
| "kl": 0.11279296875, |
| "learning_rate": 9.572083545593479e-07, |
| "loss": 0.0045, |
| "reward": 1.0190700888633728, |
| "reward_std": 0.6297826766967773, |
| "rewards/accuracy_reward": 0.04062497615814209, |
| "rewards/cosine_rewards": -0.005929919425398111, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 168 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 204.71875, |
| "epoch": 0.08609271523178808, |
| "grad_norm": 17.640294707452878, |
| "kl": 0.11572265625, |
| "learning_rate": 9.56953642384106e-07, |
| "loss": 0.0046, |
| "reward": 0.9798631221055984, |
| "reward_std": 0.20544240390881896, |
| "rewards/accuracy_reward": -0.012500017881393433, |
| "rewards/cosine_rewards": -0.007636879570782185, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 169 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 190.34375, |
| "epoch": 0.08660213958227203, |
| "grad_norm": 7.762505201079191, |
| "kl": 0.112548828125, |
| "learning_rate": 9.56698930208864e-07, |
| "loss": 0.0045, |
| "reward": 1.152494490146637, |
| "reward_std": 0.30975981056690216, |
| "rewards/accuracy_reward": 0.15625, |
| "rewards/cosine_rewards": -0.0037555836606770754, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 170 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 194.875, |
| "epoch": 0.08711156393275599, |
| "grad_norm": 11.506198147130426, |
| "kl": 0.111083984375, |
| "learning_rate": 9.564442180336219e-07, |
| "loss": 0.0045, |
| "reward": 0.9780029058456421, |
| "reward_std": 0.6867689490318298, |
| "rewards/accuracy_reward": -0.012500010430812836, |
| "rewards/cosine_rewards": -0.009497055783867836, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 171 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 189.125, |
| "epoch": 0.08762098828323994, |
| "grad_norm": 11.819517524957538, |
| "kl": 0.10546875, |
| "learning_rate": 9.5618950585838e-07, |
| "loss": 0.0042, |
| "reward": 1.3241556882858276, |
| "reward_std": 0.3773365914821625, |
| "rewards/accuracy_reward": 0.32499998807907104, |
| "rewards/cosine_rewards": -0.0008443233091384172, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 172 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 182.34375, |
| "epoch": 0.0881304126337239, |
| "grad_norm": 8.416955648144409, |
| "kl": 0.116943359375, |
| "learning_rate": 9.55934793683138e-07, |
| "loss": 0.0047, |
| "reward": 1.6661878824234009, |
| "reward_std": 0.20251824986189604, |
| "rewards/accuracy_reward": 0.6624999940395355, |
| "rewards/cosine_rewards": 0.003687863936647773, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 173 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 191.921875, |
| "epoch": 0.08863983698420784, |
| "grad_norm": 7.131693684668596, |
| "kl": 0.12060546875, |
| "learning_rate": 9.55680081507896e-07, |
| "loss": 0.0048, |
| "reward": 1.0765551328659058, |
| "reward_std": 0.3972722738981247, |
| "rewards/accuracy_reward": 0.09999998845160007, |
| "rewards/cosine_rewards": -0.007819817401468754, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 174 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 181.75, |
| "epoch": 0.0891492613346918, |
| "grad_norm": 11.781793838489648, |
| "kl": 0.11083984375, |
| "learning_rate": 9.55425369332654e-07, |
| "loss": 0.0044, |
| "reward": 1.5515506863594055, |
| "reward_std": 0.3071342632174492, |
| "rewards/accuracy_reward": 0.5500000268220901, |
| "rewards/cosine_rewards": 0.0015506702475249767, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 175 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 175.609375, |
| "epoch": 0.08965868568517575, |
| "grad_norm": 40.566147710572594, |
| "kl": 0.109130859375, |
| "learning_rate": 9.551706571574121e-07, |
| "loss": 0.0044, |
| "reward": 1.5527549982070923, |
| "reward_std": 0.39126846194267273, |
| "rewards/accuracy_reward": 0.5499999895691872, |
| "rewards/cosine_rewards": 0.0027549704536795616, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 176 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 178.53125, |
| "epoch": 0.09016811003565971, |
| "grad_norm": 12.817541000164553, |
| "kl": 0.10595703125, |
| "learning_rate": 9.5491594498217e-07, |
| "loss": 0.0042, |
| "reward": 1.9808745980262756, |
| "reward_std": 0.08480274910107255, |
| "rewards/accuracy_reward": 0.971875011920929, |
| "rewards/cosine_rewards": 0.009283588267862797, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": -0.0002840909000951797, |
| "step": 177 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 186.75, |
| "epoch": 0.09067753438614366, |
| "grad_norm": 7.914504626786531, |
| "kl": 0.103759765625, |
| "learning_rate": 9.546612328069282e-07, |
| "loss": 0.0041, |
| "reward": 1.5245178937911987, |
| "reward_std": 0.34963520616292953, |
| "rewards/accuracy_reward": 0.5218750089406967, |
| "rewards/cosine_rewards": 0.0026429439894855022, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 178 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 188.9375, |
| "epoch": 0.0911869587366276, |
| "grad_norm": 8.759963126963402, |
| "kl": 0.13037109375, |
| "learning_rate": 9.544065206316861e-07, |
| "loss": 0.0052, |
| "reward": 1.638785481452942, |
| "reward_std": 0.2284149518236518, |
| "rewards/accuracy_reward": 0.6343750059604645, |
| "rewards/cosine_rewards": 0.0044105148408561945, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 179 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 197.90625, |
| "epoch": 0.09169638308711156, |
| "grad_norm": 6.418887244829745, |
| "kl": 0.116943359375, |
| "learning_rate": 9.541518084564442e-07, |
| "loss": 0.0047, |
| "reward": 1.3810052275657654, |
| "reward_std": 0.4032685235142708, |
| "rewards/accuracy_reward": 0.3812499940395355, |
| "rewards/cosine_rewards": -0.0001616678200662136, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": -8.311169949593022e-05, |
| "step": 180 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 186.078125, |
| "epoch": 0.09220580743759552, |
| "grad_norm": 6.304857280986285, |
| "kl": 0.12890625, |
| "learning_rate": 9.538970962812022e-07, |
| "loss": 0.0051, |
| "reward": 1.2630045115947723, |
| "reward_std": 0.17365956178400666, |
| "rewards/accuracy_reward": 0.2656250037252903, |
| "rewards/cosine_rewards": -0.002620481769554317, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 181 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 190.265625, |
| "epoch": 0.09271523178807947, |
| "grad_norm": 11.35970580998742, |
| "kl": 0.11181640625, |
| "learning_rate": 9.536423841059602e-07, |
| "loss": 0.0045, |
| "reward": 1.6366259455680847, |
| "reward_std": 0.2085256204009056, |
| "rewards/accuracy_reward": 0.6343750059604645, |
| "rewards/cosine_rewards": 0.002349784132093191, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": -9.889240755001083e-05, |
| "step": 182 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 195.96875, |
| "epoch": 0.09322465613856343, |
| "grad_norm": 33.125698049494765, |
| "kl": 0.118408203125, |
| "learning_rate": 9.533876719307182e-07, |
| "loss": 0.0048, |
| "reward": 1.553468942642212, |
| "reward_std": 0.16592675540596247, |
| "rewards/accuracy_reward": 0.550000011920929, |
| "rewards/cosine_rewards": 0.003468883689492941, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": 0.0, |
| "step": 183 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 205.171875, |
| "epoch": 0.09373408048904738, |
| "grad_norm": 15.866784423418947, |
| "kl": 0.115234375, |
| "learning_rate": 9.531329597554763e-07, |
| "loss": 0.0046, |
| "reward": 1.1207141280174255, |
| "reward_std": 0.19428733736276627, |
| "rewards/accuracy_reward": 0.12812499701976776, |
| "rewards/cosine_rewards": -0.007209272123873234, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": -0.00020161290012765676, |
| "step": 184 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 205.640625, |
| "epoch": 0.09424350483953133, |
| "grad_norm": 20.352317051449248, |
| "kl": 0.3115234375, |
| "learning_rate": 9.528782475802343e-07, |
| "loss": 0.0124, |
| "reward": 1.6525439023971558, |
| "reward_std": 0.38362888991832733, |
| "rewards/accuracy_reward": 0.6625000238418579, |
| "rewards/cosine_rewards": 0.005779681145213544, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -0.00011081559932790697, |
| "step": 185 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 201.234375, |
| "epoch": 0.09475292919001528, |
| "grad_norm": 9.691736527471202, |
| "kl": 0.124755859375, |
| "learning_rate": 9.526235354049923e-07, |
| "loss": 0.005, |
| "reward": 0.9626118838787079, |
| "reward_std": 0.40054861456155777, |
| "rewards/accuracy_reward": -0.015625011175870895, |
| "rewards/cosine_rewards": -0.006138101452961564, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 186 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 223.578125, |
| "epoch": 0.09526235354049924, |
| "grad_norm": 9.437447186660206, |
| "kl": 0.123046875, |
| "learning_rate": 9.523688232297503e-07, |
| "loss": 0.0049, |
| "reward": 1.569740116596222, |
| "reward_std": 0.1397167220711708, |
| "rewards/accuracy_reward": 0.5781250298023224, |
| "rewards/cosine_rewards": 0.007240177597850561, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 187 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 233.625, |
| "epoch": 0.09577177789098319, |
| "grad_norm": 14.943967064784786, |
| "kl": 0.15087890625, |
| "learning_rate": 9.521141110545084e-07, |
| "loss": 0.006, |
| "reward": 1.0288785099983215, |
| "reward_std": 0.2980290725827217, |
| "rewards/accuracy_reward": 0.07187499105930328, |
| "rewards/cosine_rewards": -0.011638639261946082, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -0.00010775862028822303, |
| "step": 188 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 213.921875, |
| "epoch": 0.09628120224146715, |
| "grad_norm": 15.000304282598657, |
| "kl": 0.12744140625, |
| "learning_rate": 9.518593988792664e-07, |
| "loss": 0.0051, |
| "reward": 1.3486477732658386, |
| "reward_std": 0.30507488548755646, |
| "rewards/accuracy_reward": 0.34999997913837433, |
| "rewards/cosine_rewards": -0.0010681524872779846, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": -0.0002840909000951797, |
| "step": 189 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 212.96875, |
| "epoch": 0.09679062659195109, |
| "grad_norm": 5.941418391788204, |
| "kl": 0.13916015625, |
| "learning_rate": 9.516046867040244e-07, |
| "loss": 0.0056, |
| "reward": 1.6924657821655273, |
| "reward_std": 0.3630830645561218, |
| "rewards/accuracy_reward": 0.7187499850988388, |
| "rewards/cosine_rewards": 0.004965720232576132, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": 0.0, |
| "step": 190 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 230.421875, |
| "epoch": 0.09730005094243505, |
| "grad_norm": 7.708020896616392, |
| "kl": 0.14453125, |
| "learning_rate": 9.513499745287824e-07, |
| "loss": 0.0058, |
| "reward": 1.250920683145523, |
| "reward_std": 0.4801155626773834, |
| "rewards/accuracy_reward": 0.2968749776482582, |
| "rewards/cosine_rewards": 0.0009206933900713921, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": 0.0, |
| "step": 191 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 247.3125, |
| "epoch": 0.097809475292919, |
| "grad_norm": 13.916574397802314, |
| "kl": 0.13671875, |
| "learning_rate": 9.510952623535404e-07, |
| "loss": 0.0055, |
| "reward": 1.129820704460144, |
| "reward_std": 0.717576265335083, |
| "rewards/accuracy_reward": 0.21249999850988388, |
| "rewards/cosine_rewards": -0.0045542995212599635, |
| "rewards/format_reward": 0.921875, |
| "rewards/repetition_rewards": 0.0, |
| "step": 192 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 244.21875, |
| "epoch": 0.09831889964340296, |
| "grad_norm": 5.201799859391353, |
| "kl": 0.13427734375, |
| "learning_rate": 9.508405501782984e-07, |
| "loss": 0.0054, |
| "reward": 1.3024629950523376, |
| "reward_std": 0.4307016432285309, |
| "rewards/accuracy_reward": 0.37812498956918716, |
| "rewards/cosine_rewards": 0.0024630045518279076, |
| "rewards/format_reward": 0.921875, |
| "rewards/repetition_rewards": 0.0, |
| "step": 193 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 232.203125, |
| "epoch": 0.09882832399388691, |
| "grad_norm": 9.365246983313067, |
| "kl": 0.12939453125, |
| "learning_rate": 9.505858380030564e-07, |
| "loss": 0.0052, |
| "reward": 0.7090668827295303, |
| "reward_std": 0.5288920998573303, |
| "rewards/accuracy_reward": -0.23750002309679985, |
| "rewards/cosine_rewards": -0.022183137945830822, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": 0.0, |
| "step": 194 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 225.8125, |
| "epoch": 0.09933774834437085, |
| "grad_norm": 12.601377217707842, |
| "kl": 0.14501953125, |
| "learning_rate": 9.503311258278145e-07, |
| "loss": 0.0058, |
| "reward": 1.413894236087799, |
| "reward_std": 0.7254346013069153, |
| "rewards/accuracy_reward": 0.5218749940395355, |
| "rewards/cosine_rewards": 0.001518724486231804, |
| "rewards/format_reward": 0.890625, |
| "rewards/repetition_rewards": -0.00012443749437807128, |
| "step": 195 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 226.1875, |
| "epoch": 0.09984717269485481, |
| "grad_norm": 6.45339117400833, |
| "kl": 0.1376953125, |
| "learning_rate": 9.500764136525725e-07, |
| "loss": 0.0055, |
| "reward": 1.5756230354309082, |
| "reward_std": 0.48759835958480835, |
| "rewards/accuracy_reward": 0.6624999940395355, |
| "rewards/cosine_rewards": 0.007316130446270108, |
| "rewards/format_reward": 0.90625, |
| "rewards/repetition_rewards": -0.0004430353583302349, |
| "step": 196 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 227.796875, |
| "epoch": 0.10035659704533877, |
| "grad_norm": 11.359096408502863, |
| "kl": 0.2138671875, |
| "learning_rate": 9.498217014773305e-07, |
| "loss": 0.0086, |
| "reward": 1.2498727440834045, |
| "reward_std": 0.5983296632766724, |
| "rewards/accuracy_reward": 0.37812499701976776, |
| "rewards/cosine_rewards": -0.0031815596157684922, |
| "rewards/format_reward": 0.875, |
| "rewards/repetition_rewards": -7.070136052789167e-05, |
| "step": 197 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 232.234375, |
| "epoch": 0.10086602139582272, |
| "grad_norm": 13.920508776432966, |
| "kl": 0.12255859375, |
| "learning_rate": 9.495669893020886e-07, |
| "loss": 0.0049, |
| "reward": 0.6862081587314606, |
| "reward_std": 0.7485357820987701, |
| "rewards/accuracy_reward": -0.2656250223517418, |
| "rewards/cosine_rewards": -0.01686593284830451, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -5.089576370664872e-05, |
| "step": 198 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 205.140625, |
| "epoch": 0.10137544574630668, |
| "grad_norm": 8.671114765474545, |
| "kl": 0.124267578125, |
| "learning_rate": 9.493122771268466e-07, |
| "loss": 0.005, |
| "reward": 1.1818422079086304, |
| "reward_std": 0.5584293901920319, |
| "rewards/accuracy_reward": 0.29375000298023224, |
| "rewards/cosine_rewards": -0.002258662148960866, |
| "rewards/format_reward": 0.890625, |
| "rewards/repetition_rewards": -0.0002741228090599179, |
| "step": 199 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 216.875, |
| "epoch": 0.10188487009679063, |
| "grad_norm": 10.078840755345926, |
| "kl": 0.126708984375, |
| "learning_rate": 9.490575649516046e-07, |
| "loss": 0.0051, |
| "reward": 1.3181660771369934, |
| "reward_std": 0.6019489467144012, |
| "rewards/accuracy_reward": 0.40937498211860657, |
| "rewards/cosine_rewards": 0.002541057765483856, |
| "rewards/format_reward": 0.90625, |
| "rewards/repetition_rewards": 0.0, |
| "step": 200 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 199.515625, |
| "epoch": 0.10239429444727458, |
| "grad_norm": 5.555372749575627, |
| "kl": 0.130859375, |
| "learning_rate": 9.488028527763627e-07, |
| "loss": 0.0052, |
| "reward": 1.5905040502548218, |
| "reward_std": 0.41512130200862885, |
| "rewards/accuracy_reward": 0.6624999940395355, |
| "rewards/cosine_rewards": 0.006129102781414986, |
| "rewards/format_reward": 0.921875, |
| "rewards/repetition_rewards": 0.0, |
| "step": 201 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 235.03125, |
| "epoch": 0.10290371879775853, |
| "grad_norm": 10.052848791937414, |
| "kl": 0.12890625, |
| "learning_rate": 9.485481406011207e-07, |
| "loss": 0.0051, |
| "reward": 1.1239948272705078, |
| "reward_std": 0.9119550585746765, |
| "rewards/accuracy_reward": 0.26875001192092896, |
| "rewards/cosine_rewards": -0.004009488970041275, |
| "rewards/format_reward": 0.859375, |
| "rewards/repetition_rewards": -0.00012065636838087812, |
| "step": 202 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 243.5, |
| "epoch": 0.10341314314824249, |
| "grad_norm": 11.024825341246498, |
| "kl": 0.115966796875, |
| "learning_rate": 9.482934284258787e-07, |
| "loss": 0.0046, |
| "reward": 0.9447762966156006, |
| "reward_std": 0.800986647605896, |
| "rewards/accuracy_reward": 0.12812498584389687, |
| "rewards/cosine_rewards": -0.011282204184681177, |
| "rewards/format_reward": 0.828125, |
| "rewards/repetition_rewards": -0.00019145716942148283, |
| "step": 203 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 232.375, |
| "epoch": 0.10392256749872644, |
| "grad_norm": 7.842670844504004, |
| "kl": 0.118408203125, |
| "learning_rate": 9.480387162506367e-07, |
| "loss": 0.0047, |
| "reward": 1.2975149750709534, |
| "reward_std": 0.6046717762947083, |
| "rewards/accuracy_reward": 0.40937497094273567, |
| "rewards/cosine_rewards": -0.0023666354827582836, |
| "rewards/format_reward": 0.890625, |
| "rewards/repetition_rewards": -0.00011837121564894915, |
| "step": 204 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 224.234375, |
| "epoch": 0.1044319918492104, |
| "grad_norm": 20.462186918743633, |
| "kl": 0.125732421875, |
| "learning_rate": 9.477840040753947e-07, |
| "loss": 0.005, |
| "reward": 1.0097321271896362, |
| "reward_std": 0.4872446656227112, |
| "rewards/accuracy_reward": 0.1250000149011612, |
| "rewards/cosine_rewards": -0.005829372443258762, |
| "rewards/format_reward": 0.890625, |
| "rewards/repetition_rewards": -6.351625779643655e-05, |
| "step": 205 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 224.328125, |
| "epoch": 0.10494141619969434, |
| "grad_norm": 18.578745554695768, |
| "kl": 0.1298828125, |
| "learning_rate": 9.475292919001527e-07, |
| "loss": 0.0052, |
| "reward": 0.8584832549095154, |
| "reward_std": 0.5744369626045227, |
| "rewards/accuracy_reward": 0.040624991059303284, |
| "rewards/cosine_rewards": -0.010225818026810884, |
| "rewards/format_reward": 0.828125, |
| "rewards/repetition_rewards": -4.101049853488803e-05, |
| "step": 206 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 209.234375, |
| "epoch": 0.1054508405501783, |
| "grad_norm": 20.30723423804259, |
| "kl": 0.11376953125, |
| "learning_rate": 9.472745797249107e-07, |
| "loss": 0.0045, |
| "reward": 1.0917281210422516, |
| "reward_std": 0.4984763488173485, |
| "rewards/accuracy_reward": 0.24062497913837433, |
| "rewards/cosine_rewards": -0.008111415430903435, |
| "rewards/format_reward": 0.859375, |
| "rewards/repetition_rewards": -0.0001604560275154654, |
| "step": 207 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 202.28125, |
| "epoch": 0.10596026490066225, |
| "grad_norm": 12.813170138938949, |
| "kl": 0.130859375, |
| "learning_rate": 9.470198675496688e-07, |
| "loss": 0.0052, |
| "reward": 1.258288562297821, |
| "reward_std": 0.4630318433046341, |
| "rewards/accuracy_reward": 0.3812500238418579, |
| "rewards/cosine_rewards": 0.0022279657423496246, |
| "rewards/format_reward": 0.875, |
| "rewards/repetition_rewards": -0.00018934992840513587, |
| "step": 208 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 199.0, |
| "epoch": 0.10646968925114621, |
| "grad_norm": 10.281274330223528, |
| "kl": 0.15185546875, |
| "learning_rate": 9.467651553744268e-07, |
| "loss": 0.0061, |
| "reward": 1.2817729711532593, |
| "reward_std": 0.4518425017595291, |
| "rewards/accuracy_reward": 0.40937501937150955, |
| "rewards/cosine_rewards": -0.0026020415825769305, |
| "rewards/format_reward": 0.875, |
| "rewards/repetition_rewards": 0.0, |
| "step": 209 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 191.90625, |
| "epoch": 0.10697911360163016, |
| "grad_norm": 5.991490542584776, |
| "kl": 0.118896484375, |
| "learning_rate": 9.465104431991848e-07, |
| "loss": 0.0047, |
| "reward": 1.537351131439209, |
| "reward_std": 0.5478895753622055, |
| "rewards/accuracy_reward": 0.6875000149011612, |
| "rewards/cosine_rewards": 0.00610114517621696, |
| "rewards/format_reward": 0.84375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 210 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 217.015625, |
| "epoch": 0.1074885379521141, |
| "grad_norm": 60.00536554673066, |
| "kl": 0.115966796875, |
| "learning_rate": 9.462557310239428e-07, |
| "loss": 0.0046, |
| "reward": 0.7785031795501709, |
| "reward_std": 0.42832519114017487, |
| "rewards/accuracy_reward": -0.1250000223517418, |
| "rewards/cosine_rewards": -0.018284045159816742, |
| "rewards/format_reward": 0.921875, |
| "rewards/repetition_rewards": -8.778089977568015e-05, |
| "step": 211 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 205.21875, |
| "epoch": 0.10799796230259806, |
| "grad_norm": 21.72003474650607, |
| "kl": 0.1220703125, |
| "learning_rate": 9.460010188487009e-07, |
| "loss": 0.0049, |
| "reward": 1.0551989674568176, |
| "reward_std": 0.46487441658973694, |
| "rewards/accuracy_reward": 0.18437497317790985, |
| "rewards/cosine_rewards": -0.004139983095228672, |
| "rewards/format_reward": 0.875, |
| "rewards/repetition_rewards": -3.600230411393568e-05, |
| "step": 212 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 237.609375, |
| "epoch": 0.10850738665308202, |
| "grad_norm": 13.593112035734373, |
| "kl": 0.118896484375, |
| "learning_rate": 9.457463066734589e-07, |
| "loss": 0.0048, |
| "reward": 1.4092811346054077, |
| "reward_std": 0.6851305663585663, |
| "rewards/accuracy_reward": 0.6343750059604645, |
| "rewards/cosine_rewards": 0.009402429801411927, |
| "rewards/format_reward": 0.765625, |
| "rewards/repetition_rewards": -0.00012127523950766772, |
| "step": 213 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 266.65625, |
| "epoch": 0.10901681100356597, |
| "grad_norm": 102.50188740817565, |
| "kl": 0.114013671875, |
| "learning_rate": 9.45491594498217e-07, |
| "loss": 0.0046, |
| "reward": 1.3722986578941345, |
| "reward_std": 0.5870523750782013, |
| "rewards/accuracy_reward": 0.5218750089406967, |
| "rewards/cosine_rewards": -0.008539619389921427, |
| "rewards/format_reward": 0.859375, |
| "rewards/repetition_rewards": -0.000411735316447448, |
| "step": 214 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 264.796875, |
| "epoch": 0.10952623535404993, |
| "grad_norm": 17.000809347446246, |
| "kl": 0.114013671875, |
| "learning_rate": 9.452368823229751e-07, |
| "loss": 0.0046, |
| "reward": 1.182218611240387, |
| "reward_std": 0.6600647866725922, |
| "rewards/accuracy_reward": 0.265625, |
| "rewards/cosine_rewards": -0.02086095977574587, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": -4.542151145869866e-05, |
| "step": 215 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 305.796875, |
| "epoch": 0.11003565970453388, |
| "grad_norm": 21.15698667263886, |
| "kl": 0.107666015625, |
| "learning_rate": 9.449821701477331e-07, |
| "loss": 0.0043, |
| "reward": 1.1834356784820557, |
| "reward_std": 0.686463937163353, |
| "rewards/accuracy_reward": 0.2968749925494194, |
| "rewards/cosine_rewards": -0.019689313136041164, |
| "rewards/format_reward": 0.90625, |
| "rewards/repetition_rewards": 0.0, |
| "step": 216 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 400.296875, |
| "epoch": 0.11054508405501783, |
| "grad_norm": 7.9150578217370535, |
| "kl": 0.09814453125, |
| "learning_rate": 9.447274579724911e-07, |
| "loss": 0.0039, |
| "reward": 1.0662736892700195, |
| "reward_std": 0.8222787380218506, |
| "rewards/accuracy_reward": 0.18437499552965164, |
| "rewards/cosine_rewards": -0.0396728478372097, |
| "rewards/format_reward": 0.921875, |
| "rewards/repetition_rewards": -0.00030338978831423447, |
| "step": 217 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 345.671875, |
| "epoch": 0.11105450840550178, |
| "grad_norm": 10.397918174345362, |
| "kl": 0.1357421875, |
| "learning_rate": 9.444727457972492e-07, |
| "loss": 0.0054, |
| "reward": 1.6620882153511047, |
| "reward_std": 0.6701975017786026, |
| "rewards/accuracy_reward": 0.71875, |
| "rewards/cosine_rewards": 0.021556629799306393, |
| "rewards/format_reward": 0.921875, |
| "rewards/repetition_rewards": -9.343791316496208e-05, |
| "step": 218 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 361.984375, |
| "epoch": 0.11156393275598574, |
| "grad_norm": 16.664810091620872, |
| "kl": 0.09619140625, |
| "learning_rate": 9.442180336220072e-07, |
| "loss": 0.0038, |
| "reward": 0.6033791899681091, |
| "reward_std": 0.47761378437280655, |
| "rewards/accuracy_reward": -0.2656250149011612, |
| "rewards/cosine_rewards": -0.0521757323294878, |
| "rewards/format_reward": 0.921875, |
| "rewards/repetition_rewards": -0.0006950152310309932, |
| "step": 219 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 359.359375, |
| "epoch": 0.11207335710646969, |
| "grad_norm": 6.030171938320145, |
| "kl": 0.09423828125, |
| "learning_rate": 9.439633214467651e-07, |
| "loss": 0.0038, |
| "reward": 1.0732125043869019, |
| "reward_std": 0.532948449254036, |
| "rewards/accuracy_reward": 0.18437499180436134, |
| "rewards/cosine_rewards": -0.017268475145101547, |
| "rewards/format_reward": 0.90625, |
| "rewards/repetition_rewards": -0.00014401252064999426, |
| "step": 220 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 467.921875, |
| "epoch": 0.11258278145695365, |
| "grad_norm": 13.578372073106125, |
| "kl": 0.08740234375, |
| "learning_rate": 9.437086092715231e-07, |
| "loss": 0.0035, |
| "reward": 1.089949607849121, |
| "reward_std": 0.7014666199684143, |
| "rewards/accuracy_reward": 0.18437499180436134, |
| "rewards/cosine_rewards": -0.04696316970512271, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": -0.0005872593028470874, |
| "step": 221 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 295.15625, |
| "epoch": 0.11309220580743759, |
| "grad_norm": 9.243544591708364, |
| "kl": 0.099609375, |
| "learning_rate": 9.434538970962812e-07, |
| "loss": 0.004, |
| "reward": 1.25474151968956, |
| "reward_std": 0.42495501041412354, |
| "rewards/accuracy_reward": 0.2968750074505806, |
| "rewards/cosine_rewards": -0.010176160372793674, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -0.0007073541928548366, |
| "step": 222 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 327.9375, |
| "epoch": 0.11360163015792155, |
| "grad_norm": 11.854145682727308, |
| "kl": 0.09423828125, |
| "learning_rate": 9.431991849210392e-07, |
| "loss": 0.0038, |
| "reward": 1.336020827293396, |
| "reward_std": 0.5929334163665771, |
| "rewards/accuracy_reward": 0.3812499977648258, |
| "rewards/cosine_rewards": 0.0018316814675927162, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": -0.0001858295945567079, |
| "step": 223 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 224.109375, |
| "epoch": 0.1141110545084055, |
| "grad_norm": 9.830726171785136, |
| "kl": 0.13330078125, |
| "learning_rate": 9.429444727457972e-07, |
| "loss": 0.0053, |
| "reward": 0.9984832406044006, |
| "reward_std": 0.45606285333633423, |
| "rewards/accuracy_reward": 0.043749988079071045, |
| "rewards/cosine_rewards": -0.014016739558428526, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": 0.0, |
| "step": 224 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 213.640625, |
| "epoch": 0.11462047885888946, |
| "grad_norm": 9.176338530222994, |
| "kl": 0.115234375, |
| "learning_rate": 9.426897605705553e-07, |
| "loss": 0.0046, |
| "reward": 1.2019062638282776, |
| "reward_std": 0.7189642786979675, |
| "rewards/accuracy_reward": 0.29374999925494194, |
| "rewards/cosine_rewards": -0.01371871994342655, |
| "rewards/format_reward": 0.921875, |
| "rewards/repetition_rewards": 0.0, |
| "step": 225 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 199.71875, |
| "epoch": 0.11512990320937341, |
| "grad_norm": 11.349158424596924, |
| "kl": 0.110107421875, |
| "learning_rate": 9.424350483953133e-07, |
| "loss": 0.0044, |
| "reward": 1.3144216537475586, |
| "reward_std": 0.4914311468601227, |
| "rewards/accuracy_reward": 0.32499998807907104, |
| "rewards/cosine_rewards": 0.005097148037748411, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -5.056634472566657e-05, |
| "step": 226 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 232.390625, |
| "epoch": 0.11563932755985736, |
| "grad_norm": 8.61981591225416, |
| "kl": 0.105712890625, |
| "learning_rate": 9.421803362200713e-07, |
| "loss": 0.0042, |
| "reward": 1.064522534608841, |
| "reward_std": 0.3509945422410965, |
| "rewards/accuracy_reward": 0.15312501788139343, |
| "rewards/cosine_rewards": -0.010458544362336397, |
| "rewards/format_reward": 0.921875, |
| "rewards/repetition_rewards": -1.8939394067274407e-05, |
| "step": 227 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 197.75, |
| "epoch": 0.11614875191034131, |
| "grad_norm": 12.524545820572607, |
| "kl": 0.106689453125, |
| "learning_rate": 9.419256240448294e-07, |
| "loss": 0.0043, |
| "reward": 1.3913479149341583, |
| "reward_std": 0.2862061709165573, |
| "rewards/accuracy_reward": 0.40937498956918716, |
| "rewards/cosine_rewards": -0.0021625147201120853, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -0.00023957982193678617, |
| "step": 228 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 193.5, |
| "epoch": 0.11665817626082527, |
| "grad_norm": 17.83927533701907, |
| "kl": 0.13232421875, |
| "learning_rate": 9.416709118695874e-07, |
| "loss": 0.0053, |
| "reward": 1.5334136486053467, |
| "reward_std": 0.45667168498039246, |
| "rewards/accuracy_reward": 0.6062500029802322, |
| "rewards/cosine_rewards": 0.005288586835376918, |
| "rewards/format_reward": 0.921875, |
| "rewards/repetition_rewards": 0.0, |
| "step": 229 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 247.671875, |
| "epoch": 0.11716760061130922, |
| "grad_norm": 23.81656431934888, |
| "kl": 0.108642578125, |
| "learning_rate": 9.414161996943454e-07, |
| "loss": 0.0043, |
| "reward": 1.1264008283615112, |
| "reward_std": 0.6892756521701813, |
| "rewards/accuracy_reward": 0.21249999105930328, |
| "rewards/cosine_rewards": -0.02350334101356566, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": -9.586199303157628e-05, |
| "step": 230 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 262.59375, |
| "epoch": 0.11767702496179318, |
| "grad_norm": 28.467466655884312, |
| "kl": 0.12109375, |
| "learning_rate": 9.411614875191034e-07, |
| "loss": 0.0048, |
| "reward": 1.4944193363189697, |
| "reward_std": 0.3786798119544983, |
| "rewards/accuracy_reward": 0.518750011920929, |
| "rewards/cosine_rewards": 0.0069862306118011475, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -6.69164874125272e-05, |
| "step": 231 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 248.40625, |
| "epoch": 0.11818644931227713, |
| "grad_norm": 9.248494107592876, |
| "kl": 0.12060546875, |
| "learning_rate": 9.409067753438615e-07, |
| "loss": 0.0048, |
| "reward": 1.2941021919250488, |
| "reward_std": 0.5259552597999573, |
| "rewards/accuracy_reward": 0.3749999776482582, |
| "rewards/cosine_rewards": 0.012875130865722895, |
| "rewards/format_reward": 0.90625, |
| "rewards/repetition_rewards": -2.2944199372432195e-05, |
| "step": 232 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 264.15625, |
| "epoch": 0.11869587366276108, |
| "grad_norm": 14.194855529249866, |
| "kl": 0.107666015625, |
| "learning_rate": 9.406520631686195e-07, |
| "loss": 0.0043, |
| "reward": 1.4532509446144104, |
| "reward_std": 0.47306837141513824, |
| "rewards/accuracy_reward": 0.46562501788139343, |
| "rewards/cosine_rewards": 0.003419560845941305, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -0.00016866176156327128, |
| "step": 233 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 376.875, |
| "epoch": 0.11920529801324503, |
| "grad_norm": 24.881210594835412, |
| "kl": 0.0986328125, |
| "learning_rate": 9.403973509933774e-07, |
| "loss": 0.0039, |
| "reward": 0.9971878528594971, |
| "reward_std": 0.8903799057006836, |
| "rewards/accuracy_reward": 0.09375, |
| "rewards/cosine_rewards": -0.018273995257914066, |
| "rewards/format_reward": 0.921875, |
| "rewards/repetition_rewards": -0.00016315293032675982, |
| "step": 234 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 432.28125, |
| "epoch": 0.11971472236372899, |
| "grad_norm": 4.4996951306965975, |
| "kl": 0.08544921875, |
| "learning_rate": 9.401426388181355e-07, |
| "loss": 0.0034, |
| "reward": 1.3378186225891113, |
| "reward_std": 0.8512288331985474, |
| "rewards/accuracy_reward": 0.4593750089406967, |
| "rewards/cosine_rewards": -0.027322867885231972, |
| "rewards/format_reward": 0.90625, |
| "rewards/repetition_rewards": -0.000483501615235582, |
| "step": 235 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 465.765625, |
| "epoch": 0.12022414671421294, |
| "grad_norm": 5.402908709299499, |
| "kl": 0.080322265625, |
| "learning_rate": 9.398879266428935e-07, |
| "loss": 0.0032, |
| "reward": 1.4937435388565063, |
| "reward_std": 0.35082364082336426, |
| "rewards/accuracy_reward": 0.5499999821186066, |
| "rewards/cosine_rewards": -0.024816589895635843, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -0.0001899001763376873, |
| "step": 236 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 431.796875, |
| "epoch": 0.1207335710646969, |
| "grad_norm": 11.178072528468537, |
| "kl": 0.0947265625, |
| "learning_rate": 9.396332144676515e-07, |
| "loss": 0.0038, |
| "reward": 1.1548867225646973, |
| "reward_std": 0.8218154907226562, |
| "rewards/accuracy_reward": 0.23749998956918716, |
| "rewards/cosine_rewards": -0.0042152018286287785, |
| "rewards/format_reward": 0.921875, |
| "rewards/repetition_rewards": -0.0002730985652306117, |
| "step": 237 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 506.640625, |
| "epoch": 0.12124299541518084, |
| "grad_norm": 3.9623481852584983, |
| "kl": 0.078857421875, |
| "learning_rate": 9.393785022924095e-07, |
| "loss": 0.0032, |
| "reward": 1.2667301297187805, |
| "reward_std": 0.8781076371669769, |
| "rewards/accuracy_reward": 0.40937498211860657, |
| "rewards/cosine_rewards": -0.04850983805954456, |
| "rewards/format_reward": 0.90625, |
| "rewards/repetition_rewards": -0.0003849874483421445, |
| "step": 238 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 443.515625, |
| "epoch": 0.1217524197656648, |
| "grad_norm": 5.742693044990549, |
| "kl": 0.094482421875, |
| "learning_rate": 9.391237901171676e-07, |
| "loss": 0.0038, |
| "reward": 0.6565631031990051, |
| "reward_std": 0.7225559949874878, |
| "rewards/accuracy_reward": -0.15312501043081284, |
| "rewards/cosine_rewards": -0.08046763762831688, |
| "rewards/format_reward": 0.890625, |
| "rewards/repetition_rewards": -0.00046927113726269454, |
| "step": 239 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 347.265625, |
| "epoch": 0.12226184411614875, |
| "grad_norm": 7.476762059683771, |
| "kl": 0.08984375, |
| "learning_rate": 9.388690779419256e-07, |
| "loss": 0.0036, |
| "reward": 1.2646641731262207, |
| "reward_std": 0.35142165422439575, |
| "rewards/accuracy_reward": 0.296875, |
| "rewards/cosine_rewards": -0.016350463964045048, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -0.00023539320682175457, |
| "step": 240 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 241.125, |
| "epoch": 0.12277126846663271, |
| "grad_norm": 7.077545719384053, |
| "kl": 0.101318359375, |
| "learning_rate": 9.386143657666836e-07, |
| "loss": 0.0041, |
| "reward": 0.9919856488704681, |
| "reward_std": 0.6525652855634689, |
| "rewards/accuracy_reward": 0.043749988079071045, |
| "rewards/cosine_rewards": -0.020295456051826477, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -0.00021890102652832866, |
| "step": 241 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 240.765625, |
| "epoch": 0.12328069281711666, |
| "grad_norm": 8.441107538365218, |
| "kl": 0.1044921875, |
| "learning_rate": 9.383596535914417e-07, |
| "loss": 0.0042, |
| "reward": 1.616421401500702, |
| "reward_std": 0.3022947758436203, |
| "rewards/accuracy_reward": 0.6593749970197678, |
| "rewards/cosine_rewards": 0.004121019504964352, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": -0.00019959894416388124, |
| "step": 242 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 175.796875, |
| "epoch": 0.1237901171676006, |
| "grad_norm": 9.380134906229868, |
| "kl": 0.11083984375, |
| "learning_rate": 9.381049414161997e-07, |
| "loss": 0.0044, |
| "reward": 1.3176445960998535, |
| "reward_std": 0.3997122645378113, |
| "rewards/accuracy_reward": 0.32499998807907104, |
| "rewards/cosine_rewards": -0.0073250585701316595, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": -3.028100763913244e-05, |
| "step": 243 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 220.53125, |
| "epoch": 0.12429954151808456, |
| "grad_norm": 5.4472083052212765, |
| "kl": 0.109375, |
| "learning_rate": 9.378502292409577e-07, |
| "loss": 0.0044, |
| "reward": 1.6422365307807922, |
| "reward_std": 0.2728146519511938, |
| "rewards/accuracy_reward": 0.6624999940395355, |
| "rewards/cosine_rewards": 0.01141381449997425, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -0.00042724609375, |
| "step": 244 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 248.828125, |
| "epoch": 0.12480896586856852, |
| "grad_norm": 8.442731808091501, |
| "kl": 0.1083984375, |
| "learning_rate": 9.375955170657157e-07, |
| "loss": 0.0043, |
| "reward": 1.3157773613929749, |
| "reward_std": 0.4320952445268631, |
| "rewards/accuracy_reward": 0.3531249985098839, |
| "rewards/cosine_rewards": 0.009643017314374447, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": -0.00011561772407731041, |
| "step": 245 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 222.6875, |
| "epoch": 0.12531839021905247, |
| "grad_norm": 14.294535330077375, |
| "kl": 0.1171875, |
| "learning_rate": 9.373408048904738e-07, |
| "loss": 0.0047, |
| "reward": 1.331631362438202, |
| "reward_std": 0.4216170907020569, |
| "rewards/accuracy_reward": 0.3531249985098839, |
| "rewards/cosine_rewards": 0.00982090923935175, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -6.454958565882407e-05, |
| "step": 246 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 219.984375, |
| "epoch": 0.12582781456953643, |
| "grad_norm": 9.55932148178992, |
| "kl": 0.108642578125, |
| "learning_rate": 9.370860927152318e-07, |
| "loss": 0.0043, |
| "reward": 1.340530276298523, |
| "reward_std": 0.4534093588590622, |
| "rewards/accuracy_reward": 0.3531250096857548, |
| "rewards/cosine_rewards": 0.003091069171205163, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -6.0797665355494246e-05, |
| "step": 247 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 325.234375, |
| "epoch": 0.12633723892002038, |
| "grad_norm": 17.80951371391722, |
| "kl": 0.110595703125, |
| "learning_rate": 9.368313805399897e-07, |
| "loss": 0.0044, |
| "reward": 1.2751246690750122, |
| "reward_std": 0.520209550857544, |
| "rewards/accuracy_reward": 0.2968749850988388, |
| "rewards/cosine_rewards": 0.010451191570609808, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -0.0009515111669315957, |
| "step": 248 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 366.625, |
| "epoch": 0.12684666327050434, |
| "grad_norm": 12.086775377390172, |
| "kl": 0.111083984375, |
| "learning_rate": 9.365766683647478e-07, |
| "loss": 0.0044, |
| "reward": 0.8402246385812759, |
| "reward_std": 0.6177513003349304, |
| "rewards/accuracy_reward": -0.04062502086162567, |
| "rewards/cosine_rewards": -0.04077841015532613, |
| "rewards/format_reward": 0.921875, |
| "rewards/repetition_rewards": -0.0002469850951456465, |
| "step": 249 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 449.125, |
| "epoch": 0.1273560876209883, |
| "grad_norm": 11.748922331365623, |
| "kl": 0.0869140625, |
| "learning_rate": 9.363219561895058e-07, |
| "loss": 0.0035, |
| "reward": 1.7429784536361694, |
| "reward_std": 0.6669142842292786, |
| "rewards/accuracy_reward": 0.746874988079071, |
| "rewards/cosine_rewards": 0.02791230659931898, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -0.000558776329853572, |
| "step": 250 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 523.34375, |
| "epoch": 0.12786551197147222, |
| "grad_norm": 5.635036318066103, |
| "kl": 0.073974609375, |
| "learning_rate": 9.360672440142638e-07, |
| "loss": 0.003, |
| "reward": 1.3756027221679688, |
| "reward_std": 0.3430413454771042, |
| "rewards/accuracy_reward": 0.4375, |
| "rewards/cosine_rewards": -0.01460547186434269, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": -0.0004167625156696886, |
| "step": 251 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 583.78125, |
| "epoch": 0.12837493632195618, |
| "grad_norm": 3.7721855945126013, |
| "kl": 0.071533203125, |
| "learning_rate": 9.358125318390219e-07, |
| "loss": 0.0029, |
| "reward": 1.1054343283176422, |
| "reward_std": 0.9506143927574158, |
| "rewards/accuracy_reward": 0.23749998211860657, |
| "rewards/cosine_rewards": -0.06930245459079742, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": -0.00026325164799345657, |
| "step": 252 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 691.8125, |
| "epoch": 0.12888436067244013, |
| "grad_norm": 4.515905173052182, |
| "kl": 0.06494140625, |
| "learning_rate": 9.355578196637799e-07, |
| "loss": 0.0026, |
| "reward": 1.1395662426948547, |
| "reward_std": 1.164560616016388, |
| "rewards/accuracy_reward": 0.24062499403953552, |
| "rewards/cosine_rewards": -0.06932513415813446, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -0.0004836731095565483, |
| "step": 253 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 750.703125, |
| "epoch": 0.1293937850229241, |
| "grad_norm": 3.759822074251192, |
| "kl": 0.0609130859375, |
| "learning_rate": 9.353031074885379e-07, |
| "loss": 0.0024, |
| "reward": 1.3212904930114746, |
| "reward_std": 0.9738726019859314, |
| "rewards/accuracy_reward": 0.4625000059604645, |
| "rewards/cosine_rewards": -0.03070250153541565, |
| "rewards/format_reward": 0.890625, |
| "rewards/repetition_rewards": -0.0011320026533212513, |
| "step": 254 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 604.40625, |
| "epoch": 0.12990320937340805, |
| "grad_norm": 4.697641184697458, |
| "kl": 0.082763671875, |
| "learning_rate": 9.350483953132959e-07, |
| "loss": 0.0033, |
| "reward": 1.1115484535694122, |
| "reward_std": 0.7478219866752625, |
| "rewards/accuracy_reward": 0.23125000298023224, |
| "rewards/cosine_rewards": -0.05675292294472456, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": -0.00044860908383270726, |
| "step": 255 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 635.84375, |
| "epoch": 0.130412633723892, |
| "grad_norm": 4.028619362240205, |
| "kl": 0.09765625, |
| "learning_rate": 9.34793683138054e-07, |
| "loss": 0.0039, |
| "reward": 1.457118034362793, |
| "reward_std": 0.788001298904419, |
| "rewards/accuracy_reward": 0.5218749791383743, |
| "rewards/cosine_rewards": -0.0015127966180443764, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": -0.0007440973713528365, |
| "step": 256 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 589.40625, |
| "epoch": 0.13092205807437596, |
| "grad_norm": 5.63417318829876, |
| "kl": 0.07275390625, |
| "learning_rate": 9.34538970962812e-07, |
| "loss": 0.0029, |
| "reward": 1.4154019951820374, |
| "reward_std": 0.815990686416626, |
| "rewards/accuracy_reward": 0.518750011920929, |
| "rewards/cosine_rewards": -0.02476619742810726, |
| "rewards/format_reward": 0.921875, |
| "rewards/repetition_rewards": -0.00045683811185881495, |
| "step": 257 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 676.984375, |
| "epoch": 0.1314314824248599, |
| "grad_norm": 6.021417699991294, |
| "kl": 0.065673828125, |
| "learning_rate": 9.3428425878757e-07, |
| "loss": 0.0026, |
| "reward": 0.6927553117275238, |
| "reward_std": 0.8777336776256561, |
| "rewards/accuracy_reward": -0.0781250074505806, |
| "rewards/cosine_rewards": -0.15041033178567886, |
| "rewards/format_reward": 0.921875, |
| "rewards/repetition_rewards": -0.0005843567778356373, |
| "step": 258 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 555.203125, |
| "epoch": 0.13194090677534387, |
| "grad_norm": 8.306056976533926, |
| "kl": 0.082763671875, |
| "learning_rate": 9.340295466123281e-07, |
| "loss": 0.0033, |
| "reward": 1.2112269699573517, |
| "reward_std": 0.9674933552742004, |
| "rewards/accuracy_reward": 0.43437499552965164, |
| "rewards/cosine_rewards": -0.08218972198665142, |
| "rewards/format_reward": 0.859375, |
| "rewards/repetition_rewards": -0.0003333477216074243, |
| "step": 259 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 694.484375, |
| "epoch": 0.13245033112582782, |
| "grad_norm": 6.232070420425315, |
| "kl": 0.06689453125, |
| "learning_rate": 9.337748344370861e-07, |
| "loss": 0.0027, |
| "reward": 1.0117461681365967, |
| "reward_std": 0.7802118062973022, |
| "rewards/accuracy_reward": 0.21249999478459358, |
| "rewards/cosine_rewards": -0.09079772233963013, |
| "rewards/format_reward": 0.890625, |
| "rewards/repetition_rewards": -0.0005810301227029413, |
| "step": 260 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 543.5625, |
| "epoch": 0.13295975547631178, |
| "grad_norm": 6.334798789966088, |
| "kl": 0.08544921875, |
| "learning_rate": 9.335201222618441e-07, |
| "loss": 0.0034, |
| "reward": 1.0542153716087341, |
| "reward_std": 0.8291297852993011, |
| "rewards/accuracy_reward": 0.18124999105930328, |
| "rewards/cosine_rewards": -0.032575659453868866, |
| "rewards/format_reward": 0.90625, |
| "rewards/repetition_rewards": -0.0007089868013281375, |
| "step": 261 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 356.015625, |
| "epoch": 0.1334691798267957, |
| "grad_norm": 10.460916199726386, |
| "kl": 0.098388671875, |
| "learning_rate": 9.33265410086602e-07, |
| "loss": 0.0039, |
| "reward": 0.6651052087545395, |
| "reward_std": 0.9073293209075928, |
| "rewards/accuracy_reward": -0.09687501192092896, |
| "rewards/cosine_rewards": -0.03463773522526026, |
| "rewards/format_reward": 0.796875, |
| "rewards/repetition_rewards": -0.000257108491496183, |
| "step": 262 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 345.109375, |
| "epoch": 0.13397860417727966, |
| "grad_norm": 105.36035227056938, |
| "kl": 0.10546875, |
| "learning_rate": 9.330106979113601e-07, |
| "loss": 0.0042, |
| "reward": 1.6959076523780823, |
| "reward_std": 0.6433850526809692, |
| "rewards/accuracy_reward": 0.7374999523162842, |
| "rewards/cosine_rewards": 0.036790573969483376, |
| "rewards/format_reward": 0.921875, |
| "rewards/repetition_rewards": -0.0002579164138296619, |
| "step": 263 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 225.671875, |
| "epoch": 0.13448802852776362, |
| "grad_norm": 12.269079038415155, |
| "kl": 0.1044921875, |
| "learning_rate": 9.327559857361181e-07, |
| "loss": 0.0042, |
| "reward": 1.3304521441459656, |
| "reward_std": 0.7337057292461395, |
| "rewards/accuracy_reward": 0.40312500298023224, |
| "rewards/cosine_rewards": -0.009734044317156076, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": -0.0004387954395497218, |
| "step": 264 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 191.421875, |
| "epoch": 0.13499745287824758, |
| "grad_norm": 5.0495702848173805, |
| "kl": 0.12451171875, |
| "learning_rate": 9.325012735608761e-07, |
| "loss": 0.005, |
| "reward": 1.5114508867263794, |
| "reward_std": 0.4991532266139984, |
| "rewards/accuracy_reward": 0.6031249910593033, |
| "rewards/cosine_rewards": 0.0021946561755612493, |
| "rewards/format_reward": 0.90625, |
| "rewards/repetition_rewards": -0.0001188212918350473, |
| "step": 265 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 212.890625, |
| "epoch": 0.13550687722873153, |
| "grad_norm": 8.380147213080475, |
| "kl": 0.11376953125, |
| "learning_rate": 9.322465613856342e-07, |
| "loss": 0.0046, |
| "reward": 1.3185867071151733, |
| "reward_std": 0.5081266015768051, |
| "rewards/accuracy_reward": 0.37812499701976776, |
| "rewards/cosine_rewards": 0.0029779861215502024, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": -1.6225338185904548e-05, |
| "step": 266 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 163.4375, |
| "epoch": 0.1360163015792155, |
| "grad_norm": 6.7670554711392725, |
| "kl": 0.1259765625, |
| "learning_rate": 9.319918492103922e-07, |
| "loss": 0.005, |
| "reward": 1.917210876941681, |
| "reward_std": 0.2323581874370575, |
| "rewards/accuracy_reward": 0.96875, |
| "rewards/cosine_rewards": 0.011114767286926508, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": -0.00015385003644041717, |
| "step": 267 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 156.0, |
| "epoch": 0.13652572592969944, |
| "grad_norm": 7.285618988190272, |
| "kl": 0.119873046875, |
| "learning_rate": 9.317371370351502e-07, |
| "loss": 0.0048, |
| "reward": 1.2626032829284668, |
| "reward_std": 0.6921159029006958, |
| "rewards/accuracy_reward": 0.34687499701976776, |
| "rewards/cosine_rewards": -0.006146675441414118, |
| "rewards/format_reward": 0.921875, |
| "rewards/repetition_rewards": 0.0, |
| "step": 268 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 152.953125, |
| "epoch": 0.1370351502801834, |
| "grad_norm": 10.577916333197056, |
| "kl": 0.140625, |
| "learning_rate": 9.314824248599083e-07, |
| "loss": 0.0056, |
| "reward": 1.2036974430084229, |
| "reward_std": 0.5991593599319458, |
| "rewards/accuracy_reward": 0.2968749850988388, |
| "rewards/cosine_rewards": 0.0007108037825673819, |
| "rewards/format_reward": 0.90625, |
| "rewards/repetition_rewards": -0.0001382743357680738, |
| "step": 269 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 158.34375, |
| "epoch": 0.13754457463066735, |
| "grad_norm": 14.293416719573916, |
| "kl": 0.1201171875, |
| "learning_rate": 9.312277126846663e-07, |
| "loss": 0.0048, |
| "reward": 1.2185573279857635, |
| "reward_std": 0.43015679717063904, |
| "rewards/accuracy_reward": 0.24062500149011612, |
| "rewards/cosine_rewards": -0.006263321032747626, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -0.00017934850984602235, |
| "step": 270 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 166.46875, |
| "epoch": 0.1380539989811513, |
| "grad_norm": 9.393354579467495, |
| "kl": 0.1240234375, |
| "learning_rate": 9.309730005094243e-07, |
| "loss": 0.005, |
| "reward": 1.5472444295883179, |
| "reward_std": 0.5279964953660965, |
| "rewards/accuracy_reward": 0.606249988079071, |
| "rewards/cosine_rewards": 0.003494387026876211, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 271 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 149.609375, |
| "epoch": 0.13856342333163527, |
| "grad_norm": 7.200843784037537, |
| "kl": 0.117431640625, |
| "learning_rate": 9.307182883341823e-07, |
| "loss": 0.0047, |
| "reward": 1.3769221901893616, |
| "reward_std": 0.4806235730648041, |
| "rewards/accuracy_reward": 0.40937501937150955, |
| "rewards/cosine_rewards": -0.0011419787188060582, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -6.0797665355494246e-05, |
| "step": 272 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 143.828125, |
| "epoch": 0.1390728476821192, |
| "grad_norm": 11.003082967675637, |
| "kl": 0.18359375, |
| "learning_rate": 9.304635761589404e-07, |
| "loss": 0.0073, |
| "reward": 1.3800683617591858, |
| "reward_std": 0.4095611423254013, |
| "rewards/accuracy_reward": 0.40937498211860657, |
| "rewards/cosine_rewards": 0.0019433526322245598, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": 0.0, |
| "step": 273 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 144.453125, |
| "epoch": 0.13958227203260315, |
| "grad_norm": 6.9562429559567285, |
| "kl": 0.130859375, |
| "learning_rate": 9.302088639836984e-07, |
| "loss": 0.0052, |
| "reward": 1.424567699432373, |
| "reward_std": 0.2551300157792866, |
| "rewards/accuracy_reward": 0.4375, |
| "rewards/cosine_rewards": 0.0028896235453430563, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -0.0001969077275134623, |
| "step": 274 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 149.296875, |
| "epoch": 0.1400916963830871, |
| "grad_norm": 9.330566123712217, |
| "kl": 0.1240234375, |
| "learning_rate": 9.299541518084564e-07, |
| "loss": 0.005, |
| "reward": 1.2650930285453796, |
| "reward_std": 0.42955365777015686, |
| "rewards/accuracy_reward": 0.32499999552965164, |
| "rewards/cosine_rewards": 0.0025930306874215603, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 275 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 207.75, |
| "epoch": 0.14060112073357106, |
| "grad_norm": 8.495983071057866, |
| "kl": 0.11962890625, |
| "learning_rate": 9.296994396332144e-07, |
| "loss": 0.0048, |
| "reward": 1.8627826571464539, |
| "reward_std": 0.2839447557926178, |
| "rewards/accuracy_reward": 0.859375, |
| "rewards/cosine_rewards": 0.019193909130990505, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -0.0001612851265235804, |
| "step": 276 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 162.515625, |
| "epoch": 0.14111054508405502, |
| "grad_norm": 12.855580556594866, |
| "kl": 0.14306640625, |
| "learning_rate": 9.294447274579724e-07, |
| "loss": 0.0057, |
| "reward": 1.5162805318832397, |
| "reward_std": 0.6588033437728882, |
| "rewards/accuracy_reward": 0.6343750059604645, |
| "rewards/cosine_rewards": -0.008702149149030447, |
| "rewards/format_reward": 0.890625, |
| "rewards/repetition_rewards": -1.7208149074576795e-05, |
| "step": 277 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 281.484375, |
| "epoch": 0.14161996943453897, |
| "grad_norm": 10.357622467045976, |
| "kl": 0.102783203125, |
| "learning_rate": 9.291900152827304e-07, |
| "loss": 0.0041, |
| "reward": 1.1203789710998535, |
| "reward_std": 0.6814777851104736, |
| "rewards/accuracy_reward": 0.17812500894069672, |
| "rewards/cosine_rewards": -0.010577938985079527, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": -0.0002930604387074709, |
| "step": 278 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 235.390625, |
| "epoch": 0.14212939378502293, |
| "grad_norm": 23.483530147678458, |
| "kl": 0.114013671875, |
| "learning_rate": 9.289353031074884e-07, |
| "loss": 0.0046, |
| "reward": 1.3522316813468933, |
| "reward_std": 0.28182537853717804, |
| "rewards/accuracy_reward": 0.3812500238418579, |
| "rewards/cosine_rewards": 0.0023158364929258823, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -8.418447396252304e-05, |
| "step": 279 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 271.96875, |
| "epoch": 0.14263881813550688, |
| "grad_norm": 5.804128123317947, |
| "kl": 0.109619140625, |
| "learning_rate": 9.286805909322465e-07, |
| "loss": 0.0044, |
| "reward": 1.2828457355499268, |
| "reward_std": 0.5574119389057159, |
| "rewards/accuracy_reward": 0.3500000163912773, |
| "rewards/cosine_rewards": -0.004654169548302889, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": 0.0, |
| "step": 280 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 346.984375, |
| "epoch": 0.14314824248599084, |
| "grad_norm": 9.062411976412948, |
| "kl": 0.09130859375, |
| "learning_rate": 9.284258787570045e-07, |
| "loss": 0.0037, |
| "reward": 1.9385767579078674, |
| "reward_std": 0.3151838555932045, |
| "rewards/accuracy_reward": 0.9437500238418579, |
| "rewards/cosine_rewards": 0.04203657992184162, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": -0.0003348248792462982, |
| "step": 281 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 395.875, |
| "epoch": 0.1436576668364748, |
| "grad_norm": 8.221976106115973, |
| "kl": 0.104248046875, |
| "learning_rate": 9.281711665817625e-07, |
| "loss": 0.0042, |
| "reward": 1.323907494544983, |
| "reward_std": 0.6098371148109436, |
| "rewards/accuracy_reward": 0.40312501788139343, |
| "rewards/cosine_rewards": 0.014727211673744023, |
| "rewards/format_reward": 0.90625, |
| "rewards/repetition_rewards": -0.00019470852021186147, |
| "step": 282 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 603.21875, |
| "epoch": 0.14416709118695872, |
| "grad_norm": 7.812929807725803, |
| "kl": 0.084228515625, |
| "learning_rate": 9.279164544065206e-07, |
| "loss": 0.0034, |
| "reward": 1.3660696744918823, |
| "reward_std": 0.6207956671714783, |
| "rewards/accuracy_reward": 0.46562500298023224, |
| "rewards/cosine_rewards": -0.005538210505619645, |
| "rewards/format_reward": 0.90625, |
| "rewards/repetition_rewards": -0.0002670584217412397, |
| "step": 283 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 525.6875, |
| "epoch": 0.14467651553744268, |
| "grad_norm": 8.491615227525342, |
| "kl": 0.08056640625, |
| "learning_rate": 9.276617422312786e-07, |
| "loss": 0.0032, |
| "reward": 1.3353699743747711, |
| "reward_std": 0.5946642160415649, |
| "rewards/accuracy_reward": 0.40937500447034836, |
| "rewards/cosine_rewards": -0.02690817415714264, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": -0.0002218634108430706, |
| "step": 284 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 599.984375, |
| "epoch": 0.14518593988792663, |
| "grad_norm": 16.68462964732329, |
| "kl": 0.077880859375, |
| "learning_rate": 9.274070300560366e-07, |
| "loss": 0.0031, |
| "reward": 0.9605185687541962, |
| "reward_std": 0.7793702185153961, |
| "rewards/accuracy_reward": 0.09999999031424522, |
| "rewards/cosine_rewards": -0.06100003980100155, |
| "rewards/format_reward": 0.921875, |
| "rewards/repetition_rewards": -0.0003564156068023294, |
| "step": 285 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 649.3125, |
| "epoch": 0.1456953642384106, |
| "grad_norm": 10.597989534798653, |
| "kl": 0.068115234375, |
| "learning_rate": 9.271523178807946e-07, |
| "loss": 0.0027, |
| "reward": 1.1927469968795776, |
| "reward_std": 1.0099957585334778, |
| "rewards/accuracy_reward": 0.34999997913837433, |
| "rewards/cosine_rewards": -0.04736426845192909, |
| "rewards/format_reward": 0.890625, |
| "rewards/repetition_rewards": -0.0005137407861184329, |
| "step": 286 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 621.234375, |
| "epoch": 0.14620478858889455, |
| "grad_norm": 5.399955402557674, |
| "kl": 0.072265625, |
| "learning_rate": 9.268976057055527e-07, |
| "loss": 0.0029, |
| "reward": 0.821646511554718, |
| "reward_std": 0.9464232325553894, |
| "rewards/accuracy_reward": 0.03749999776482582, |
| "rewards/cosine_rewards": -0.10573448240756989, |
| "rewards/format_reward": 0.890625, |
| "rewards/repetition_rewards": -0.0007440397967002355, |
| "step": 287 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 646.796875, |
| "epoch": 0.1467142129393785, |
| "grad_norm": 5.9108976297695355, |
| "kl": 0.075439453125, |
| "learning_rate": 9.266428935303107e-07, |
| "loss": 0.003, |
| "reward": 1.8053097128868103, |
| "reward_std": 0.5278272330760956, |
| "rewards/accuracy_reward": 0.7749999761581421, |
| "rewards/cosine_rewards": 0.061956772580742836, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -0.0003969733224948868, |
| "step": 288 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 628.484375, |
| "epoch": 0.14722363728986246, |
| "grad_norm": 4.280094122642851, |
| "kl": 0.0692138671875, |
| "learning_rate": 9.263881813550687e-07, |
| "loss": 0.0028, |
| "reward": 0.7580513060092926, |
| "reward_std": 0.9215057492256165, |
| "rewards/accuracy_reward": -0.04062502086162567, |
| "rewards/cosine_rewards": -0.1223737820982933, |
| "rewards/format_reward": 0.921875, |
| "rewards/repetition_rewards": -0.0008249446109402925, |
| "step": 289 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 697.03125, |
| "epoch": 0.1477330616403464, |
| "grad_norm": 4.704795726585343, |
| "kl": 0.068359375, |
| "learning_rate": 9.261334691798267e-07, |
| "loss": 0.0027, |
| "reward": 1.0915009379386902, |
| "reward_std": 0.6004486382007599, |
| "rewards/accuracy_reward": 0.21249999105930328, |
| "rewards/cosine_rewards": -0.05759305879473686, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": -0.0009060115553438663, |
| "step": 290 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 739.375, |
| "epoch": 0.14824248599083037, |
| "grad_norm": 5.468968593755717, |
| "kl": 0.065185546875, |
| "learning_rate": 9.258787570045847e-07, |
| "loss": 0.0026, |
| "reward": 1.328648567199707, |
| "reward_std": 0.8502229452133179, |
| "rewards/accuracy_reward": 0.40312500298023224, |
| "rewards/cosine_rewards": -0.027191368862986565, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": -0.00041003923979587853, |
| "step": 291 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 800.390625, |
| "epoch": 0.14875191034131433, |
| "grad_norm": 2.644913201802289, |
| "kl": 0.07861328125, |
| "learning_rate": 9.256240448293427e-07, |
| "loss": 0.0031, |
| "reward": 1.5775163769721985, |
| "reward_std": 0.6978716552257538, |
| "rewards/accuracy_reward": 0.6562500149011612, |
| "rewards/cosine_rewards": 0.031187113374471664, |
| "rewards/format_reward": 0.890625, |
| "rewards/repetition_rewards": -0.0005458263913169503, |
| "step": 292 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 969.328125, |
| "epoch": 0.14926133469179828, |
| "grad_norm": 3.858694298808609, |
| "kl": 0.0548095703125, |
| "learning_rate": 9.253693326541008e-07, |
| "loss": 0.0022, |
| "reward": 0.39561687409877777, |
| "reward_std": 1.1356619894504547, |
| "rewards/accuracy_reward": -0.1625000238418579, |
| "rewards/cosine_rewards": -0.23805859684944153, |
| "rewards/format_reward": 0.796875, |
| "rewards/repetition_rewards": -0.0006995665607973933, |
| "step": 293 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1044.703125, |
| "epoch": 0.1497707590422822, |
| "grad_norm": 2.0660275837501643, |
| "kl": 0.0902099609375, |
| "learning_rate": 9.251146204788588e-07, |
| "loss": 0.0036, |
| "reward": 1.0626700818538666, |
| "reward_std": 1.1662874221801758, |
| "rewards/accuracy_reward": 0.3531249985098839, |
| "rewards/cosine_rewards": -0.055325835943222046, |
| "rewards/format_reward": 0.765625, |
| "rewards/repetition_rewards": -0.0007540385995525867, |
| "step": 294 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 934.28125, |
| "epoch": 0.15028018339276616, |
| "grad_norm": 7.548230617974183, |
| "kl": 0.0538330078125, |
| "learning_rate": 9.248599083036168e-07, |
| "loss": 0.0022, |
| "reward": 1.2535955309867859, |
| "reward_std": 1.0525287985801697, |
| "rewards/accuracy_reward": 0.3750000223517418, |
| "rewards/cosine_rewards": -0.04287016252055764, |
| "rewards/format_reward": 0.921875, |
| "rewards/repetition_rewards": -0.00040922046173363924, |
| "step": 295 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 790.09375, |
| "epoch": 0.15078960774325012, |
| "grad_norm": 3.7401563979919654, |
| "kl": 0.0584716796875, |
| "learning_rate": 9.246051961283748e-07, |
| "loss": 0.0023, |
| "reward": 1.1489249467849731, |
| "reward_std": 0.5376773178577423, |
| "rewards/accuracy_reward": 0.2937499899417162, |
| "rewards/cosine_rewards": -0.08189126010984182, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": -0.0004337812424637377, |
| "step": 296 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 836.375, |
| "epoch": 0.15129903209373408, |
| "grad_norm": 2.8153067499507105, |
| "kl": 0.0618896484375, |
| "learning_rate": 9.243504839531329e-07, |
| "loss": 0.0025, |
| "reward": 1.3525272011756897, |
| "reward_std": 0.8126451969146729, |
| "rewards/accuracy_reward": 0.4906250238418579, |
| "rewards/cosine_rewards": -0.012518584728240967, |
| "rewards/format_reward": 0.875, |
| "rewards/repetition_rewards": -0.000579186889808625, |
| "step": 297 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 890.625, |
| "epoch": 0.15180845644421803, |
| "grad_norm": 5.316949650260697, |
| "kl": 0.0552978515625, |
| "learning_rate": 9.240957717778909e-07, |
| "loss": 0.0022, |
| "reward": 1.2640092372894287, |
| "reward_std": 0.8870376944541931, |
| "rewards/accuracy_reward": 0.4062499850988388, |
| "rewards/cosine_rewards": -0.0009442958980798721, |
| "rewards/format_reward": 0.859375, |
| "rewards/repetition_rewards": -0.0006714609917253256, |
| "step": 298 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 813.953125, |
| "epoch": 0.152317880794702, |
| "grad_norm": 3.8825478721674953, |
| "kl": 0.0574951171875, |
| "learning_rate": 9.23841059602649e-07, |
| "loss": 0.0023, |
| "reward": 1.2717376947402954, |
| "reward_std": 0.830648809671402, |
| "rewards/accuracy_reward": 0.4375, |
| "rewards/cosine_rewards": -0.03994514420628548, |
| "rewards/format_reward": 0.875, |
| "rewards/repetition_rewards": -0.0008170758956111968, |
| "step": 299 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 752.140625, |
| "epoch": 0.15282730514518594, |
| "grad_norm": 5.06920521582769, |
| "kl": 0.059814453125, |
| "learning_rate": 9.235863474274071e-07, |
| "loss": 0.0024, |
| "reward": 1.1217154264450073, |
| "reward_std": 0.8524642586708069, |
| "rewards/accuracy_reward": 0.24062498286366463, |
| "rewards/cosine_rewards": -0.04011305421590805, |
| "rewards/format_reward": 0.921875, |
| "rewards/repetition_rewards": -0.0006714656192343682, |
| "step": 300 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 684.625, |
| "epoch": 0.1533367294956699, |
| "grad_norm": 8.555507127767159, |
| "kl": 0.0672607421875, |
| "learning_rate": 9.233316352521651e-07, |
| "loss": 0.0027, |
| "reward": 1.1471417546272278, |
| "reward_std": 0.7909112870693207, |
| "rewards/accuracy_reward": 0.2656249962747097, |
| "rewards/cosine_rewards": -0.039835451170802116, |
| "rewards/format_reward": 0.921875, |
| "rewards/repetition_rewards": -0.0005227623041719198, |
| "step": 301 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 667.78125, |
| "epoch": 0.15384615384615385, |
| "grad_norm": 2.9040824114504877, |
| "kl": 0.064697265625, |
| "learning_rate": 9.230769230769231e-07, |
| "loss": 0.0026, |
| "reward": 0.9261243343353271, |
| "reward_std": 0.668161928653717, |
| "rewards/accuracy_reward": 0.1281249988824129, |
| "rewards/cosine_rewards": -0.06077958270907402, |
| "rewards/format_reward": 0.859375, |
| "rewards/repetition_rewards": -0.0005960852140560746, |
| "step": 302 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 702.484375, |
| "epoch": 0.1543555781966378, |
| "grad_norm": 4.4461209381275655, |
| "kl": 0.06298828125, |
| "learning_rate": 9.228222109016812e-07, |
| "loss": 0.0025, |
| "reward": 1.506935715675354, |
| "reward_std": 0.6653757691383362, |
| "rewards/accuracy_reward": 0.5468749850988388, |
| "rewards/cosine_rewards": 0.03871871158480644, |
| "rewards/format_reward": 0.921875, |
| "rewards/repetition_rewards": -0.0005329845298547298, |
| "step": 303 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 633.140625, |
| "epoch": 0.15486500254712177, |
| "grad_norm": 3.9254091150548933, |
| "kl": 0.069091796875, |
| "learning_rate": 9.225674987264391e-07, |
| "loss": 0.0028, |
| "reward": 1.3886016011238098, |
| "reward_std": 0.9017740190029144, |
| "rewards/accuracy_reward": 0.5749999731779099, |
| "rewards/cosine_rewards": -0.02933959849178791, |
| "rewards/format_reward": 0.84375, |
| "rewards/repetition_rewards": -0.000808820070233196, |
| "step": 304 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 648.625, |
| "epoch": 0.1553744268976057, |
| "grad_norm": 6.070022774209878, |
| "kl": 0.068115234375, |
| "learning_rate": 9.223127865511971e-07, |
| "loss": 0.0027, |
| "reward": 1.6925800442695618, |
| "reward_std": 0.6231902837753296, |
| "rewards/accuracy_reward": 0.6625000238418579, |
| "rewards/cosine_rewards": 0.06164960749447346, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -0.0003195497556589544, |
| "step": 305 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 614.15625, |
| "epoch": 0.15588385124808965, |
| "grad_norm": 11.091865062468658, |
| "kl": 0.317138671875, |
| "learning_rate": 9.220580743759551e-07, |
| "loss": 0.0127, |
| "reward": 1.5423057079315186, |
| "reward_std": 0.3847469687461853, |
| "rewards/accuracy_reward": 0.5468749962747097, |
| "rewards/cosine_rewards": 0.05880427733063698, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": -0.0008736126183066517, |
| "step": 306 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 612.578125, |
| "epoch": 0.1563932755985736, |
| "grad_norm": 3.103459103182676, |
| "kl": 0.0673828125, |
| "learning_rate": 9.218033622007132e-07, |
| "loss": 0.0027, |
| "reward": 1.6744784712791443, |
| "reward_std": 0.659433513879776, |
| "rewards/accuracy_reward": 0.6875, |
| "rewards/cosine_rewards": 0.06587037723511457, |
| "rewards/format_reward": 0.921875, |
| "rewards/repetition_rewards": -0.0007668640464544296, |
| "step": 307 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 657.265625, |
| "epoch": 0.15690269994905756, |
| "grad_norm": 4.50781660421839, |
| "kl": 0.068115234375, |
| "learning_rate": 9.215486500254712e-07, |
| "loss": 0.0027, |
| "reward": 1.145881563425064, |
| "reward_std": 1.0458006858825684, |
| "rewards/accuracy_reward": 0.34062499180436134, |
| "rewards/cosine_rewards": -0.03727734461426735, |
| "rewards/format_reward": 0.84375, |
| "rewards/repetition_rewards": -0.0012161528575234115, |
| "step": 308 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 771.5625, |
| "epoch": 0.15741212429954152, |
| "grad_norm": 6.920330329994176, |
| "kl": 0.064208984375, |
| "learning_rate": 9.212939378502292e-07, |
| "loss": 0.0026, |
| "reward": 0.8615269958972931, |
| "reward_std": 0.9165626764297485, |
| "rewards/accuracy_reward": 0.140625, |
| "rewards/cosine_rewards": -0.07544910162687302, |
| "rewards/format_reward": 0.796875, |
| "rewards/repetition_rewards": -0.0005239159800112247, |
| "step": 309 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 758.96875, |
| "epoch": 0.15792154865002547, |
| "grad_norm": 11.706103376756111, |
| "kl": 0.056396484375, |
| "learning_rate": 9.210392256749873e-07, |
| "loss": 0.0023, |
| "reward": 1.567901074886322, |
| "reward_std": 1.1157508492469788, |
| "rewards/accuracy_reward": 0.6437499821186066, |
| "rewards/cosine_rewards": 0.08178849518299103, |
| "rewards/format_reward": 0.84375, |
| "rewards/repetition_rewards": -0.0013874000869691372, |
| "step": 310 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 694.390625, |
| "epoch": 0.15843097300050943, |
| "grad_norm": 4.42128730127276, |
| "kl": 0.062255859375, |
| "learning_rate": 9.207845134997453e-07, |
| "loss": 0.0025, |
| "reward": 0.944963201880455, |
| "reward_std": 0.9125352203845978, |
| "rewards/accuracy_reward": 0.16249998658895493, |
| "rewards/cosine_rewards": -0.04496639594435692, |
| "rewards/format_reward": 0.828125, |
| "rewards/repetition_rewards": -0.0006953877746127546, |
| "step": 311 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 812.46875, |
| "epoch": 0.15894039735099338, |
| "grad_norm": 5.888648418898334, |
| "kl": 0.0587158203125, |
| "learning_rate": 9.205298013245033e-07, |
| "loss": 0.0023, |
| "reward": 0.6825668215751648, |
| "reward_std": 1.0514086484909058, |
| "rewards/accuracy_reward": 0.04999999701976776, |
| "rewards/cosine_rewards": -0.13240730948746204, |
| "rewards/format_reward": 0.765625, |
| "rewards/repetition_rewards": -0.0006508340884465724, |
| "step": 312 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 703.703125, |
| "epoch": 0.15944982170147734, |
| "grad_norm": 5.201587660434957, |
| "kl": 0.0626220703125, |
| "learning_rate": 9.202750891492613e-07, |
| "loss": 0.0025, |
| "reward": 0.849999725818634, |
| "reward_std": 1.2490254640579224, |
| "rewards/accuracy_reward": 0.16249999590218067, |
| "rewards/cosine_rewards": -0.04634671099483967, |
| "rewards/format_reward": 0.734375, |
| "rewards/repetition_rewards": -0.0005285786173772067, |
| "step": 313 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 732.84375, |
| "epoch": 0.1599592460519613, |
| "grad_norm": 41.79369545195822, |
| "kl": 0.0654296875, |
| "learning_rate": 9.200203769740194e-07, |
| "loss": 0.0026, |
| "reward": 1.359117031097412, |
| "reward_std": 1.1281075477600098, |
| "rewards/accuracy_reward": 0.49687501788139343, |
| "rewards/cosine_rewards": 0.06599474605172873, |
| "rewards/format_reward": 0.796875, |
| "rewards/repetition_rewards": -0.0006276974454522133, |
| "step": 314 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 633.609375, |
| "epoch": 0.16046867040244522, |
| "grad_norm": 5.659472242303819, |
| "kl": 0.090087890625, |
| "learning_rate": 9.197656647987774e-07, |
| "loss": 0.0036, |
| "reward": 1.149334043264389, |
| "reward_std": 1.1551178693771362, |
| "rewards/accuracy_reward": 0.3593749962747097, |
| "rewards/cosine_rewards": 0.025284748524427414, |
| "rewards/format_reward": 0.765625, |
| "rewards/repetition_rewards": -0.0009507373906672001, |
| "step": 315 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 704.09375, |
| "epoch": 0.16097809475292918, |
| "grad_norm": 5.702114455603425, |
| "kl": 0.071044921875, |
| "learning_rate": 9.195109526235354e-07, |
| "loss": 0.0028, |
| "reward": 1.3667227029800415, |
| "reward_std": 0.6237545907497406, |
| "rewards/accuracy_reward": 0.4031249713152647, |
| "rewards/cosine_rewards": 0.01146969199180603, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": -0.0009969472303055227, |
| "step": 316 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 615.984375, |
| "epoch": 0.16148751910341314, |
| "grad_norm": 7.41926766137556, |
| "kl": 0.072998046875, |
| "learning_rate": 9.192562404482935e-07, |
| "loss": 0.0029, |
| "reward": 1.2655977010726929, |
| "reward_std": 0.7071200311183929, |
| "rewards/accuracy_reward": 0.37187498807907104, |
| "rewards/cosine_rewards": -0.0119027029722929, |
| "rewards/format_reward": 0.90625, |
| "rewards/repetition_rewards": -0.00062458252068609, |
| "step": 317 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 655.65625, |
| "epoch": 0.1619969434538971, |
| "grad_norm": 6.476802877604566, |
| "kl": 0.072265625, |
| "learning_rate": 9.190015282730514e-07, |
| "loss": 0.0029, |
| "reward": 1.4926868677139282, |
| "reward_std": 0.5651115030050278, |
| "rewards/accuracy_reward": 0.4906250089406967, |
| "rewards/cosine_rewards": 0.05030408315360546, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": -0.0013672530185431242, |
| "step": 318 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 694.4375, |
| "epoch": 0.16250636780438105, |
| "grad_norm": 5.651262358287829, |
| "kl": 0.078125, |
| "learning_rate": 9.187468160978094e-07, |
| "loss": 0.0031, |
| "reward": 1.6467930674552917, |
| "reward_std": 0.6130897700786591, |
| "rewards/accuracy_reward": 0.6343749761581421, |
| "rewards/cosine_rewards": 0.060116952285170555, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": -0.0008239042945206165, |
| "step": 319 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 627.1875, |
| "epoch": 0.163015792154865, |
| "grad_norm": 8.752414113774137, |
| "kl": 0.087890625, |
| "learning_rate": 9.184921039225674e-07, |
| "loss": 0.0035, |
| "reward": 1.2824658155441284, |
| "reward_std": 0.6804981231689453, |
| "rewards/accuracy_reward": 0.4281250089406967, |
| "rewards/cosine_rewards": -0.004211767576634884, |
| "rewards/format_reward": 0.859375, |
| "rewards/repetition_rewards": -0.0008223777404054999, |
| "step": 320 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 647.421875, |
| "epoch": 0.16352521650534896, |
| "grad_norm": 24.555397071153298, |
| "kl": 0.10791015625, |
| "learning_rate": 9.182373917473255e-07, |
| "loss": 0.0043, |
| "reward": 1.5703404545783997, |
| "reward_std": 0.6466428339481354, |
| "rewards/accuracy_reward": 0.5781249850988388, |
| "rewards/cosine_rewards": 0.023975687101483345, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -0.0005103159819555003, |
| "step": 321 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 679.453125, |
| "epoch": 0.1640346408558329, |
| "grad_norm": 10.65571265954387, |
| "kl": 0.0751953125, |
| "learning_rate": 9.179826795720835e-07, |
| "loss": 0.003, |
| "reward": 1.6510714292526245, |
| "reward_std": 1.0072646141052246, |
| "rewards/accuracy_reward": 0.7062499523162842, |
| "rewards/cosine_rewards": 0.0703657679259777, |
| "rewards/format_reward": 0.875, |
| "rewards/repetition_rewards": -0.0005443187110358849, |
| "step": 322 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 815.84375, |
| "epoch": 0.16454406520631687, |
| "grad_norm": 3.0291641455139042, |
| "kl": 0.0577392578125, |
| "learning_rate": 9.177279673968415e-07, |
| "loss": 0.0023, |
| "reward": 0.7281904220581055, |
| "reward_std": 0.7364227771759033, |
| "rewards/accuracy_reward": -0.07187500596046448, |
| "rewards/cosine_rewards": -0.15170371532440186, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": -0.0013558552600443363, |
| "step": 323 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 644.84375, |
| "epoch": 0.16505348955680083, |
| "grad_norm": 4.950572350987556, |
| "kl": 0.081787109375, |
| "learning_rate": 9.174732552215996e-07, |
| "loss": 0.0033, |
| "reward": 1.5456467270851135, |
| "reward_std": 0.3990190625190735, |
| "rewards/accuracy_reward": 0.5750000178813934, |
| "rewards/cosine_rewards": 0.03384638950228691, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": -0.000699635551427491, |
| "step": 324 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 692.96875, |
| "epoch": 0.16556291390728478, |
| "grad_norm": 5.615028773473959, |
| "kl": 0.0675048828125, |
| "learning_rate": 9.172185430463576e-07, |
| "loss": 0.0027, |
| "reward": 1.4643962979316711, |
| "reward_std": 0.538501039147377, |
| "rewards/accuracy_reward": 0.4906250089406967, |
| "rewards/cosine_rewards": 0.021510865539312363, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": -0.000864640751387924, |
| "step": 325 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 748.71875, |
| "epoch": 0.1660723382577687, |
| "grad_norm": 12.207464085563803, |
| "kl": 0.071533203125, |
| "learning_rate": 9.169638308711156e-07, |
| "loss": 0.0029, |
| "reward": 1.1908642947673798, |
| "reward_std": 0.8150831162929535, |
| "rewards/accuracy_reward": 0.3156250100582838, |
| "rewards/cosine_rewards": 0.0008599106222391129, |
| "rewards/format_reward": 0.875, |
| "rewards/repetition_rewards": -0.000620643695583567, |
| "step": 326 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 692.828125, |
| "epoch": 0.16658176260825266, |
| "grad_norm": 4.633110149052728, |
| "kl": 0.067626953125, |
| "learning_rate": 9.167091186958737e-07, |
| "loss": 0.0027, |
| "reward": 1.3975687623023987, |
| "reward_std": 0.6602180898189545, |
| "rewards/accuracy_reward": 0.40937499701976776, |
| "rewards/cosine_rewards": 0.020020989701151848, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -0.0005772198055638, |
| "step": 327 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 886.234375, |
| "epoch": 0.16709118695873662, |
| "grad_norm": 11.480819900317456, |
| "kl": 0.0567626953125, |
| "learning_rate": 9.164544065206317e-07, |
| "loss": 0.0023, |
| "reward": 1.3322511315345764, |
| "reward_std": 0.7808408439159393, |
| "rewards/accuracy_reward": 0.3812499865889549, |
| "rewards/cosine_rewards": -0.0007541030645370483, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": -0.0013698027469217777, |
| "step": 328 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 869.515625, |
| "epoch": 0.16760061130922058, |
| "grad_norm": 7.997306620734284, |
| "kl": 0.0577392578125, |
| "learning_rate": 9.161996943453897e-07, |
| "loss": 0.0023, |
| "reward": 1.1871361136436462, |
| "reward_std": 0.9155566692352295, |
| "rewards/accuracy_reward": 0.3218750059604645, |
| "rewards/cosine_rewards": -0.0396097619086504, |
| "rewards/format_reward": 0.90625, |
| "rewards/repetition_rewards": -0.0013791794190183282, |
| "step": 329 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 879.125, |
| "epoch": 0.16811003565970453, |
| "grad_norm": 3.8086916601970175, |
| "kl": 0.05810546875, |
| "learning_rate": 9.159449821701477e-07, |
| "loss": 0.0023, |
| "reward": 1.3816418051719666, |
| "reward_std": 0.8457719385623932, |
| "rewards/accuracy_reward": 0.43437500298023224, |
| "rewards/cosine_rewards": 0.026831649709492922, |
| "rewards/format_reward": 0.921875, |
| "rewards/repetition_rewards": -0.0014398820349015296, |
| "step": 330 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1077.125, |
| "epoch": 0.1686194600101885, |
| "grad_norm": 3.3635095209387, |
| "kl": 0.05029296875, |
| "learning_rate": 9.156902699949058e-07, |
| "loss": 0.002, |
| "reward": 1.4233552813529968, |
| "reward_std": 0.8923040926456451, |
| "rewards/accuracy_reward": 0.5718750357627869, |
| "rewards/cosine_rewards": 0.05618499033153057, |
| "rewards/format_reward": 0.796875, |
| "rewards/repetition_rewards": -0.0015796992811374366, |
| "step": 331 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1025.625, |
| "epoch": 0.16912888436067244, |
| "grad_norm": 2.6795836472345886, |
| "kl": 0.053955078125, |
| "learning_rate": 9.154355578196637e-07, |
| "loss": 0.0022, |
| "reward": 1.5009884238243103, |
| "reward_std": 0.7616147696971893, |
| "rewards/accuracy_reward": 0.46562496945261955, |
| "rewards/cosine_rewards": 0.08306753821671009, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": -0.0008290903642773628, |
| "step": 332 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1130.390625, |
| "epoch": 0.1696383087111564, |
| "grad_norm": 2.8897374461041068, |
| "kl": 0.05615234375, |
| "learning_rate": 9.151808456444217e-07, |
| "loss": 0.0022, |
| "reward": 0.9568201899528503, |
| "reward_std": 0.883324146270752, |
| "rewards/accuracy_reward": 0.18437499552965164, |
| "rewards/cosine_rewards": -0.14675537310540676, |
| "rewards/format_reward": 0.921875, |
| "rewards/repetition_rewards": -0.0026744193164631724, |
| "step": 333 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1140.796875, |
| "epoch": 0.17014773306164035, |
| "grad_norm": 3.24780768226595, |
| "kl": 0.053955078125, |
| "learning_rate": 9.149261334691798e-07, |
| "loss": 0.0022, |
| "reward": 0.4763996750116348, |
| "reward_std": 1.3050541877746582, |
| "rewards/accuracy_reward": -0.07187501713633537, |
| "rewards/cosine_rewards": -0.26269275695085526, |
| "rewards/format_reward": 0.8125, |
| "rewards/repetition_rewards": -0.0015325736021623015, |
| "step": 334 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1162.21875, |
| "epoch": 0.1706571574121243, |
| "grad_norm": 7.96370989509143, |
| "kl": 0.0509033203125, |
| "learning_rate": 9.146714212939378e-07, |
| "loss": 0.002, |
| "reward": 1.0168579816818237, |
| "reward_std": 1.0622537732124329, |
| "rewards/accuracy_reward": 0.23749998211860657, |
| "rewards/cosine_rewards": -0.06289426982402802, |
| "rewards/format_reward": 0.84375, |
| "rewards/repetition_rewards": -0.0014976929523982108, |
| "step": 335 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1223.59375, |
| "epoch": 0.17116658176260827, |
| "grad_norm": 5.774523253643062, |
| "kl": 0.083251953125, |
| "learning_rate": 9.144167091186958e-07, |
| "loss": 0.0033, |
| "reward": 0.9260146915912628, |
| "reward_std": 1.3471828699111938, |
| "rewards/accuracy_reward": 0.26249998807907104, |
| "rewards/cosine_rewards": -0.11641103774309158, |
| "rewards/format_reward": 0.78125, |
| "rewards/repetition_rewards": -0.0013242715504020452, |
| "step": 336 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1069.484375, |
| "epoch": 0.1716760061130922, |
| "grad_norm": 7.732047491992381, |
| "kl": 0.0555419921875, |
| "learning_rate": 9.141619969434538e-07, |
| "loss": 0.0022, |
| "reward": 1.0389263331890106, |
| "reward_std": 0.9250738620758057, |
| "rewards/accuracy_reward": 0.20937499403953552, |
| "rewards/cosine_rewards": -0.09034883230924606, |
| "rewards/format_reward": 0.921875, |
| "rewards/repetition_rewards": -0.001974849379621446, |
| "step": 337 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 846.234375, |
| "epoch": 0.17218543046357615, |
| "grad_norm": 6.146520612031918, |
| "kl": 0.06689453125, |
| "learning_rate": 9.139072847682119e-07, |
| "loss": 0.0027, |
| "reward": 1.5287657380104065, |
| "reward_std": 0.7281034886837006, |
| "rewards/accuracy_reward": 0.5218749940395355, |
| "rewards/cosine_rewards": 0.055092147551476955, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": -0.0013264745939522982, |
| "step": 338 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 878.59375, |
| "epoch": 0.1726948548140601, |
| "grad_norm": 5.859040533770109, |
| "kl": 0.059814453125, |
| "learning_rate": 9.136525725929699e-07, |
| "loss": 0.0024, |
| "reward": 1.309591829776764, |
| "reward_std": 0.8282720148563385, |
| "rewards/accuracy_reward": 0.3781249839812517, |
| "rewards/cosine_rewards": 0.02607971802353859, |
| "rewards/format_reward": 0.90625, |
| "rewards/repetition_rewards": -0.0008628710638731718, |
| "step": 339 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 706.234375, |
| "epoch": 0.17320427916454406, |
| "grad_norm": 4.068095402441544, |
| "kl": 0.066162109375, |
| "learning_rate": 9.133978604177279e-07, |
| "loss": 0.0026, |
| "reward": 1.1101016998291016, |
| "reward_std": 0.7019257247447968, |
| "rewards/accuracy_reward": 0.20624998956918716, |
| "rewards/cosine_rewards": -0.03270102944225073, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": -0.0009472573874518275, |
| "step": 340 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 782.09375, |
| "epoch": 0.17371370351502802, |
| "grad_norm": 8.092723935778833, |
| "kl": 0.07080078125, |
| "learning_rate": 9.13143148242486e-07, |
| "loss": 0.0028, |
| "reward": 1.3624014258384705, |
| "reward_std": 0.6876442432403564, |
| "rewards/accuracy_reward": 0.40937499701976776, |
| "rewards/cosine_rewards": 0.0012194328010082245, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": -0.0013180217938497663, |
| "step": 341 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 725.109375, |
| "epoch": 0.17422312786551197, |
| "grad_norm": 8.756999335905311, |
| "kl": 0.130126953125, |
| "learning_rate": 9.12888436067244e-07, |
| "loss": 0.0052, |
| "reward": 1.1084296703338623, |
| "reward_std": 1.0551597476005554, |
| "rewards/accuracy_reward": 0.2343750037252903, |
| "rewards/cosine_rewards": -0.062263866886496544, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": -0.0011814486351795495, |
| "step": 342 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 664.5625, |
| "epoch": 0.17473255221599593, |
| "grad_norm": 4.670760280414413, |
| "kl": 0.07275390625, |
| "learning_rate": 9.12633723892002e-07, |
| "loss": 0.0029, |
| "reward": 1.379169523715973, |
| "reward_std": 0.6884946823120117, |
| "rewards/accuracy_reward": 0.40937499701976776, |
| "rewards/cosine_rewards": -0.014050468802452087, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -0.0005300141347106546, |
| "step": 343 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 667.359375, |
| "epoch": 0.17524197656647988, |
| "grad_norm": 27.098795999332967, |
| "kl": 0.08056640625, |
| "learning_rate": 9.123790117167601e-07, |
| "loss": 0.0032, |
| "reward": 1.6130830645561218, |
| "reward_std": 0.44565099477767944, |
| "rewards/accuracy_reward": 0.5781249701976776, |
| "rewards/cosine_rewards": 0.051279583014547825, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -0.0006965193606447428, |
| "step": 344 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 654.65625, |
| "epoch": 0.17575140091696384, |
| "grad_norm": 10.133659329005896, |
| "kl": 0.075439453125, |
| "learning_rate": 9.121242995415181e-07, |
| "loss": 0.003, |
| "reward": 1.6888669729232788, |
| "reward_std": 0.506424754858017, |
| "rewards/accuracy_reward": 0.690625011920929, |
| "rewards/cosine_rewards": 0.06190674379467964, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": -0.0011647465871647, |
| "step": 345 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 647.6875, |
| "epoch": 0.1762608252674478, |
| "grad_norm": 5.129606334927334, |
| "kl": 0.07958984375, |
| "learning_rate": 9.11869587366276e-07, |
| "loss": 0.0032, |
| "reward": 1.2593636512756348, |
| "reward_std": 0.41098763048648834, |
| "rewards/accuracy_reward": 0.296875, |
| "rewards/cosine_rewards": -0.005569446831941605, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -0.0006919201114214957, |
| "step": 346 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 696.46875, |
| "epoch": 0.17677024961793172, |
| "grad_norm": 11.616711482358948, |
| "kl": 0.074462890625, |
| "learning_rate": 9.11614875191034e-07, |
| "loss": 0.003, |
| "reward": 1.4469356536865234, |
| "reward_std": 0.6090122163295746, |
| "rewards/accuracy_reward": 0.46562500298023224, |
| "rewards/cosine_rewards": -0.0018481542356312275, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -0.001216164615470916, |
| "step": 347 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 660.0, |
| "epoch": 0.17727967396841568, |
| "grad_norm": 16.70982931235053, |
| "kl": 0.092041015625, |
| "learning_rate": 9.113601630157921e-07, |
| "loss": 0.0037, |
| "reward": 1.3860605359077454, |
| "reward_std": 0.5821886360645294, |
| "rewards/accuracy_reward": 0.40625, |
| "rewards/cosine_rewards": 0.01206381805241108, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -0.0010032225982286036, |
| "step": 348 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 819.25, |
| "epoch": 0.17778909831889964, |
| "grad_norm": 10.312325223777075, |
| "kl": 0.0694580078125, |
| "learning_rate": 9.111054508405501e-07, |
| "loss": 0.0028, |
| "reward": 1.3597615957260132, |
| "reward_std": 0.5677385032176971, |
| "rewards/accuracy_reward": 0.4375000074505806, |
| "rewards/cosine_rewards": 0.0019306838512420654, |
| "rewards/format_reward": 0.921875, |
| "rewards/repetition_rewards": -0.0015441215364262462, |
| "step": 349 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 816.125, |
| "epoch": 0.1782985226693836, |
| "grad_norm": 3.8656926374469416, |
| "kl": 0.07080078125, |
| "learning_rate": 9.108507386653081e-07, |
| "loss": 0.0028, |
| "reward": 1.1428874135017395, |
| "reward_std": 0.40452495217323303, |
| "rewards/accuracy_reward": 0.21249999105930328, |
| "rewards/cosine_rewards": -0.05335182696580887, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -0.0006357444362947717, |
| "step": 350 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 839.09375, |
| "epoch": 0.17880794701986755, |
| "grad_norm": 10.545025641089767, |
| "kl": 0.062744140625, |
| "learning_rate": 9.105960264900662e-07, |
| "loss": 0.0025, |
| "reward": 1.440682828426361, |
| "reward_std": 0.7122917473316193, |
| "rewards/accuracy_reward": 0.4375, |
| "rewards/cosine_rewards": 0.004494791850447655, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": -0.001311894680839032, |
| "step": 351 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 745.34375, |
| "epoch": 0.1793173713703515, |
| "grad_norm": 5.236449549563, |
| "kl": 0.081787109375, |
| "learning_rate": 9.103413143148242e-07, |
| "loss": 0.0033, |
| "reward": 1.7106852531433105, |
| "reward_std": 0.4475601017475128, |
| "rewards/accuracy_reward": 0.6875, |
| "rewards/cosine_rewards": 0.07085046917200089, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": -0.0007901439967099577, |
| "step": 352 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 782.65625, |
| "epoch": 0.17982679572083546, |
| "grad_norm": 4.40390803368756, |
| "kl": 0.07568359375, |
| "learning_rate": 9.100866021395822e-07, |
| "loss": 0.003, |
| "reward": 1.321226179599762, |
| "reward_std": 0.5729265064001083, |
| "rewards/accuracy_reward": 0.3812500238418579, |
| "rewards/cosine_rewards": -0.04316529631614685, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -0.001233478484209627, |
| "step": 353 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 911.4375, |
| "epoch": 0.18033622007131941, |
| "grad_norm": 4.585368026245459, |
| "kl": 0.083740234375, |
| "learning_rate": 9.098318899643402e-07, |
| "loss": 0.0034, |
| "reward": 1.2610972821712494, |
| "reward_std": 0.5936008393764496, |
| "rewards/accuracy_reward": 0.3812499828636646, |
| "rewards/cosine_rewards": -0.02527322620153427, |
| "rewards/format_reward": 0.90625, |
| "rewards/repetition_rewards": -0.0011294231517240405, |
| "step": 354 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 961.59375, |
| "epoch": 0.18084564442180337, |
| "grad_norm": 7.408796362493787, |
| "kl": 0.0693359375, |
| "learning_rate": 9.095771777890983e-07, |
| "loss": 0.0028, |
| "reward": 1.2509925812482834, |
| "reward_std": 0.5566798448562622, |
| "rewards/accuracy_reward": 0.3499999940395355, |
| "rewards/cosine_rewards": -0.034961797297000885, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": -0.001545542269013822, |
| "step": 355 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 875.421875, |
| "epoch": 0.18135506877228733, |
| "grad_norm": 6.46350391356739, |
| "kl": 0.08251953125, |
| "learning_rate": 9.093224656138563e-07, |
| "loss": 0.0033, |
| "reward": 1.1181039810180664, |
| "reward_std": 0.674926146864891, |
| "rewards/accuracy_reward": 0.23749998956918716, |
| "rewards/cosine_rewards": -0.05567748658359051, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": -0.0012184783699922264, |
| "step": 356 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1032.265625, |
| "epoch": 0.18186449312277128, |
| "grad_norm": 6.831510020206218, |
| "kl": 0.0609130859375, |
| "learning_rate": 9.090677534386143e-07, |
| "loss": 0.0024, |
| "reward": 1.559360921382904, |
| "reward_std": 0.6407117247581482, |
| "rewards/accuracy_reward": 0.518750011920929, |
| "rewards/cosine_rewards": 0.057725198566913605, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -0.0014893330517224967, |
| "step": 357 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1343.859375, |
| "epoch": 0.1823739174732552, |
| "grad_norm": 4.523279934522272, |
| "kl": 0.05419921875, |
| "learning_rate": 9.088130412633724e-07, |
| "loss": 0.0022, |
| "reward": 1.3108936548233032, |
| "reward_std": 1.3749122023582458, |
| "rewards/accuracy_reward": 0.4843749850988388, |
| "rewards/cosine_rewards": -0.015468426048755646, |
| "rewards/format_reward": 0.84375, |
| "rewards/repetition_rewards": -0.0017629386857151985, |
| "step": 358 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1283.875, |
| "epoch": 0.18288334182373917, |
| "grad_norm": 2.9278358224956733, |
| "kl": 0.046875, |
| "learning_rate": 9.085583290881304e-07, |
| "loss": 0.0019, |
| "reward": 0.899000346660614, |
| "reward_std": 1.2357721328735352, |
| "rewards/accuracy_reward": 0.20000001043081284, |
| "rewards/cosine_rewards": -0.12764177471399307, |
| "rewards/format_reward": 0.828125, |
| "rewards/repetition_rewards": -0.0014829274150542915, |
| "step": 359 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1351.765625, |
| "epoch": 0.18339276617422312, |
| "grad_norm": 5.747269025294299, |
| "kl": 0.05029296875, |
| "learning_rate": 9.083036169128883e-07, |
| "loss": 0.002, |
| "reward": 0.685440868139267, |
| "reward_std": 1.0775729417800903, |
| "rewards/accuracy_reward": 0.062499986961483955, |
| "rewards/cosine_rewards": -0.23488027602434158, |
| "rewards/format_reward": 0.859375, |
| "rewards/repetition_rewards": -0.0015538162551820278, |
| "step": 360 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1328.359375, |
| "epoch": 0.18390219052470708, |
| "grad_norm": 5.5666444239146, |
| "kl": 0.046630859375, |
| "learning_rate": 9.080489047376463e-07, |
| "loss": 0.0019, |
| "reward": 1.440912902355194, |
| "reward_std": 1.3688839673995972, |
| "rewards/accuracy_reward": 0.546875, |
| "rewards/cosine_rewards": 0.05198000371456146, |
| "rewards/format_reward": 0.84375, |
| "rewards/repetition_rewards": -0.0016921277856454253, |
| "step": 361 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1222.359375, |
| "epoch": 0.18441161487519103, |
| "grad_norm": 3.8469994374841496, |
| "kl": 0.063720703125, |
| "learning_rate": 9.077941925624044e-07, |
| "loss": 0.0025, |
| "reward": 1.2567678689956665, |
| "reward_std": 1.0096549689769745, |
| "rewards/accuracy_reward": 0.3531249761581421, |
| "rewards/cosine_rewards": -0.06352230161428452, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -0.001584898098371923, |
| "step": 362 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1071.90625, |
| "epoch": 0.184921039225675, |
| "grad_norm": 10.748025785151507, |
| "kl": 0.080810546875, |
| "learning_rate": 9.075394803871624e-07, |
| "loss": 0.0032, |
| "reward": 1.4773434400558472, |
| "reward_std": 0.7611989676952362, |
| "rewards/accuracy_reward": 0.518750011920929, |
| "rewards/cosine_rewards": 0.022458821535110474, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": -0.0013653661007992923, |
| "step": 363 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1068.84375, |
| "epoch": 0.18543046357615894, |
| "grad_norm": 6.961531569684862, |
| "kl": 0.0966796875, |
| "learning_rate": 9.072847682119204e-07, |
| "loss": 0.0039, |
| "reward": 1.3342331051826477, |
| "reward_std": 0.9612607657909393, |
| "rewards/accuracy_reward": 0.4906249940395355, |
| "rewards/cosine_rewards": -0.029875734820961952, |
| "rewards/format_reward": 0.875, |
| "rewards/repetition_rewards": -0.0015161921037361026, |
| "step": 364 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1151.765625, |
| "epoch": 0.1859398879266429, |
| "grad_norm": 4.787495962824196, |
| "kl": 0.0526123046875, |
| "learning_rate": 9.070300560366785e-07, |
| "loss": 0.0021, |
| "reward": 0.44851796329021454, |
| "reward_std": 0.6482652425765991, |
| "rewards/accuracy_reward": -0.18125002831220627, |
| "rewards/cosine_rewards": -0.3225611299276352, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": -0.0007959024223964661, |
| "step": 365 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1087.453125, |
| "epoch": 0.18644931227712686, |
| "grad_norm": 3.5882611622656704, |
| "kl": 0.05517578125, |
| "learning_rate": 9.067753438614365e-07, |
| "loss": 0.0022, |
| "reward": 1.0280417203903198, |
| "reward_std": 0.8365518152713776, |
| "rewards/accuracy_reward": 0.2093750163912773, |
| "rewards/cosine_rewards": -0.10154062137007713, |
| "rewards/format_reward": 0.921875, |
| "rewards/repetition_rewards": -0.0016676230588927865, |
| "step": 366 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 921.3125, |
| "epoch": 0.1869587366276108, |
| "grad_norm": 10.03430669886217, |
| "kl": 0.07080078125, |
| "learning_rate": 9.065206316861945e-07, |
| "loss": 0.0028, |
| "reward": 1.1769609451293945, |
| "reward_std": 0.880241334438324, |
| "rewards/accuracy_reward": 0.2656249925494194, |
| "rewards/cosine_rewards": -0.04036855325102806, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": -0.0014205531333573163, |
| "step": 367 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 841.6875, |
| "epoch": 0.18746816097809477, |
| "grad_norm": 27.23508906311087, |
| "kl": 0.07861328125, |
| "learning_rate": 9.062659195109526e-07, |
| "loss": 0.0031, |
| "reward": 1.685433030128479, |
| "reward_std": 0.49864277243614197, |
| "rewards/accuracy_reward": 0.6875, |
| "rewards/cosine_rewards": 0.1084844060242176, |
| "rewards/format_reward": 0.890625, |
| "rewards/repetition_rewards": -0.0011764070368371904, |
| "step": 368 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 737.03125, |
| "epoch": 0.1879775853285787, |
| "grad_norm": 18.41490109995637, |
| "kl": 0.08740234375, |
| "learning_rate": 9.060112073357106e-07, |
| "loss": 0.0035, |
| "reward": 1.3734083771705627, |
| "reward_std": 0.4119359850883484, |
| "rewards/accuracy_reward": 0.37812499701976776, |
| "rewards/cosine_rewards": 0.01141296117566526, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -0.000504543146234937, |
| "step": 369 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 707.796875, |
| "epoch": 0.18848700967906265, |
| "grad_norm": 35.58611363543668, |
| "kl": 0.084716796875, |
| "learning_rate": 9.057564951604686e-07, |
| "loss": 0.0034, |
| "reward": 1.6782256960868835, |
| "reward_std": 0.5156250298023224, |
| "rewards/accuracy_reward": 0.6343749761581421, |
| "rewards/cosine_rewards": 0.07835755217820406, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -0.0032568235765211284, |
| "step": 370 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 647.015625, |
| "epoch": 0.1889964340295466, |
| "grad_norm": 7.982503076191807, |
| "kl": 0.086669921875, |
| "learning_rate": 9.055017829852266e-07, |
| "loss": 0.0035, |
| "reward": 1.760904848575592, |
| "reward_std": 0.49070215225219727, |
| "rewards/accuracy_reward": 0.690625011920929, |
| "rewards/cosine_rewards": 0.0864610131829977, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -0.0005561279249377549, |
| "step": 371 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 716.359375, |
| "epoch": 0.18950585838003056, |
| "grad_norm": 10.914576943859945, |
| "kl": 0.077880859375, |
| "learning_rate": 9.052470708099847e-07, |
| "loss": 0.0031, |
| "reward": 1.9701185822486877, |
| "reward_std": 0.40086938440799713, |
| "rewards/accuracy_reward": 0.831250011920929, |
| "rewards/cosine_rewards": 0.1397455483675003, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": -0.0008769762353040278, |
| "step": 372 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 738.546875, |
| "epoch": 0.19001528273051452, |
| "grad_norm": 6.034059111318338, |
| "kl": 0.08544921875, |
| "learning_rate": 9.049923586347427e-07, |
| "loss": 0.0034, |
| "reward": 1.8058127164840698, |
| "reward_std": 0.41330619156360626, |
| "rewards/accuracy_reward": 0.7468750178813934, |
| "rewards/cosine_rewards": 0.1067701168358326, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": -0.0009574841533321887, |
| "step": 373 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 728.0, |
| "epoch": 0.19052470708099847, |
| "grad_norm": 8.532753416320839, |
| "kl": 0.07861328125, |
| "learning_rate": 9.047376464595006e-07, |
| "loss": 0.0031, |
| "reward": 1.0842646658420563, |
| "reward_std": 0.44039003551006317, |
| "rewards/accuracy_reward": 0.15312500298023224, |
| "rewards/cosine_rewards": -0.052397772669792175, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -0.0008375749748665839, |
| "step": 374 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 804.21875, |
| "epoch": 0.19103413143148243, |
| "grad_norm": 7.4599789196405375, |
| "kl": 0.078125, |
| "learning_rate": 9.044829342842587e-07, |
| "loss": 0.0031, |
| "reward": 0.974018394947052, |
| "reward_std": 0.5849625766277313, |
| "rewards/accuracy_reward": 0.09999999403953552, |
| "rewards/cosine_rewards": -0.10963174607604742, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -0.0007248484616866335, |
| "step": 375 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 767.84375, |
| "epoch": 0.19154355578196638, |
| "grad_norm": 7.073657321582574, |
| "kl": 0.0703125, |
| "learning_rate": 9.042282221090167e-07, |
| "loss": 0.0028, |
| "reward": 0.914261519908905, |
| "reward_std": 0.7159627079963684, |
| "rewards/accuracy_reward": 0.09999998658895493, |
| "rewards/cosine_rewards": -0.13804471492767334, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": -0.0008187246276065707, |
| "step": 376 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 898.65625, |
| "epoch": 0.19205298013245034, |
| "grad_norm": 5.152812978669099, |
| "kl": 0.060791015625, |
| "learning_rate": 9.039735099337747e-07, |
| "loss": 0.0024, |
| "reward": 1.3070060014724731, |
| "reward_std": 0.5369542390108109, |
| "rewards/accuracy_reward": 0.3531250059604645, |
| "rewards/cosine_rewards": 0.01811320334672928, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": -0.0017322039348073304, |
| "step": 377 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 824.65625, |
| "epoch": 0.1925624044829343, |
| "grad_norm": 4.4264228290413055, |
| "kl": 0.071044921875, |
| "learning_rate": 9.037187977585327e-07, |
| "loss": 0.0028, |
| "reward": 1.9969289302825928, |
| "reward_std": 0.36238182336091995, |
| "rewards/accuracy_reward": 0.887499988079071, |
| "rewards/cosine_rewards": 0.1572401076555252, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": -0.0009361990523757413, |
| "step": 378 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1041.234375, |
| "epoch": 0.19307182883341822, |
| "grad_norm": 2.966764644317296, |
| "kl": 0.0531005859375, |
| "learning_rate": 9.034640855832908e-07, |
| "loss": 0.0021, |
| "reward": 1.9203879237174988, |
| "reward_std": 0.6297050192952156, |
| "rewards/accuracy_reward": 0.831250011920929, |
| "rewards/cosine_rewards": 0.1525670364499092, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": -0.0009290309972129762, |
| "step": 379 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1066.578125, |
| "epoch": 0.19358125318390218, |
| "grad_norm": 6.45607299399273, |
| "kl": 0.0604248046875, |
| "learning_rate": 9.032093734080488e-07, |
| "loss": 0.0024, |
| "reward": 1.5978580713272095, |
| "reward_std": 0.7550583779811859, |
| "rewards/accuracy_reward": 0.546875, |
| "rewards/cosine_rewards": 0.0831909030675888, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -0.0009578557801432908, |
| "step": 380 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1159.84375, |
| "epoch": 0.19409067753438614, |
| "grad_norm": 15.218618980246333, |
| "kl": 0.0557861328125, |
| "learning_rate": 9.029546612328068e-07, |
| "loss": 0.0022, |
| "reward": 1.495898723602295, |
| "reward_std": 0.8071758449077606, |
| "rewards/accuracy_reward": 0.5187499970197678, |
| "rewards/cosine_rewards": 0.04102367162704468, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": -0.0013749129138886929, |
| "step": 381 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1371.671875, |
| "epoch": 0.1946001018848701, |
| "grad_norm": 2.5671889777891415, |
| "kl": 0.0416259765625, |
| "learning_rate": 9.026999490575649e-07, |
| "loss": 0.0017, |
| "reward": 1.4654145240783691, |
| "reward_std": 0.9137448668479919, |
| "rewards/accuracy_reward": 0.5468749925494194, |
| "rewards/cosine_rewards": 0.029642254114151, |
| "rewards/format_reward": 0.890625, |
| "rewards/repetition_rewards": -0.0017276888247579336, |
| "step": 382 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1303.015625, |
| "epoch": 0.19510952623535405, |
| "grad_norm": 3.460987727073742, |
| "kl": 0.0421142578125, |
| "learning_rate": 9.024452368823229e-07, |
| "loss": 0.0017, |
| "reward": 1.348323106765747, |
| "reward_std": 0.42551596462726593, |
| "rewards/accuracy_reward": 0.40937499701976776, |
| "rewards/cosine_rewards": 0.0025482475757598877, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": -0.0011001455131918192, |
| "step": 383 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1507.828125, |
| "epoch": 0.195618950585838, |
| "grad_norm": 2.4731324069110165, |
| "kl": 0.040771484375, |
| "learning_rate": 9.021905247070809e-07, |
| "loss": 0.0016, |
| "reward": 1.2840899229049683, |
| "reward_std": 1.3389369249343872, |
| "rewards/accuracy_reward": 0.43437498807907104, |
| "rewards/cosine_rewards": 0.007216873578727245, |
| "rewards/format_reward": 0.84375, |
| "rewards/repetition_rewards": -0.0012519625015556812, |
| "step": 384 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1392.5625, |
| "epoch": 0.19612837493632196, |
| "grad_norm": 3.9847528096417464, |
| "kl": 0.0401611328125, |
| "learning_rate": 9.019358125318391e-07, |
| "loss": 0.0016, |
| "reward": 0.952269122004509, |
| "reward_std": 1.110903412103653, |
| "rewards/accuracy_reward": 0.21249999478459358, |
| "rewards/cosine_rewards": -0.16534814983606339, |
| "rewards/format_reward": 0.90625, |
| "rewards/repetition_rewards": -0.001132699428126216, |
| "step": 385 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1509.515625, |
| "epoch": 0.19663779928680591, |
| "grad_norm": 1.3917929153423205, |
| "kl": 0.0386962890625, |
| "learning_rate": 9.016811003565971e-07, |
| "loss": 0.0015, |
| "reward": 1.3951207399368286, |
| "reward_std": 1.3895853757858276, |
| "rewards/accuracy_reward": 0.49062497913837433, |
| "rewards/cosine_rewards": 0.030975546687841415, |
| "rewards/format_reward": 0.875, |
| "rewards/repetition_rewards": -0.0014798620832152665, |
| "step": 386 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1453.75, |
| "epoch": 0.19714722363728987, |
| "grad_norm": 4.5123123100553215, |
| "kl": 0.040283203125, |
| "learning_rate": 9.014263881813551e-07, |
| "loss": 0.0016, |
| "reward": 1.0250075459480286, |
| "reward_std": 1.1121925115585327, |
| "rewards/accuracy_reward": 0.2656249925494194, |
| "rewards/cosine_rewards": -0.06678299978375435, |
| "rewards/format_reward": 0.828125, |
| "rewards/repetition_rewards": -0.001959475106559694, |
| "step": 387 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1502.953125, |
| "epoch": 0.19765664798777383, |
| "grad_norm": 6.3687413391252665, |
| "kl": 0.0384521484375, |
| "learning_rate": 9.011716760061131e-07, |
| "loss": 0.0015, |
| "reward": 0.6134699061512947, |
| "reward_std": 0.8420631885528564, |
| "rewards/accuracy_reward": 0.015624985098838806, |
| "rewards/cosine_rewards": -0.2909963075071573, |
| "rewards/format_reward": 0.890625, |
| "rewards/repetition_rewards": -0.0017838198109529912, |
| "step": 388 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1398.71875, |
| "epoch": 0.19816607233825778, |
| "grad_norm": 3.252398002176852, |
| "kl": 0.04052734375, |
| "learning_rate": 9.009169638308711e-07, |
| "loss": 0.0016, |
| "reward": 0.6156338006258011, |
| "reward_std": 1.171474575996399, |
| "rewards/accuracy_reward": -0.012500010430812836, |
| "rewards/cosine_rewards": -0.2769355773925781, |
| "rewards/format_reward": 0.90625, |
| "rewards/repetition_rewards": -0.0011806105903815478, |
| "step": 389 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1356.796875, |
| "epoch": 0.1986754966887417, |
| "grad_norm": 2.9564973631467386, |
| "kl": 0.0411376953125, |
| "learning_rate": 9.006622516556291e-07, |
| "loss": 0.0016, |
| "reward": 1.4809187650680542, |
| "reward_std": 0.4241075813770294, |
| "rewards/accuracy_reward": 0.4656249899417162, |
| "rewards/cosine_rewards": 0.0635819137096405, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": -0.0014131638454273343, |
| "step": 390 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1314.8125, |
| "epoch": 0.19918492103922567, |
| "grad_norm": 1.9868954496027869, |
| "kl": 0.040283203125, |
| "learning_rate": 9.004075394803871e-07, |
| "loss": 0.0016, |
| "reward": 0.3289404660463333, |
| "reward_std": 0.6832451522350311, |
| "rewards/accuracy_reward": -0.23750004172325134, |
| "rewards/cosine_rewards": -0.38559940457344055, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": -0.0010851426632143557, |
| "step": 391 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1211.65625, |
| "epoch": 0.19969434538970962, |
| "grad_norm": 2.3384012636409524, |
| "kl": 0.0426025390625, |
| "learning_rate": 9.001528273051452e-07, |
| "loss": 0.0017, |
| "reward": 1.7431849241256714, |
| "reward_std": 0.5287438631057739, |
| "rewards/accuracy_reward": 0.6624999940395355, |
| "rewards/cosine_rewards": 0.0974309928715229, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -0.0011210814118385315, |
| "step": 392 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1199.40625, |
| "epoch": 0.20020376974019358, |
| "grad_norm": 8.037383660003005, |
| "kl": 0.0426025390625, |
| "learning_rate": 8.998981151299032e-07, |
| "loss": 0.0017, |
| "reward": 1.205706238746643, |
| "reward_std": 0.5482289791107178, |
| "rewards/accuracy_reward": 0.2968749925494194, |
| "rewards/cosine_rewards": -0.09018014371395111, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": -0.0009886454208754003, |
| "step": 393 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1215.25, |
| "epoch": 0.20071319409067753, |
| "grad_norm": 2.7015176132022205, |
| "kl": 0.04150390625, |
| "learning_rate": 8.996434029546612e-07, |
| "loss": 0.0017, |
| "reward": 1.3461086750030518, |
| "reward_std": 0.36276355385780334, |
| "rewards/accuracy_reward": 0.3812499940395355, |
| "rewards/cosine_rewards": -0.033333455212414265, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": -0.0018078879220411181, |
| "step": 394 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1148.140625, |
| "epoch": 0.2012226184411615, |
| "grad_norm": 2.4525739585224064, |
| "kl": 0.0447998046875, |
| "learning_rate": 8.993886907794193e-07, |
| "loss": 0.0018, |
| "reward": 1.6304560899734497, |
| "reward_std": 0.6783818304538727, |
| "rewards/accuracy_reward": 0.5781249850988388, |
| "rewards/cosine_rewards": 0.0690329410135746, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -0.0010768624488264322, |
| "step": 395 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1234.03125, |
| "epoch": 0.20173204279164544, |
| "grad_norm": 2.620518407503657, |
| "kl": 0.0426025390625, |
| "learning_rate": 8.991339786041773e-07, |
| "loss": 0.0017, |
| "reward": 1.0580366849899292, |
| "reward_std": 0.45367684960365295, |
| "rewards/accuracy_reward": 0.18437499552965164, |
| "rewards/cosine_rewards": -0.09430436789989471, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -0.0007839706668164581, |
| "step": 396 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1255.140625, |
| "epoch": 0.2022414671421294, |
| "grad_norm": 2.848324792333859, |
| "kl": 0.0416259765625, |
| "learning_rate": 8.988792664289353e-07, |
| "loss": 0.0017, |
| "reward": 1.396336853504181, |
| "reward_std": 0.6851004362106323, |
| "rewards/accuracy_reward": 0.40937498584389687, |
| "rewards/cosine_rewards": 0.003251887857913971, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -0.00066499671083875, |
| "step": 397 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1243.234375, |
| "epoch": 0.20275089149261336, |
| "grad_norm": 2.5122988909394457, |
| "kl": 0.04150390625, |
| "learning_rate": 8.986245542536933e-07, |
| "loss": 0.0017, |
| "reward": 2.053937077522278, |
| "reward_std": 0.5187530070543289, |
| "rewards/accuracy_reward": 0.8312499821186066, |
| "rewards/cosine_rewards": 0.22372649610042572, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": -0.0010393889679107815, |
| "step": 398 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1395.28125, |
| "epoch": 0.2032603158430973, |
| "grad_norm": 8.131421667160394, |
| "kl": 0.039306640625, |
| "learning_rate": 8.983698420784514e-07, |
| "loss": 0.0016, |
| "reward": 1.9118317365646362, |
| "reward_std": 0.3381110727787018, |
| "rewards/accuracy_reward": 0.7187500149011612, |
| "rewards/cosine_rewards": 0.19487697072327137, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": -0.001795282296370715, |
| "step": 399 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1477.328125, |
| "epoch": 0.20376974019358127, |
| "grad_norm": 2.5663546513961992, |
| "kl": 0.0489501953125, |
| "learning_rate": 8.981151299032094e-07, |
| "loss": 0.002, |
| "reward": 0.616385743021965, |
| "reward_std": 0.5365406274795532, |
| "rewards/accuracy_reward": -0.012500017881393433, |
| "rewards/cosine_rewards": -0.27611421793699265, |
| "rewards/format_reward": 0.90625, |
| "rewards/repetition_rewards": -0.0012499869335442781, |
| "step": 400 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1641.53125, |
| "epoch": 0.2042791645440652, |
| "grad_norm": 2.3476982545455947, |
| "kl": 0.0382080078125, |
| "learning_rate": 8.978604177279674e-07, |
| "loss": 0.0015, |
| "reward": 0.38990160822868347, |
| "reward_std": 1.22097048163414, |
| "rewards/accuracy_reward": -0.06875001266598701, |
| "rewards/cosine_rewards": -0.352715402841568, |
| "rewards/format_reward": 0.8125, |
| "rewards/repetition_rewards": -0.0011329837725497782, |
| "step": 401 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1740.96875, |
| "epoch": 0.20478858889454915, |
| "grad_norm": 1.6789909982175664, |
| "kl": 0.036376953125, |
| "learning_rate": 8.976057055527255e-07, |
| "loss": 0.0015, |
| "reward": 0.7690124660730362, |
| "reward_std": 1.7883394956588745, |
| "rewards/accuracy_reward": 0.24062499403953552, |
| "rewards/cosine_rewards": -0.15724666975438595, |
| "rewards/format_reward": 0.6875, |
| "rewards/repetition_rewards": -0.001865879981778562, |
| "step": 402 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1715.625, |
| "epoch": 0.2052980132450331, |
| "grad_norm": 1.732486740072958, |
| "kl": 0.035400390625, |
| "learning_rate": 8.973509933774834e-07, |
| "loss": 0.0014, |
| "reward": 0.6791011095046997, |
| "reward_std": 1.0334843397140503, |
| "rewards/accuracy_reward": 0.1249999925494194, |
| "rewards/cosine_rewards": -0.21049801260232925, |
| "rewards/format_reward": 0.765625, |
| "rewards/repetition_rewards": -0.0010258048423565924, |
| "step": 403 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1585.046875, |
| "epoch": 0.20580743759551706, |
| "grad_norm": 1.6162362158227377, |
| "kl": 0.037109375, |
| "learning_rate": 8.970962812022414e-07, |
| "loss": 0.0015, |
| "reward": 0.9881232976913452, |
| "reward_std": 1.0253838300704956, |
| "rewards/accuracy_reward": 0.24062499403953552, |
| "rewards/cosine_rewards": -0.12615075334906578, |
| "rewards/format_reward": 0.875, |
| "rewards/repetition_rewards": -0.001350913429632783, |
| "step": 404 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1497.734375, |
| "epoch": 0.20631686194600102, |
| "grad_norm": 5.362930427796704, |
| "kl": 0.039306640625, |
| "learning_rate": 8.968415690269994e-07, |
| "loss": 0.0016, |
| "reward": 1.5187935531139374, |
| "reward_std": 0.5071015954017639, |
| "rewards/accuracy_reward": 0.5218749940395355, |
| "rewards/cosine_rewards": 0.07592727243900299, |
| "rewards/format_reward": 0.921875, |
| "rewards/repetition_rewards": -0.0008836896740831435, |
| "step": 405 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1471.890625, |
| "epoch": 0.20682628629648497, |
| "grad_norm": 2.5474971754837896, |
| "kl": 0.0374755859375, |
| "learning_rate": 8.965868568517575e-07, |
| "loss": 0.0015, |
| "reward": 1.7093470096588135, |
| "reward_std": 0.26929083466529846, |
| "rewards/accuracy_reward": 0.6062499955296516, |
| "rewards/cosine_rewards": 0.13566255569458008, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -0.0013155650231055915, |
| "step": 406 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1517.765625, |
| "epoch": 0.20733571064696893, |
| "grad_norm": 2.3954440093211695, |
| "kl": 0.0372314453125, |
| "learning_rate": 8.963321446765155e-07, |
| "loss": 0.0015, |
| "reward": 1.6693125367164612, |
| "reward_std": 0.8508188724517822, |
| "rewards/accuracy_reward": 0.5781250149011612, |
| "rewards/cosine_rewards": 0.12336396798491478, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -0.000926460576010868, |
| "step": 407 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1427.40625, |
| "epoch": 0.20784513499745289, |
| "grad_norm": 4.487502302070771, |
| "kl": 0.037109375, |
| "learning_rate": 8.960774325012735e-07, |
| "loss": 0.0015, |
| "reward": 1.6373254656791687, |
| "reward_std": 0.37433764338493347, |
| "rewards/accuracy_reward": 0.550000011920929, |
| "rewards/cosine_rewards": 0.11931294947862625, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -0.0007375250570476055, |
| "step": 408 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1500.875, |
| "epoch": 0.20835455934793684, |
| "grad_norm": 5.555469475832445, |
| "kl": 0.0374755859375, |
| "learning_rate": 8.958227203260316e-07, |
| "loss": 0.0015, |
| "reward": 1.398006021976471, |
| "reward_std": 1.336867332458496, |
| "rewards/accuracy_reward": 0.4375, |
| "rewards/cosine_rewards": 0.02401774376630783, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": -0.001011726533761248, |
| "step": 409 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1434.859375, |
| "epoch": 0.2088639836984208, |
| "grad_norm": 3.6143044040934105, |
| "kl": 0.0435791015625, |
| "learning_rate": 8.955680081507896e-07, |
| "loss": 0.0017, |
| "reward": 1.618862271308899, |
| "reward_std": 0.7050271332263947, |
| "rewards/accuracy_reward": 0.5468750074505806, |
| "rewards/cosine_rewards": 0.1038745865225792, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -0.000637321179965511, |
| "step": 410 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1506.828125, |
| "epoch": 0.20937340804890472, |
| "grad_norm": 3.854404990598997, |
| "kl": 0.0361328125, |
| "learning_rate": 8.953132959755476e-07, |
| "loss": 0.0014, |
| "reward": 1.6651726961135864, |
| "reward_std": 0.45976050198078156, |
| "rewards/accuracy_reward": 0.5781250074505806, |
| "rewards/cosine_rewards": 0.11916181445121765, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -0.0008640679297968745, |
| "step": 411 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1526.96875, |
| "epoch": 0.20988283239938868, |
| "grad_norm": 2.3422021364641736, |
| "kl": 0.03662109375, |
| "learning_rate": 8.950585838003057e-07, |
| "loss": 0.0015, |
| "reward": 0.6352521181106567, |
| "reward_std": 1.1320685744285583, |
| "rewards/accuracy_reward": -0.012500017881393433, |
| "rewards/cosine_rewards": -0.28875819593667984, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": -0.0009897005802486092, |
| "step": 412 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1510.03125, |
| "epoch": 0.21039225674987264, |
| "grad_norm": 2.1794044547587275, |
| "kl": 0.0567626953125, |
| "learning_rate": 8.948038716250637e-07, |
| "loss": 0.0023, |
| "reward": 1.4266446828842163, |
| "reward_std": 0.8459653854370117, |
| "rewards/accuracy_reward": 0.4624999910593033, |
| "rewards/cosine_rewards": 0.07424483820796013, |
| "rewards/format_reward": 0.890625, |
| "rewards/repetition_rewards": -0.0007251804636325687, |
| "step": 413 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1505.8125, |
| "epoch": 0.2109016811003566, |
| "grad_norm": 2.019375037035424, |
| "kl": 0.042236328125, |
| "learning_rate": 8.945491594498217e-07, |
| "loss": 0.0017, |
| "reward": 1.4379878044128418, |
| "reward_std": 0.6174334287643433, |
| "rewards/accuracy_reward": 0.4374999888241291, |
| "rewards/cosine_rewards": 0.04812653362751007, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": -0.0007637535745743662, |
| "step": 414 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1476.9375, |
| "epoch": 0.21141110545084055, |
| "grad_norm": 2.647595808512136, |
| "kl": 0.041259765625, |
| "learning_rate": 8.942944472745797e-07, |
| "loss": 0.0016, |
| "reward": 0.9988905191421509, |
| "reward_std": 0.6921209692955017, |
| "rewards/accuracy_reward": 0.20937499403953552, |
| "rewards/cosine_rewards": -0.1462814100086689, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": -0.0017031602037604898, |
| "step": 415 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1508.578125, |
| "epoch": 0.2119205298013245, |
| "grad_norm": 2.6566052420282933, |
| "kl": 0.03466796875, |
| "learning_rate": 8.940397350993378e-07, |
| "loss": 0.0014, |
| "reward": 1.211571991443634, |
| "reward_std": 1.0560529828071594, |
| "rewards/accuracy_reward": 0.32499998807907104, |
| "rewards/cosine_rewards": -0.0497976616024971, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": -0.00113033052184619, |
| "step": 416 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1513.40625, |
| "epoch": 0.21242995415180846, |
| "grad_norm": 2.084079343038072, |
| "kl": 0.0411376953125, |
| "learning_rate": 8.937850229240957e-07, |
| "loss": 0.0016, |
| "reward": 0.5206416845321655, |
| "reward_std": 0.49498558044433594, |
| "rewards/accuracy_reward": -0.09687501192092896, |
| "rewards/cosine_rewards": -0.36537329852581024, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -0.0014850463485345244, |
| "step": 417 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1506.921875, |
| "epoch": 0.21293937850229241, |
| "grad_norm": 1.7101934155068432, |
| "kl": 0.036865234375, |
| "learning_rate": 8.935303107488537e-07, |
| "loss": 0.0015, |
| "reward": 1.16130793094635, |
| "reward_std": 0.738935075700283, |
| "rewards/accuracy_reward": 0.2968749850988388, |
| "rewards/cosine_rewards": -0.08809526264667511, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": -0.0005968308250885457, |
| "step": 418 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1452.859375, |
| "epoch": 0.21344880285277637, |
| "grad_norm": 2.6364264984634236, |
| "kl": 0.037109375, |
| "learning_rate": 8.932755985736118e-07, |
| "loss": 0.0015, |
| "reward": 1.4882609844207764, |
| "reward_std": 0.6527669131755829, |
| "rewards/accuracy_reward": 0.4937499836087227, |
| "rewards/cosine_rewards": 0.042041175067424774, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": -0.0006552368577104062, |
| "step": 419 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1425.0, |
| "epoch": 0.21395822720326033, |
| "grad_norm": 22.24294483419425, |
| "kl": 0.0374755859375, |
| "learning_rate": 8.930208863983698e-07, |
| "loss": 0.0015, |
| "reward": 1.5828353762626648, |
| "reward_std": 0.6265529096126556, |
| "rewards/accuracy_reward": 0.5468749850988388, |
| "rewards/cosine_rewards": 0.08372939098626375, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": -0.0008939505496528, |
| "step": 420 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1396.171875, |
| "epoch": 0.21446765155374428, |
| "grad_norm": 2.8168572000468366, |
| "kl": 0.049560546875, |
| "learning_rate": 8.927661742231278e-07, |
| "loss": 0.002, |
| "reward": 1.6206639409065247, |
| "reward_std": 0.5450826287269592, |
| "rewards/accuracy_reward": 0.546875, |
| "rewards/cosine_rewards": 0.12136101722717285, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": -0.0006969515234231949, |
| "step": 421 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1395.9375, |
| "epoch": 0.2149770759042282, |
| "grad_norm": 1.840733711397487, |
| "kl": 0.0379638671875, |
| "learning_rate": 8.925114620478858e-07, |
| "loss": 0.0015, |
| "reward": 1.8798171877861023, |
| "reward_std": 0.5979900360107422, |
| "rewards/accuracy_reward": 0.690625011920929, |
| "rewards/cosine_rewards": 0.18991604819893837, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": -0.0007238158723339438, |
| "step": 422 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1503.703125, |
| "epoch": 0.21548650025471217, |
| "grad_norm": 2.327429653832842, |
| "kl": 0.0377197265625, |
| "learning_rate": 8.922567498726439e-07, |
| "loss": 0.0015, |
| "reward": 1.1887712478637695, |
| "reward_std": 0.615043044090271, |
| "rewards/accuracy_reward": 0.2968749850988388, |
| "rewards/cosine_rewards": -0.09194361418485641, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -0.0005351053987396881, |
| "step": 423 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1528.765625, |
| "epoch": 0.21599592460519612, |
| "grad_norm": 3.1639646848610017, |
| "kl": 0.0347900390625, |
| "learning_rate": 8.920020376974019e-07, |
| "loss": 0.0014, |
| "reward": 1.1957539916038513, |
| "reward_std": 1.2394747734069824, |
| "rewards/accuracy_reward": 0.3531249985098839, |
| "rewards/cosine_rewards": -0.03115752711892128, |
| "rewards/format_reward": 0.875, |
| "rewards/repetition_rewards": -0.001213467272464186, |
| "step": 424 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1674.765625, |
| "epoch": 0.21650534895568008, |
| "grad_norm": 2.5043711144165126, |
| "kl": 0.0338134765625, |
| "learning_rate": 8.917473255221599e-07, |
| "loss": 0.0014, |
| "reward": 1.1731443107128143, |
| "reward_std": 0.8068048655986786, |
| "rewards/accuracy_reward": 0.3218749836087227, |
| "rewards/cosine_rewards": -0.038288604468107224, |
| "rewards/format_reward": 0.890625, |
| "rewards/repetition_rewards": -0.0010670205520000309, |
| "step": 425 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1664.5, |
| "epoch": 0.21701477330616403, |
| "grad_norm": 3.6500533940846327, |
| "kl": 0.03515625, |
| "learning_rate": 8.91492613346918e-07, |
| "loss": 0.0014, |
| "reward": 0.6680706441402435, |
| "reward_std": 1.1470927596092224, |
| "rewards/accuracy_reward": 0.012499995529651642, |
| "rewards/cosine_rewards": -0.28094063699245453, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": -0.000988698098808527, |
| "step": 426 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1733.359375, |
| "epoch": 0.217524197656648, |
| "grad_norm": 1.7264860203774577, |
| "kl": 0.033203125, |
| "learning_rate": 8.91237901171676e-07, |
| "loss": 0.0013, |
| "reward": 1.151515543460846, |
| "reward_std": 1.0065627694129944, |
| "rewards/accuracy_reward": 0.37812499701976776, |
| "rewards/cosine_rewards": -0.02256488800048828, |
| "rewards/format_reward": 0.796875, |
| "rewards/repetition_rewards": -0.0009195689344778657, |
| "step": 427 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1738.171875, |
| "epoch": 0.21803362200713194, |
| "grad_norm": 1.8028678441679502, |
| "kl": 0.033447265625, |
| "learning_rate": 8.90983188996434e-07, |
| "loss": 0.0013, |
| "reward": 0.33622707426548004, |
| "reward_std": 1.554500699043274, |
| "rewards/accuracy_reward": -0.046875011175870895, |
| "rewards/cosine_rewards": -0.35029861330986023, |
| "rewards/format_reward": 0.734375, |
| "rewards/repetition_rewards": -0.0009743365517351776, |
| "step": 428 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1710.0, |
| "epoch": 0.2185430463576159, |
| "grad_norm": 1.7315216890300187, |
| "kl": 0.0386962890625, |
| "learning_rate": 8.90728476821192e-07, |
| "loss": 0.0015, |
| "reward": 1.2531213760375977, |
| "reward_std": 1.7619973421096802, |
| "rewards/accuracy_reward": 0.4624999910593033, |
| "rewards/cosine_rewards": 0.010814379900693893, |
| "rewards/format_reward": 0.78125, |
| "rewards/repetition_rewards": -0.0014429978909902275, |
| "step": 429 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1681.5625, |
| "epoch": 0.21905247070809986, |
| "grad_norm": 1.448581293364632, |
| "kl": 0.0350341796875, |
| "learning_rate": 8.904737646459501e-07, |
| "loss": 0.0014, |
| "reward": 0.5936174094676971, |
| "reward_std": 1.1982838213443756, |
| "rewards/accuracy_reward": 0.015624940395355225, |
| "rewards/cosine_rewards": -0.2807646095752716, |
| "rewards/format_reward": 0.859375, |
| "rewards/repetition_rewards": -0.0006179730116855353, |
| "step": 430 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1480.1875, |
| "epoch": 0.2195618950585838, |
| "grad_norm": 4.830088485887878, |
| "kl": 0.0394287109375, |
| "learning_rate": 8.90219052470708e-07, |
| "loss": 0.0016, |
| "reward": 1.1779060363769531, |
| "reward_std": 1.0625053942203522, |
| "rewards/accuracy_reward": 0.31562499701976776, |
| "rewards/cosine_rewards": -0.05869085341691971, |
| "rewards/format_reward": 0.921875, |
| "rewards/repetition_rewards": -0.0009031399386003613, |
| "step": 431 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1440.796875, |
| "epoch": 0.22007131940906777, |
| "grad_norm": 2.4405157931321124, |
| "kl": 0.037109375, |
| "learning_rate": 8.89964340295466e-07, |
| "loss": 0.0015, |
| "reward": 0.9002698361873627, |
| "reward_std": 0.7880153059959412, |
| "rewards/accuracy_reward": 0.09999998845160007, |
| "rewards/cosine_rewards": -0.18298358470201492, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -0.0011215846752747893, |
| "step": 432 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1348.515625, |
| "epoch": 0.2205807437595517, |
| "grad_norm": 2.1326390000811806, |
| "kl": 0.0418701171875, |
| "learning_rate": 8.897096281202241e-07, |
| "loss": 0.0017, |
| "reward": 0.7409723997116089, |
| "reward_std": 0.7918355762958527, |
| "rewards/accuracy_reward": 0.015624990686774254, |
| "rewards/cosine_rewards": -0.21155225485563278, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": -0.000600404484430328, |
| "step": 433 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1298.125, |
| "epoch": 0.22109016811003565, |
| "grad_norm": 3.815337891096848, |
| "kl": 0.0418701171875, |
| "learning_rate": 8.894549159449821e-07, |
| "loss": 0.0017, |
| "reward": 1.8587952256202698, |
| "reward_std": 0.6939655542373657, |
| "rewards/accuracy_reward": 0.7187499701976776, |
| "rewards/cosine_rewards": 0.1717987135052681, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -0.0005034840432927012, |
| "step": 434 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1156.0, |
| "epoch": 0.2215995924605196, |
| "grad_norm": 4.479065196602373, |
| "kl": 0.0440673828125, |
| "learning_rate": 8.892002037697401e-07, |
| "loss": 0.0018, |
| "reward": 1.4347090125083923, |
| "reward_std": 0.3772214949131012, |
| "rewards/accuracy_reward": 0.43749997206032276, |
| "rewards/cosine_rewards": -0.0022302046418190002, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": -0.0005607931379927322, |
| "step": 435 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1158.46875, |
| "epoch": 0.22210901681100356, |
| "grad_norm": 4.149290240827553, |
| "kl": 0.0455322265625, |
| "learning_rate": 8.889454915944982e-07, |
| "loss": 0.0018, |
| "reward": 1.0905642956495285, |
| "reward_std": 0.5234603583812714, |
| "rewards/accuracy_reward": 0.2124999761581421, |
| "rewards/cosine_rewards": -0.10600101202726364, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -0.00030972264357842505, |
| "step": 436 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1058.734375, |
| "epoch": 0.22261844116148752, |
| "grad_norm": 6.6780990750455205, |
| "kl": 0.046630859375, |
| "learning_rate": 8.886907794192562e-07, |
| "loss": 0.0019, |
| "reward": 0.9731817841529846, |
| "reward_std": 0.8772869110107422, |
| "rewards/accuracy_reward": 0.09687498956918716, |
| "rewards/cosine_rewards": -0.09189720638096333, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -0.0005459659732878208, |
| "step": 437 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1056.5, |
| "epoch": 0.22312786551197147, |
| "grad_norm": 2.737447142182153, |
| "kl": 0.044189453125, |
| "learning_rate": 8.884360672440142e-07, |
| "loss": 0.0018, |
| "reward": 1.1522070169448853, |
| "reward_std": 0.6963326930999756, |
| "rewards/accuracy_reward": 0.24062499403953552, |
| "rewards/cosine_rewards": -0.07206200063228607, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -0.0007310137443710119, |
| "step": 438 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1072.171875, |
| "epoch": 0.22363728986245543, |
| "grad_norm": 2.6298041678582953, |
| "kl": 0.046875, |
| "learning_rate": 8.881813550687722e-07, |
| "loss": 0.0019, |
| "reward": 1.4510762691497803, |
| "reward_std": 0.5045955777168274, |
| "rewards/accuracy_reward": 0.49375002086162567, |
| "rewards/cosine_rewards": 0.020084097981452942, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": -0.0002578186395112425, |
| "step": 439 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1094.546875, |
| "epoch": 0.22414671421293939, |
| "grad_norm": 2.047145097208152, |
| "kl": 0.0438232421875, |
| "learning_rate": 8.879266428935303e-07, |
| "loss": 0.0018, |
| "reward": 1.5189008712768555, |
| "reward_std": 0.34239334613084793, |
| "rewards/accuracy_reward": 0.4906250089406967, |
| "rewards/cosine_rewards": 0.0758383758366108, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": -0.0006874670943943784, |
| "step": 440 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1083.09375, |
| "epoch": 0.22465613856342334, |
| "grad_norm": 2.966919929118076, |
| "kl": 0.0457763671875, |
| "learning_rate": 8.876719307182883e-07, |
| "loss": 0.0018, |
| "reward": 1.2056291699409485, |
| "reward_std": 0.828714907169342, |
| "rewards/accuracy_reward": 0.29687498696148396, |
| "rewards/cosine_rewards": -0.043857116252183914, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": -0.0005137350672157481, |
| "step": 441 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1103.375, |
| "epoch": 0.2251655629139073, |
| "grad_norm": 3.389083005059361, |
| "kl": 0.042724609375, |
| "learning_rate": 8.874172185430463e-07, |
| "loss": 0.0017, |
| "reward": 1.483572542667389, |
| "reward_std": 0.5207121074199677, |
| "rewards/accuracy_reward": 0.4624999910593033, |
| "rewards/cosine_rewards": 0.053015733137726784, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -0.0006932187097845599, |
| "step": 442 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1217.421875, |
| "epoch": 0.22567498726439122, |
| "grad_norm": 1.8084576051013326, |
| "kl": 0.0426025390625, |
| "learning_rate": 8.871625063678044e-07, |
| "loss": 0.0017, |
| "reward": 1.604416847229004, |
| "reward_std": 0.68864506483078, |
| "rewards/accuracy_reward": 0.578125, |
| "rewards/cosine_rewards": 0.10491618514060974, |
| "rewards/format_reward": 0.921875, |
| "rewards/repetition_rewards": -0.0004992800822947174, |
| "step": 443 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1285.484375, |
| "epoch": 0.22618441161487518, |
| "grad_norm": 20.129612374292474, |
| "kl": 0.042236328125, |
| "learning_rate": 8.869077941925624e-07, |
| "loss": 0.0017, |
| "reward": 1.7994786500930786, |
| "reward_std": 0.3120774105191231, |
| "rewards/accuracy_reward": 0.6624999940395355, |
| "rewards/cosine_rewards": 0.15342308580875397, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -0.0008193884277716279, |
| "step": 444 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1370.578125, |
| "epoch": 0.22669383596535914, |
| "grad_norm": 3.234143174544009, |
| "kl": 0.0433349609375, |
| "learning_rate": 8.866530820173203e-07, |
| "loss": 0.0017, |
| "reward": 1.3398171067237854, |
| "reward_std": 0.7532171607017517, |
| "rewards/accuracy_reward": 0.3812500014901161, |
| "rewards/cosine_rewards": -0.0252380333840847, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -0.0005697726446669549, |
| "step": 445 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1474.875, |
| "epoch": 0.2272032603158431, |
| "grad_norm": 1.7926893560129409, |
| "kl": 0.040283203125, |
| "learning_rate": 8.863983698420783e-07, |
| "loss": 0.0016, |
| "reward": 1.4312800765037537, |
| "reward_std": 0.6859093904495239, |
| "rewards/accuracy_reward": 0.43437500298023224, |
| "rewards/cosine_rewards": 0.02916320227086544, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -0.0010081499349325895, |
| "step": 446 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1589.5625, |
| "epoch": 0.22771268466632705, |
| "grad_norm": 1.796069908482133, |
| "kl": 0.036865234375, |
| "learning_rate": 8.861436576668364e-07, |
| "loss": 0.0015, |
| "reward": 1.443231225013733, |
| "reward_std": 0.6114392578601837, |
| "rewards/accuracy_reward": 0.4375000074505806, |
| "rewards/cosine_rewards": 0.038026634603738785, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -0.001045387762133032, |
| "step": 447 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1672.09375, |
| "epoch": 0.228222109016811, |
| "grad_norm": 2.1300405555940802, |
| "kl": 0.0377197265625, |
| "learning_rate": 8.858889454915944e-07, |
| "loss": 0.0015, |
| "reward": 1.5443891882896423, |
| "reward_std": 0.5743480771780014, |
| "rewards/accuracy_reward": 0.550000011920929, |
| "rewards/cosine_rewards": 0.10473084449768066, |
| "rewards/format_reward": 0.890625, |
| "rewards/repetition_rewards": -0.0009665640536695719, |
| "step": 448 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1766.0, |
| "epoch": 0.22873153336729496, |
| "grad_norm": 4.287974732646472, |
| "kl": 0.037841796875, |
| "learning_rate": 8.856342333163524e-07, |
| "loss": 0.0015, |
| "reward": 1.3031042218208313, |
| "reward_std": 1.7085354328155518, |
| "rewards/accuracy_reward": 0.4593749940395355, |
| "rewards/cosine_rewards": 0.06326716393232346, |
| "rewards/format_reward": 0.78125, |
| "rewards/repetition_rewards": -0.0007880023040343076, |
| "step": 449 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1844.640625, |
| "epoch": 0.22924095771777891, |
| "grad_norm": 1.6528382190432698, |
| "kl": 0.0341796875, |
| "learning_rate": 8.853795211411105e-07, |
| "loss": 0.0014, |
| "reward": 0.5885469168424606, |
| "reward_std": 1.7182486653327942, |
| "rewards/accuracy_reward": 0.17812500149011612, |
| "rewards/cosine_rewards": -0.1980201005935669, |
| "rewards/format_reward": 0.609375, |
| "rewards/repetition_rewards": -0.0009329892345704138, |
| "step": 450 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1887.9375, |
| "epoch": 0.22975038206826287, |
| "grad_norm": 1.8787523511209072, |
| "kl": 0.0335693359375, |
| "learning_rate": 8.851248089658685e-07, |
| "loss": 0.0013, |
| "reward": 0.7307622581720352, |
| "reward_std": 1.600571632385254, |
| "rewards/accuracy_reward": 0.24062500894069672, |
| "rewards/cosine_rewards": -0.13401341438293457, |
| "rewards/format_reward": 0.625, |
| "rewards/repetition_rewards": -0.0008493586792610586, |
| "step": 451 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1833.78125, |
| "epoch": 0.23025980641874683, |
| "grad_norm": 6.893897138558536, |
| "kl": 0.0357666015625, |
| "learning_rate": 8.848700967906265e-07, |
| "loss": 0.0014, |
| "reward": 1.0294001996517181, |
| "reward_std": 1.7062013149261475, |
| "rewards/accuracy_reward": 0.40312499552965164, |
| "rewards/cosine_rewards": 0.0025026053190231323, |
| "rewards/format_reward": 0.625, |
| "rewards/repetition_rewards": -0.0012274246546439826, |
| "step": 452 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1943.453125, |
| "epoch": 0.23076923076923078, |
| "grad_norm": 2.3588444541934273, |
| "kl": 0.0322265625, |
| "learning_rate": 8.846153846153846e-07, |
| "loss": 0.0013, |
| "reward": 0.19596866890788078, |
| "reward_std": 1.8146210312843323, |
| "rewards/accuracy_reward": -0.02500000223517418, |
| "rewards/cosine_rewards": -0.3406580686569214, |
| "rewards/format_reward": 0.5625, |
| "rewards/repetition_rewards": -0.0008732638962101191, |
| "step": 453 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1863.109375, |
| "epoch": 0.2312786551197147, |
| "grad_norm": 1.5475872027740656, |
| "kl": 0.0400390625, |
| "learning_rate": 8.843606724401426e-07, |
| "loss": 0.0016, |
| "reward": 0.2073364406824112, |
| "reward_std": 1.7386137247085571, |
| "rewards/accuracy_reward": -0.043750010430812836, |
| "rewards/cosine_rewards": -0.3414689302444458, |
| "rewards/format_reward": 0.59375, |
| "rewards/repetition_rewards": -0.0011946168669965118, |
| "step": 454 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1659.65625, |
| "epoch": 0.23178807947019867, |
| "grad_norm": 3.381623642320965, |
| "kl": 0.0540771484375, |
| "learning_rate": 8.841059602649006e-07, |
| "loss": 0.0022, |
| "reward": 1.5414963960647583, |
| "reward_std": 1.3864411413669586, |
| "rewards/accuracy_reward": 0.6218750178813934, |
| "rewards/cosine_rewards": 0.20184022560715675, |
| "rewards/format_reward": 0.71875, |
| "rewards/repetition_rewards": -0.000968798267422244, |
| "step": 455 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1617.046875, |
| "epoch": 0.23229750382068262, |
| "grad_norm": 7.137141646753771, |
| "kl": 0.0372314453125, |
| "learning_rate": 8.838512480896586e-07, |
| "loss": 0.0015, |
| "reward": 1.1092736423015594, |
| "reward_std": 0.9871836006641388, |
| "rewards/accuracy_reward": 0.2687499839812517, |
| "rewards/cosine_rewards": -0.09614543057978153, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": -0.0008309493132401258, |
| "step": 456 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1511.703125, |
| "epoch": 0.23280692817116658, |
| "grad_norm": 2.858407735203425, |
| "kl": 0.0447998046875, |
| "learning_rate": 8.835965359144167e-07, |
| "loss": 0.0018, |
| "reward": 1.4474474489688873, |
| "reward_std": 0.8567388504743576, |
| "rewards/accuracy_reward": 0.4937500078231096, |
| "rewards/cosine_rewards": 0.06434839963912964, |
| "rewards/format_reward": 0.890625, |
| "rewards/repetition_rewards": -0.0012760092504322529, |
| "step": 457 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1550.828125, |
| "epoch": 0.23331635252165053, |
| "grad_norm": 2.432372091234863, |
| "kl": 0.0408935546875, |
| "learning_rate": 8.833418237391747e-07, |
| "loss": 0.0016, |
| "reward": 1.0046057403087616, |
| "reward_std": 1.0828097462654114, |
| "rewards/accuracy_reward": 0.20937498658895493, |
| "rewards/cosine_rewards": -0.14123845472931862, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": -0.0010308316559530795, |
| "step": 458 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1519.9375, |
| "epoch": 0.2338257768721345, |
| "grad_norm": 2.65385943980829, |
| "kl": 0.0380859375, |
| "learning_rate": 8.830871115639326e-07, |
| "loss": 0.0015, |
| "reward": 1.5737290382385254, |
| "reward_std": 0.676769882440567, |
| "rewards/accuracy_reward": 0.5187499821186066, |
| "rewards/cosine_rewards": 0.10273971408605576, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": -0.0008856799395289272, |
| "step": 459 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1412.171875, |
| "epoch": 0.23433520122261844, |
| "grad_norm": 7.668944991953695, |
| "kl": 0.03955078125, |
| "learning_rate": 8.828323993886907e-07, |
| "loss": 0.0016, |
| "reward": 1.2575648427009583, |
| "reward_std": 0.8219007402658463, |
| "rewards/accuracy_reward": 0.3499999940395355, |
| "rewards/cosine_rewards": -0.029430712573230267, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": -0.0005044575809733942, |
| "step": 460 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1428.671875, |
| "epoch": 0.2348446255731024, |
| "grad_norm": 3.3174885087942747, |
| "kl": 0.041259765625, |
| "learning_rate": 8.825776872134487e-07, |
| "loss": 0.0017, |
| "reward": 0.5390121340751648, |
| "reward_std": 0.6499587297439575, |
| "rewards/accuracy_reward": -0.09687501192092896, |
| "rewards/cosine_rewards": -0.3163621127605438, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": -0.0008757157484069467, |
| "step": 461 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1389.328125, |
| "epoch": 0.23535404992358636, |
| "grad_norm": 1.976395582002063, |
| "kl": 0.040771484375, |
| "learning_rate": 8.823229750382067e-07, |
| "loss": 0.0016, |
| "reward": 1.6086109280586243, |
| "reward_std": 0.5066869556903839, |
| "rewards/accuracy_reward": 0.5218749716877937, |
| "rewards/cosine_rewards": 0.08817524462938309, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": -0.0014392710290849209, |
| "step": 462 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1436.625, |
| "epoch": 0.2358634742740703, |
| "grad_norm": 2.4713376444103146, |
| "kl": 0.039794921875, |
| "learning_rate": 8.820682628629647e-07, |
| "loss": 0.0016, |
| "reward": 1.111421525478363, |
| "reward_std": 0.9693822264671326, |
| "rewards/accuracy_reward": 0.24062500149011612, |
| "rewards/cosine_rewards": -0.11270357295870781, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -0.0008748299151193351, |
| "step": 463 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1397.96875, |
| "epoch": 0.23637289862455427, |
| "grad_norm": 2.688736588573913, |
| "kl": 0.0450439453125, |
| "learning_rate": 8.818135506877228e-07, |
| "loss": 0.0018, |
| "reward": 1.0273907780647278, |
| "reward_std": 0.6092932820320129, |
| "rewards/accuracy_reward": 0.20937500149011612, |
| "rewards/cosine_rewards": -0.11841067671775818, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": -0.0010735246760305017, |
| "step": 464 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1502.5, |
| "epoch": 0.2368823229750382, |
| "grad_norm": 2.233484328017706, |
| "kl": 0.03955078125, |
| "learning_rate": 8.815588385124808e-07, |
| "loss": 0.0016, |
| "reward": 2.0390628576278687, |
| "reward_std": 0.4271709471940994, |
| "rewards/accuracy_reward": 0.7750000059604645, |
| "rewards/cosine_rewards": 0.26496873423457146, |
| "rewards/format_reward": 1.0, |
| "rewards/repetition_rewards": -0.0009058607101906091, |
| "step": 465 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1610.734375, |
| "epoch": 0.23739174732552215, |
| "grad_norm": 3.1291938497316867, |
| "kl": 0.0394287109375, |
| "learning_rate": 8.813041263372388e-07, |
| "loss": 0.0016, |
| "reward": 1.7407687306404114, |
| "reward_std": 0.8337388634681702, |
| "rewards/accuracy_reward": 0.659375011920929, |
| "rewards/cosine_rewards": 0.17613628506660461, |
| "rewards/format_reward": 0.90625, |
| "rewards/repetition_rewards": -0.0009925005142576993, |
| "step": 466 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1568.890625, |
| "epoch": 0.2379011716760061, |
| "grad_norm": 2.121477514968572, |
| "kl": 0.0380859375, |
| "learning_rate": 8.810494141619969e-07, |
| "loss": 0.0015, |
| "reward": 1.3490102887153625, |
| "reward_std": 0.7418502867221832, |
| "rewards/accuracy_reward": 0.37812500447034836, |
| "rewards/cosine_rewards": 0.003062829375267029, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -0.0009275085176341236, |
| "step": 467 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1698.8125, |
| "epoch": 0.23841059602649006, |
| "grad_norm": 2.4246076523727025, |
| "kl": 0.03662109375, |
| "learning_rate": 8.807947019867549e-07, |
| "loss": 0.0015, |
| "reward": 1.3666119575500488, |
| "reward_std": 1.175959825515747, |
| "rewards/accuracy_reward": 0.46562501788139343, |
| "rewards/cosine_rewards": 0.02737235650420189, |
| "rewards/format_reward": 0.875, |
| "rewards/repetition_rewards": -0.0013854140415787697, |
| "step": 468 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1691.90625, |
| "epoch": 0.23892002037697402, |
| "grad_norm": 1.3535821296606787, |
| "kl": 0.039306640625, |
| "learning_rate": 8.805399898115129e-07, |
| "loss": 0.0016, |
| "reward": 1.2414605617523193, |
| "reward_std": 1.0560136437416077, |
| "rewards/accuracy_reward": 0.3812499940395355, |
| "rewards/cosine_rewards": -0.02948123589158058, |
| "rewards/format_reward": 0.890625, |
| "rewards/repetition_rewards": -0.0009331759065389633, |
| "step": 469 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1686.46875, |
| "epoch": 0.23942944472745797, |
| "grad_norm": 1.6084637763929415, |
| "kl": 0.0467529296875, |
| "learning_rate": 8.802852776362711e-07, |
| "loss": 0.0019, |
| "reward": 2.0340508222579956, |
| "reward_std": 1.1313848793506622, |
| "rewards/accuracy_reward": 0.8312499821186066, |
| "rewards/cosine_rewards": 0.3132530748844147, |
| "rewards/format_reward": 0.890625, |
| "rewards/repetition_rewards": -0.001077289809472859, |
| "step": 470 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1638.4375, |
| "epoch": 0.23993886907794193, |
| "grad_norm": 3.9206215218528553, |
| "kl": 0.0384521484375, |
| "learning_rate": 8.800305654610291e-07, |
| "loss": 0.0015, |
| "reward": 1.417995810508728, |
| "reward_std": 0.7844535112380981, |
| "rewards/accuracy_reward": 0.4375, |
| "rewards/cosine_rewards": 0.04390082508325577, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": -0.0009050128574017435, |
| "step": 471 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1583.171875, |
| "epoch": 0.24044829342842589, |
| "grad_norm": 1.862615408125007, |
| "kl": 0.0404052734375, |
| "learning_rate": 8.797758532857871e-07, |
| "loss": 0.0016, |
| "reward": 1.3552428185939789, |
| "reward_std": 0.831163614988327, |
| "rewards/accuracy_reward": 0.40937500074505806, |
| "rewards/cosine_rewards": -0.005982518196105957, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": -0.001274671230930835, |
| "step": 472 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1606.359375, |
| "epoch": 0.24095771777890984, |
| "grad_norm": 6.217207017794225, |
| "kl": 0.039794921875, |
| "learning_rate": 8.795211411105451e-07, |
| "loss": 0.0016, |
| "reward": 1.677711844444275, |
| "reward_std": 0.8612502366304398, |
| "rewards/accuracy_reward": 0.5781249701976776, |
| "rewards/cosine_rewards": 0.11638512089848518, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -0.0011732576531358063, |
| "step": 473 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1521.84375, |
| "epoch": 0.2414671421293938, |
| "grad_norm": 2.9743621746841677, |
| "kl": 0.0421142578125, |
| "learning_rate": 8.792664289353031e-07, |
| "loss": 0.0017, |
| "reward": 1.578629732131958, |
| "reward_std": 0.6186130940914154, |
| "rewards/accuracy_reward": 0.5218749940395355, |
| "rewards/cosine_rewards": 0.08938230201601982, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -0.0013775942497886717, |
| "step": 474 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1560.25, |
| "epoch": 0.24197656647987772, |
| "grad_norm": 3.931202850710427, |
| "kl": 0.0396728515625, |
| "learning_rate": 8.790117167600611e-07, |
| "loss": 0.0016, |
| "reward": 1.8553311824798584, |
| "reward_std": 0.5484062433242798, |
| "rewards/accuracy_reward": 0.6906249970197678, |
| "rewards/cosine_rewards": 0.1969544254243374, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -0.0009982050396502018, |
| "step": 475 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1487.171875, |
| "epoch": 0.24248599083036168, |
| "grad_norm": 1.8115703582475124, |
| "kl": 0.0428466796875, |
| "learning_rate": 8.787570045848191e-07, |
| "loss": 0.0017, |
| "reward": 1.0939862728118896, |
| "reward_std": 0.652959406375885, |
| "rewards/accuracy_reward": 0.24062499776482582, |
| "rewards/cosine_rewards": -0.0988575927913189, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": -0.0009061352466233075, |
| "step": 476 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1421.203125, |
| "epoch": 0.24299541518084564, |
| "grad_norm": 50.509798046645564, |
| "kl": 0.0455322265625, |
| "learning_rate": 8.785022924095772e-07, |
| "loss": 0.0018, |
| "reward": 1.1556105613708496, |
| "reward_std": 0.8080581426620483, |
| "rewards/accuracy_reward": 0.26874998956918716, |
| "rewards/cosine_rewards": -0.06538418680429459, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": -0.0008802659867797047, |
| "step": 477 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1423.25, |
| "epoch": 0.2435048395313296, |
| "grad_norm": 2.143196092673566, |
| "kl": 0.042724609375, |
| "learning_rate": 8.782475802343352e-07, |
| "loss": 0.0017, |
| "reward": 1.4646123051643372, |
| "reward_std": 0.4019291028380394, |
| "rewards/accuracy_reward": 0.4374999925494194, |
| "rewards/cosine_rewards": 0.04411640763282776, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -0.0013792455429211259, |
| "step": 478 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1525.359375, |
| "epoch": 0.24401426388181355, |
| "grad_norm": 1.4650345234014313, |
| "kl": 0.043701171875, |
| "learning_rate": 8.779928680590932e-07, |
| "loss": 0.0018, |
| "reward": 1.7237411737442017, |
| "reward_std": 0.6819100677967072, |
| "rewards/accuracy_reward": 0.6031249761581421, |
| "rewards/cosine_rewards": 0.13805609196424484, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -0.0018149468814954162, |
| "step": 479 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1518.6875, |
| "epoch": 0.2445236882322975, |
| "grad_norm": 2.4194184684625166, |
| "kl": 0.0440673828125, |
| "learning_rate": 8.777381558838512e-07, |
| "loss": 0.0018, |
| "reward": 1.4634617269039154, |
| "reward_std": 0.4558331221342087, |
| "rewards/accuracy_reward": 0.46562497317790985, |
| "rewards/cosine_rewards": 0.03047458827495575, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -0.0013878352474421263, |
| "step": 480 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1559.859375, |
| "epoch": 0.24503311258278146, |
| "grad_norm": 5.778258363606284, |
| "kl": 0.041015625, |
| "learning_rate": 8.774834437086093e-07, |
| "loss": 0.0016, |
| "reward": 1.1754435896873474, |
| "reward_std": 0.629539430141449, |
| "rewards/accuracy_reward": 0.2968749850988388, |
| "rewards/cosine_rewards": -0.058087632060050964, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": -0.000843802816234529, |
| "step": 481 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1613.171875, |
| "epoch": 0.24554253693326542, |
| "grad_norm": 1.6143411312623293, |
| "kl": 0.0389404296875, |
| "learning_rate": 8.772287315333673e-07, |
| "loss": 0.0016, |
| "reward": 0.8586589694023132, |
| "reward_std": 0.45200832188129425, |
| "rewards/accuracy_reward": 0.09999999403953552, |
| "rewards/cosine_rewards": -0.22472049295902252, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -0.0009954352863132954, |
| "step": 482 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1670.359375, |
| "epoch": 0.24605196128374937, |
| "grad_norm": 2.239924603374423, |
| "kl": 0.0592041015625, |
| "learning_rate": 8.769740193581253e-07, |
| "loss": 0.0024, |
| "reward": 1.522126853466034, |
| "reward_std": 0.8742709904909134, |
| "rewards/accuracy_reward": 0.4937499910593033, |
| "rewards/cosine_rewards": 0.060823358595371246, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -0.0011965514277108014, |
| "step": 483 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1628.625, |
| "epoch": 0.24656138563423333, |
| "grad_norm": 8.106075584628705, |
| "kl": 0.0419921875, |
| "learning_rate": 8.767193071828834e-07, |
| "loss": 0.0017, |
| "reward": 0.9253878593444824, |
| "reward_std": 1.307717740535736, |
| "rewards/accuracy_reward": 0.18437500298023224, |
| "rewards/cosine_rewards": -0.14811599627137184, |
| "rewards/format_reward": 0.890625, |
| "rewards/repetition_rewards": -0.0014961253036744893, |
| "step": 484 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1648.875, |
| "epoch": 0.24707080998471728, |
| "grad_norm": 1.6847159437413002, |
| "kl": 0.0389404296875, |
| "learning_rate": 8.764645950076414e-07, |
| "loss": 0.0016, |
| "reward": 1.4973651766777039, |
| "reward_std": 0.9533334523439407, |
| "rewards/accuracy_reward": 0.5218749791383743, |
| "rewards/cosine_rewards": 0.0704129058867693, |
| "rewards/format_reward": 0.90625, |
| "rewards/repetition_rewards": -0.0011727037781383842, |
| "step": 485 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1511.140625, |
| "epoch": 0.2475802343352012, |
| "grad_norm": 1.736665525794252, |
| "kl": 0.0399169921875, |
| "learning_rate": 8.762098828323994e-07, |
| "loss": 0.0016, |
| "reward": 0.7075473368167877, |
| "reward_std": 0.8960316479206085, |
| "rewards/accuracy_reward": 0.015624992549419403, |
| "rewards/cosine_rewards": -0.2443552017211914, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": -0.001222497143317014, |
| "step": 486 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1567.671875, |
| "epoch": 0.24808965868568517, |
| "grad_norm": 3.031931829741236, |
| "kl": 0.0386962890625, |
| "learning_rate": 8.759551706571575e-07, |
| "loss": 0.0015, |
| "reward": 1.3812061548233032, |
| "reward_std": 0.8888083398342133, |
| "rewards/accuracy_reward": 0.4093749672174454, |
| "rewards/cosine_rewards": 0.004479339346289635, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -0.0013981764786876738, |
| "step": 487 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1457.359375, |
| "epoch": 0.24859908303616912, |
| "grad_norm": 5.524727047487839, |
| "kl": 0.0506591796875, |
| "learning_rate": 8.757004584819154e-07, |
| "loss": 0.002, |
| "reward": 1.8092041611671448, |
| "reward_std": 0.5159921646118164, |
| "rewards/accuracy_reward": 0.6343750059604645, |
| "rewards/cosine_rewards": 0.19165128469467163, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -0.0011971485218964517, |
| "step": 488 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1379.078125, |
| "epoch": 0.24910850738665308, |
| "grad_norm": 7.278746642143023, |
| "kl": 0.055419921875, |
| "learning_rate": 8.754457463066734e-07, |
| "loss": 0.0022, |
| "reward": 1.194389447569847, |
| "reward_std": 0.5000828057527542, |
| "rewards/accuracy_reward": 0.26874998211860657, |
| "rewards/cosine_rewards": -0.04208715260028839, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -0.0010234276414848864, |
| "step": 489 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1397.78125, |
| "epoch": 0.24961793173713703, |
| "grad_norm": 2.9100594613125352, |
| "kl": 0.0543212890625, |
| "learning_rate": 8.751910341314314e-07, |
| "loss": 0.0022, |
| "reward": 1.6665399670600891, |
| "reward_std": 0.6835527420043945, |
| "rewards/accuracy_reward": 0.6625000089406967, |
| "rewards/cosine_rewards": 0.11463410407304764, |
| "rewards/format_reward": 0.890625, |
| "rewards/repetition_rewards": -0.00121912601753138, |
| "step": 490 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1359.703125, |
| "epoch": 0.250127356087621, |
| "grad_norm": 12.325171247664876, |
| "kl": 0.0457763671875, |
| "learning_rate": 8.749363219561895e-07, |
| "loss": 0.0018, |
| "reward": 1.8410940766334534, |
| "reward_std": 0.4001428484916687, |
| "rewards/accuracy_reward": 0.690625011920929, |
| "rewards/cosine_rewards": 0.18309018202126026, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -0.0013711884384974837, |
| "step": 491 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1470.75, |
| "epoch": 0.25063678043810494, |
| "grad_norm": 15.076912789505334, |
| "kl": 0.041015625, |
| "learning_rate": 8.746816097809475e-07, |
| "loss": 0.0016, |
| "reward": 1.2887136340141296, |
| "reward_std": 0.8450455367565155, |
| "rewards/accuracy_reward": 0.3531249761581421, |
| "rewards/cosine_rewards": -0.03244372457265854, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -0.0007176562794484198, |
| "step": 492 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1348.140625, |
| "epoch": 0.2511462047885889, |
| "grad_norm": 10.32545410862146, |
| "kl": 0.05810546875, |
| "learning_rate": 8.744268976057055e-07, |
| "loss": 0.0023, |
| "reward": 1.227342277765274, |
| "reward_std": 1.0202240645885468, |
| "rewards/accuracy_reward": 0.3500000238418579, |
| "rewards/cosine_rewards": 0.002939566969871521, |
| "rewards/format_reward": 0.875, |
| "rewards/repetition_rewards": -0.0005973072838969529, |
| "step": 493 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1301.828125, |
| "epoch": 0.25165562913907286, |
| "grad_norm": 3.667270735290933, |
| "kl": 0.0645751953125, |
| "learning_rate": 8.741721854304636e-07, |
| "loss": 0.0026, |
| "reward": 1.2533040046691895, |
| "reward_std": 0.7434202134609222, |
| "rewards/accuracy_reward": 0.32500000298023224, |
| "rewards/cosine_rewards": -0.03934769332408905, |
| "rewards/format_reward": 0.96875, |
| "rewards/repetition_rewards": -0.0010982811218127608, |
| "step": 494 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1290.1875, |
| "epoch": 0.2521650534895568, |
| "grad_norm": 4.3465734782527345, |
| "kl": 0.0535888671875, |
| "learning_rate": 8.739174732552216e-07, |
| "loss": 0.0021, |
| "reward": 0.6353173404932022, |
| "reward_std": 0.6600025594234467, |
| "rewards/accuracy_reward": -0.040625013411045074, |
| "rewards/cosine_rewards": -0.2607284113764763, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": -0.0008293068385683, |
| "step": 495 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1355.375, |
| "epoch": 0.25267447784004077, |
| "grad_norm": 4.901543791199254, |
| "kl": 0.060546875, |
| "learning_rate": 8.736627610799796e-07, |
| "loss": 0.0024, |
| "reward": 1.1404387950897217, |
| "reward_std": 0.670623242855072, |
| "rewards/accuracy_reward": 0.26874999701976776, |
| "rewards/cosine_rewards": -0.11177334189414978, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -0.0009129364043474197, |
| "step": 496 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1328.6875, |
| "epoch": 0.2531839021905247, |
| "grad_norm": 3.9728562721335745, |
| "kl": 0.0489501953125, |
| "learning_rate": 8.734080489047376e-07, |
| "loss": 0.002, |
| "reward": 1.1998997032642365, |
| "reward_std": 0.630705714225769, |
| "rewards/accuracy_reward": 0.32500001788139343, |
| "rewards/cosine_rewards": -0.06150183826684952, |
| "rewards/format_reward": 0.9375, |
| "rewards/repetition_rewards": -0.0010984738764818758, |
| "step": 497 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1373.671875, |
| "epoch": 0.2536933265410087, |
| "grad_norm": 3.325255325148765, |
| "kl": 0.04833984375, |
| "learning_rate": 8.731533367294957e-07, |
| "loss": 0.0019, |
| "reward": 1.2019822597503662, |
| "reward_std": 0.38447779417037964, |
| "rewards/accuracy_reward": 0.296875, |
| "rewards/cosine_rewards": -0.04712319001555443, |
| "rewards/format_reward": 0.953125, |
| "rewards/repetition_rewards": -0.0008945107110776007, |
| "step": 498 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1309.28125, |
| "epoch": 0.25420275089149263, |
| "grad_norm": 5.499632802088616, |
| "kl": 0.072265625, |
| "learning_rate": 8.728986245542537e-07, |
| "loss": 0.0029, |
| "reward": 1.6111189126968384, |
| "reward_std": 0.1892632469534874, |
| "rewards/accuracy_reward": 0.550000011920929, |
| "rewards/cosine_rewards": 0.07773812115192413, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -0.0009941596072167158, |
| "step": 499 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1363.625, |
| "epoch": 0.2547121752419766, |
| "grad_norm": 7.195546109062687, |
| "kl": 0.0482177734375, |
| "learning_rate": 8.726439123790117e-07, |
| "loss": 0.0019, |
| "reward": 1.9320534467697144, |
| "reward_std": 0.4148600548505783, |
| "rewards/accuracy_reward": 0.7468750178813934, |
| "rewards/cosine_rewards": 0.20207761228084564, |
| "rewards/format_reward": 0.984375, |
| "rewards/repetition_rewards": -0.0012741541431751102, |
| "step": 500 |
| } |
| ], |
| "logging_steps": 1.0, |
| "max_steps": 3926, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 2, |
| "save_steps": 500, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 4, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
