| { |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.1713796058269066, |
| "eval_steps": 500, |
| "global_step": 100, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "completion_length": 2842.9896545410156, |
| "epoch": 0.001713796058269066, |
| "grad_norm": 0.16336098313331604, |
| "kl": 0.0, |
| "learning_rate": 1e-07, |
| "loss": 0.0, |
| "reward": 0.0586470989510417, |
| "reward_std": 0.1930437758564949, |
| "rewards/cosine_scaled_reward": -0.0629267618060112, |
| "rewards/format_reward": 0.3541666716337204, |
| "step": 1 |
| }, |
| { |
| "completion_length": 3090.7500610351562, |
| "epoch": 0.003427592116538132, |
| "grad_norm": 0.15128685534000397, |
| "kl": 0.0, |
| "learning_rate": 2e-07, |
| "loss": -0.0, |
| "reward": 0.013502426445484161, |
| "reward_std": 0.14039192721247673, |
| "rewards/cosine_scaled_reward": -0.10344066098332405, |
| "rewards/format_reward": 0.260416679084301, |
| "step": 2 |
| }, |
| { |
| "completion_length": 2600.1875610351562, |
| "epoch": 0.005141388174807198, |
| "grad_norm": 0.1910804808139801, |
| "kl": 3.987550735473633e-05, |
| "learning_rate": 3e-07, |
| "loss": 0.0, |
| "reward": 0.037791766226291656, |
| "reward_std": 0.1596581730991602, |
| "rewards/cosine_scaled_reward": -0.1509031739551574, |
| "rewards/format_reward": 0.4479166865348816, |
| "step": 3 |
| }, |
| { |
| "completion_length": 3076.8438110351562, |
| "epoch": 0.006855184233076264, |
| "grad_norm": 0.15531130135059357, |
| "kl": 4.374980926513672e-05, |
| "learning_rate": 4e-07, |
| "loss": 0.0, |
| "reward": -0.010211404645815492, |
| "reward_std": 0.18583546206355095, |
| "rewards/cosine_scaled_reward": -0.19252582639455795, |
| "rewards/format_reward": 0.3437500037252903, |
| "step": 4 |
| }, |
| { |
| "completion_length": 3045.604248046875, |
| "epoch": 0.00856898029134533, |
| "grad_norm": 0.11871814727783203, |
| "kl": 2.682209014892578e-05, |
| "learning_rate": 5e-07, |
| "loss": 0.0, |
| "reward": 0.0951396506279707, |
| "reward_std": 0.17187905684113503, |
| "rewards/cosine_scaled_reward": -0.026736530475318432, |
| "rewards/format_reward": 0.4270833358168602, |
| "step": 5 |
| }, |
| { |
| "completion_length": 2959.3438720703125, |
| "epoch": 0.010282776349614395, |
| "grad_norm": 0.13790491223335266, |
| "kl": 3.451108932495117e-05, |
| "learning_rate": 6e-07, |
| "loss": 0.0, |
| "reward": 0.11859842715784907, |
| "reward_std": 0.17868740856647491, |
| "rewards/cosine_scaled_reward": 0.010096830315887928, |
| "rewards/format_reward": 0.447916679084301, |
| "step": 6 |
| }, |
| { |
| "completion_length": 2884.6771850585938, |
| "epoch": 0.011996572407883462, |
| "grad_norm": 0.1611727625131607, |
| "kl": 2.6971101760864258e-05, |
| "learning_rate": 7e-07, |
| "loss": 0.0, |
| "reward": 0.03227711468935013, |
| "reward_std": 0.19281568750739098, |
| "rewards/cosine_scaled_reward": -0.12487578392028809, |
| "rewards/format_reward": 0.3750000149011612, |
| "step": 7 |
| }, |
| { |
| "completion_length": 2835.6666870117188, |
| "epoch": 0.013710368466152529, |
| "grad_norm": 0.1475144475698471, |
| "kl": 3.361701965332031e-05, |
| "learning_rate": 8e-07, |
| "loss": 0.0, |
| "reward": 0.045130204409360886, |
| "reward_std": 0.18995673581957817, |
| "rewards/cosine_scaled_reward": -0.13529739249497652, |
| "rewards/format_reward": 0.44791667722165585, |
| "step": 8 |
| }, |
| { |
| "completion_length": 3167.3541870117188, |
| "epoch": 0.015424164524421594, |
| "grad_norm": 0.13677193224430084, |
| "kl": 2.440810203552246e-05, |
| "learning_rate": 9e-07, |
| "loss": 0.0, |
| "reward": 0.020484148059040308, |
| "reward_std": 0.16632195003330708, |
| "rewards/cosine_scaled_reward": -0.11485888389870524, |
| "rewards/format_reward": 0.3125000111758709, |
| "step": 9 |
| }, |
| { |
| "completion_length": 2629.4584350585938, |
| "epoch": 0.01713796058269066, |
| "grad_norm": 0.14359042048454285, |
| "kl": 2.0831823348999023e-05, |
| "learning_rate": 1e-06, |
| "loss": 0.0, |
| "reward": 0.1312947366386652, |
| "reward_std": 0.1554134227335453, |
| "rewards/cosine_scaled_reward": 0.009967727586627007, |
| "rewards/format_reward": 0.5000000074505806, |
| "step": 10 |
| }, |
| { |
| "completion_length": 2906.1875610351562, |
| "epoch": 0.018851756640959727, |
| "grad_norm": 0.20071381330490112, |
| "kl": 3.3527612686157227e-05, |
| "learning_rate": 9.997258721585931e-07, |
| "loss": 0.0, |
| "reward": 0.033789366483688354, |
| "reward_std": 0.19262398406863213, |
| "rewards/cosine_scaled_reward": -0.08468496927525848, |
| "rewards/format_reward": 0.3020833460614085, |
| "step": 11 |
| }, |
| { |
| "completion_length": 2942.4168090820312, |
| "epoch": 0.02056555269922879, |
| "grad_norm": 0.15123063325881958, |
| "kl": 2.7477741241455078e-05, |
| "learning_rate": 9.989038226169207e-07, |
| "loss": 0.0, |
| "reward": 0.052781688049435616, |
| "reward_std": 0.19968328438699245, |
| "rewards/cosine_scaled_reward": -0.08891183231025934, |
| "rewards/format_reward": 0.3854166865348816, |
| "step": 12 |
| }, |
| { |
| "completion_length": 2924.7501220703125, |
| "epoch": 0.022279348757497857, |
| "grad_norm": 0.1488049179315567, |
| "kl": 1.4431774616241455e-05, |
| "learning_rate": 9.975348529157229e-07, |
| "loss": 0.0, |
| "reward": 0.1416405034251511, |
| "reward_std": 0.20871220529079437, |
| "rewards/cosine_scaled_reward": 0.05599783733487129, |
| "rewards/format_reward": 0.4479166865348816, |
| "step": 13 |
| }, |
| { |
| "completion_length": 2709.6250915527344, |
| "epoch": 0.023993144815766924, |
| "grad_norm": 0.18846973776817322, |
| "kl": 3.9637088775634766e-05, |
| "learning_rate": 9.956206309337066e-07, |
| "loss": 0.0, |
| "reward": 0.06054589990526438, |
| "reward_std": 0.1687684953212738, |
| "rewards/cosine_scaled_reward": -0.10551007091999054, |
| "rewards/format_reward": 0.4479166865348816, |
| "step": 14 |
| }, |
| { |
| "completion_length": 1950.8646545410156, |
| "epoch": 0.02570694087403599, |
| "grad_norm": 0.18432216346263885, |
| "kl": 4.6759843826293945e-05, |
| "learning_rate": 9.931634888554935e-07, |
| "loss": 0.0, |
| "reward": 0.23200466111302376, |
| "reward_std": 0.2355622537434101, |
| "rewards/cosine_scaled_reward": 0.06849630922079086, |
| "rewards/format_reward": 0.7708333432674408, |
| "step": 15 |
| }, |
| { |
| "completion_length": 3007.6875, |
| "epoch": 0.027420736932305057, |
| "grad_norm": 0.15038132667541504, |
| "kl": 3.84598970413208e-05, |
| "learning_rate": 9.901664203302124e-07, |
| "loss": 0.0, |
| "reward": 0.06949906051158905, |
| "reward_std": 0.21871956065297127, |
| "rewards/cosine_scaled_reward": -0.07155923545360565, |
| "rewards/format_reward": 0.416666679084301, |
| "step": 16 |
| }, |
| { |
| "completion_length": 2888.8438110351562, |
| "epoch": 0.02913453299057412, |
| "grad_norm": 0.17018933594226837, |
| "kl": 5.860626697540283e-05, |
| "learning_rate": 9.866330768241983e-07, |
| "loss": 0.0, |
| "reward": 0.08268930693157017, |
| "reward_std": 0.2000725194811821, |
| "rewards/cosine_scaled_reward": -0.03420639084652066, |
| "rewards/format_reward": 0.3958333469927311, |
| "step": 17 |
| }, |
| { |
| "completion_length": 2994.1666870117188, |
| "epoch": 0.030848329048843187, |
| "grad_norm": 0.14627192914485931, |
| "kl": 3.9910897612571716e-05, |
| "learning_rate": 9.825677631722435e-07, |
| "loss": 0.0, |
| "reward": 0.1066256552003324, |
| "reward_std": 0.1765430960804224, |
| "rewards/cosine_scaled_reward": -0.012849166989326477, |
| "rewards/format_reward": 0.447916679084301, |
| "step": 18 |
| }, |
| { |
| "completion_length": 2986.885498046875, |
| "epoch": 0.032562125107112254, |
| "grad_norm": 0.11996503919363022, |
| "kl": 7.943809032440186e-05, |
| "learning_rate": 9.779754323328192e-07, |
| "loss": 0.0, |
| "reward": 0.037288530729711056, |
| "reward_std": 0.1901964209973812, |
| "rewards/cosine_scaled_reward": -0.08846686349716038, |
| "rewards/format_reward": 0.3229166753590107, |
| "step": 19 |
| }, |
| { |
| "completion_length": 3068.0000610351562, |
| "epoch": 0.03427592116538132, |
| "grad_norm": 0.15175943076610565, |
| "kl": 0.00019240379333496094, |
| "learning_rate": 9.728616793536587e-07, |
| "loss": 0.0, |
| "reward": 0.03987372852861881, |
| "reward_std": 0.18592991679906845, |
| "rewards/cosine_scaled_reward": -0.11358156427741051, |
| "rewards/format_reward": 0.3854166828095913, |
| "step": 20 |
| }, |
| { |
| "completion_length": 3086.6771240234375, |
| "epoch": 0.03598971722365039, |
| "grad_norm": 0.1517907977104187, |
| "kl": 0.00036978721618652344, |
| "learning_rate": 9.672327345550543e-07, |
| "loss": 0.0, |
| "reward": 0.03412920422852039, |
| "reward_std": 0.1651427522301674, |
| "rewards/cosine_scaled_reward": -0.09963592886924744, |
| "rewards/format_reward": 0.33333334419876337, |
| "step": 21 |
| }, |
| { |
| "completion_length": 3277.1458740234375, |
| "epoch": 0.037703513281919454, |
| "grad_norm": 0.11310048401355743, |
| "kl": 0.00019124150276184082, |
| "learning_rate": 9.610954559391704e-07, |
| "loss": 0.0, |
| "reward": -0.0021694526076316833, |
| "reward_std": 0.1807849071919918, |
| "rewards/cosine_scaled_reward": -0.1448658797889948, |
| "rewards/format_reward": 0.2812500074505806, |
| "step": 22 |
| }, |
| { |
| "completion_length": 2485.1875915527344, |
| "epoch": 0.03941730934018852, |
| "grad_norm": 0.21775458753108978, |
| "kl": 0.0002810359001159668, |
| "learning_rate": 9.54457320834625e-07, |
| "loss": 0.0, |
| "reward": 0.1259636478498578, |
| "reward_std": 0.21954844892024994, |
| "rewards/cosine_scaled_reward": 0.0026862788945436478, |
| "rewards/format_reward": 0.4895833395421505, |
| "step": 23 |
| }, |
| { |
| "completion_length": 3387.8646240234375, |
| "epoch": 0.04113110539845758, |
| "grad_norm": 0.11246265470981598, |
| "kl": 0.00029206275939941406, |
| "learning_rate": 9.473264167865171e-07, |
| "loss": 0.0, |
| "reward": -0.0444450975919608, |
| "reward_std": 0.19019928015768528, |
| "rewards/cosine_scaled_reward": -0.17542381957173347, |
| "rewards/format_reward": 0.1770833358168602, |
| "step": 24 |
| }, |
| { |
| "completion_length": 3383.3229370117188, |
| "epoch": 0.04284490145672665, |
| "grad_norm": 0.11982468515634537, |
| "kl": 0.00018139183521270752, |
| "learning_rate": 9.397114317029974e-07, |
| "loss": 0.0, |
| "reward": -0.08237307704985142, |
| "reward_std": 0.10484589636325836, |
| "rewards/cosine_scaled_reward": -0.2248186320066452, |
| "rewards/format_reward": 0.1250000074505806, |
| "step": 25 |
| }, |
| { |
| "completion_length": 2839.0625610351562, |
| "epoch": 0.044558697514995714, |
| "grad_norm": 0.1347901076078415, |
| "kl": 0.0003955364227294922, |
| "learning_rate": 9.316216432703916e-07, |
| "loss": 0.0, |
| "reward": 0.11097948951646686, |
| "reward_std": 0.18024785444140434, |
| "rewards/cosine_scaled_reward": 0.0011768988333642483, |
| "rewards/format_reward": 0.4375000111758709, |
| "step": 26 |
| }, |
| { |
| "completion_length": 2636.5729370117188, |
| "epoch": 0.04627249357326478, |
| "grad_norm": 0.13496308028697968, |
| "kl": 0.0006954669952392578, |
| "learning_rate": 9.230669076497687e-07, |
| "loss": 0.0, |
| "reward": 0.05549928080290556, |
| "reward_std": 0.14237392507493496, |
| "rewards/cosine_scaled_reward": -0.12638001888990402, |
| "rewards/format_reward": 0.4687500111758709, |
| "step": 27 |
| }, |
| { |
| "completion_length": 2851.2396240234375, |
| "epoch": 0.04798628963153385, |
| "grad_norm": 0.16802968084812164, |
| "kl": 0.00039768218994140625, |
| "learning_rate": 9.140576474687263e-07, |
| "loss": 0.0, |
| "reward": 0.005757941631600261, |
| "reward_std": 0.16327618807554245, |
| "rewards/cosine_scaled_reward": -0.18160032108426094, |
| "rewards/format_reward": 0.385416679084301, |
| "step": 28 |
| }, |
| { |
| "completion_length": 2759.5208740234375, |
| "epoch": 0.049700085689802914, |
| "grad_norm": 0.1442675143480301, |
| "kl": 0.0005460977554321289, |
| "learning_rate": 9.046048391230247e-07, |
| "loss": 0.0, |
| "reward": 0.09482090920209885, |
| "reward_std": 0.16877998411655426, |
| "rewards/cosine_scaled_reward": -0.010849054902791977, |
| "rewards/format_reward": 0.3958333432674408, |
| "step": 29 |
| }, |
| { |
| "completion_length": 3377.59375, |
| "epoch": 0.05141388174807198, |
| "grad_norm": 0.12018371373414993, |
| "kl": 0.0006362795829772949, |
| "learning_rate": 8.9471999940354e-07, |
| "loss": 0.0, |
| "reward": 0.028713669627904892, |
| "reward_std": 0.18587654829025269, |
| "rewards/cosine_scaled_reward": -0.07821773737668991, |
| "rewards/format_reward": 0.2708333358168602, |
| "step": 30 |
| }, |
| { |
| "completion_length": 3138.104248046875, |
| "epoch": 0.05312767780634105, |
| "grad_norm": 0.1430775374174118, |
| "kl": 0.0004627704620361328, |
| "learning_rate": 8.844151714648274e-07, |
| "loss": 0.0, |
| "reward": -0.008235585875809193, |
| "reward_std": 0.14394641481339931, |
| "rewards/cosine_scaled_reward": -0.13156688585877419, |
| "rewards/format_reward": 0.22916666977107525, |
| "step": 31 |
| }, |
| { |
| "completion_length": 2630.5938720703125, |
| "epoch": 0.054841473864610114, |
| "grad_norm": 0.17797145247459412, |
| "kl": 0.0012159347534179688, |
| "learning_rate": 8.737029101523929e-07, |
| "loss": 0.0, |
| "reward": 0.13316596718505025, |
| "reward_std": 0.22369278594851494, |
| "rewards/cosine_scaled_reward": -0.002324562519788742, |
| "rewards/format_reward": 0.5312500149011612, |
| "step": 32 |
| }, |
| { |
| "completion_length": 2592.5000610351562, |
| "epoch": 0.056555269922879174, |
| "grad_norm": 0.15806646645069122, |
| "kl": 0.0012989044189453125, |
| "learning_rate": 8.625962667065487e-07, |
| "loss": 0.0001, |
| "reward": 0.11066696606576443, |
| "reward_std": 0.14327863790094852, |
| "rewards/cosine_scaled_reward": -0.010827179066836834, |
| "rewards/format_reward": 0.4583333432674408, |
| "step": 33 |
| }, |
| { |
| "completion_length": 2626.5313110351562, |
| "epoch": 0.05826906598114824, |
| "grad_norm": 0.1691393107175827, |
| "kl": 0.0020818710327148438, |
| "learning_rate": 8.511087728614862e-07, |
| "loss": 0.0001, |
| "reward": 0.09107339568436146, |
| "reward_std": 0.18131284788250923, |
| "rewards/cosine_scaled_reward": -0.04873488505836576, |
| "rewards/format_reward": 0.4583333507180214, |
| "step": 34 |
| }, |
| { |
| "completion_length": 3051.9896850585938, |
| "epoch": 0.05998286203941731, |
| "grad_norm": 0.17819511890411377, |
| "kl": 0.0019023418426513672, |
| "learning_rate": 8.392544243589427e-07, |
| "loss": 0.0001, |
| "reward": 0.10884217114653438, |
| "reward_std": 0.23975158482789993, |
| "rewards/cosine_scaled_reward": 0.007055521011352539, |
| "rewards/format_reward": 0.416666679084301, |
| "step": 35 |
| }, |
| { |
| "completion_length": 2809.8646240234375, |
| "epoch": 0.061696658097686374, |
| "grad_norm": 0.14143270254135132, |
| "kl": 0.0004401206970214844, |
| "learning_rate": 8.270476638965461e-07, |
| "loss": 0.0, |
| "reward": 0.15133970603346825, |
| "reward_std": 0.25536274164915085, |
| "rewards/cosine_scaled_reward": 0.04777143523097038, |
| "rewards/format_reward": 0.5000000149011612, |
| "step": 36 |
| }, |
| { |
| "completion_length": 3153.0938110351562, |
| "epoch": 0.06341045415595545, |
| "grad_norm": 0.12571778893470764, |
| "kl": 0.0004382133483886719, |
| "learning_rate": 8.145033635316128e-07, |
| "loss": 0.0, |
| "reward": 0.020705484319478273, |
| "reward_std": 0.17584117874503136, |
| "rewards/cosine_scaled_reward": -0.13113276287913322, |
| "rewards/format_reward": 0.3437500074505806, |
| "step": 37 |
| }, |
| { |
| "completion_length": 3118.322998046875, |
| "epoch": 0.06512425021422451, |
| "grad_norm": 0.11900849640369415, |
| "kl": 0.00022101402282714844, |
| "learning_rate": 8.01636806561836e-07, |
| "loss": 0.0, |
| "reward": 0.07037997618317604, |
| "reward_std": 0.2210748754441738, |
| "rewards/cosine_scaled_reward": -0.028155604377388954, |
| "rewards/format_reward": 0.33333333395421505, |
| "step": 38 |
| }, |
| { |
| "completion_length": 2472.3646240234375, |
| "epoch": 0.06683804627249357, |
| "grad_norm": 0.1329745203256607, |
| "kl": 0.0012660026550292969, |
| "learning_rate": 7.884636689049422e-07, |
| "loss": 0.0001, |
| "reward": 0.11453905934467912, |
| "reward_std": 0.20037821307778358, |
| "rewards/cosine_scaled_reward": -0.056616831570863724, |
| "rewards/format_reward": 0.5625000037252903, |
| "step": 39 |
| }, |
| { |
| "completion_length": 3018.072998046875, |
| "epoch": 0.06855184233076264, |
| "grad_norm": 0.11555609107017517, |
| "kl": 0.00041091442108154297, |
| "learning_rate": 7.75e-07, |
| "loss": 0.0, |
| "reward": 0.01033751294016838, |
| "reward_std": 0.1491969907656312, |
| "rewards/cosine_scaled_reward": -0.13083185872528702, |
| "rewards/format_reward": 0.3020833386108279, |
| "step": 40 |
| }, |
| { |
| "completion_length": 2939.229248046875, |
| "epoch": 0.0702656383890317, |
| "grad_norm": 0.12907516956329346, |
| "kl": 0.0007364749908447266, |
| "learning_rate": 7.612622032536507e-07, |
| "loss": 0.0, |
| "reward": 0.12070174579275772, |
| "reward_std": 0.2728600651025772, |
| "rewards/cosine_scaled_reward": -0.001188491762150079, |
| "rewards/format_reward": 0.479166679084301, |
| "step": 41 |
| }, |
| { |
| "completion_length": 2293.854248046875, |
| "epoch": 0.07197943444730077, |
| "grad_norm": 0.15430894494056702, |
| "kl": 0.001895904541015625, |
| "learning_rate": 7.472670160550848e-07, |
| "loss": 0.0001, |
| "reward": 0.1563777169212699, |
| "reward_std": 0.2286355346441269, |
| "rewards/cosine_scaled_reward": -0.015645149163901806, |
| "rewards/format_reward": 0.6458333432674408, |
| "step": 42 |
| }, |
| { |
| "completion_length": 2741.0833740234375, |
| "epoch": 0.07369323050556983, |
| "grad_norm": 0.11995150148868561, |
| "kl": 0.0004520416259765625, |
| "learning_rate": 7.330314893841101e-07, |
| "loss": 0.0, |
| "reward": 0.13986177369952202, |
| "reward_std": 0.22300692275166512, |
| "rewards/cosine_scaled_reward": 0.05043928697705269, |
| "rewards/format_reward": 0.4479166753590107, |
| "step": 43 |
| }, |
| { |
| "completion_length": 2305.854217529297, |
| "epoch": 0.07540702656383891, |
| "grad_norm": 0.21461769938468933, |
| "kl": 0.0007772445678710938, |
| "learning_rate": 7.185729670371604e-07, |
| "loss": 0.0, |
| "reward": 0.07035261578857899, |
| "reward_std": 0.13939796388149261, |
| "rewards/cosine_scaled_reward": -0.09040920436382294, |
| "rewards/format_reward": 0.4583333386108279, |
| "step": 44 |
| }, |
| { |
| "completion_length": 2846.8958435058594, |
| "epoch": 0.07712082262210797, |
| "grad_norm": 0.13480690121650696, |
| "kl": 0.0009918212890625, |
| "learning_rate": 7.039090644965509e-07, |
| "loss": 0.0, |
| "reward": -0.018673544749617577, |
| "reward_std": 0.16758902743458748, |
| "rewards/cosine_scaled_reward": -0.20985213853418827, |
| "rewards/format_reward": 0.3437500149011612, |
| "step": 45 |
| }, |
| { |
| "completion_length": 2584.1563110351562, |
| "epoch": 0.07883461868037704, |
| "grad_norm": 0.17533442378044128, |
| "kl": 0.0022344589233398438, |
| "learning_rate": 6.890576474687263e-07, |
| "loss": 0.0001, |
| "reward": 0.022231539245694876, |
| "reward_std": 0.16662119142711163, |
| "rewards/cosine_scaled_reward": -0.17729704454541206, |
| "rewards/format_reward": 0.4375000223517418, |
| "step": 46 |
| }, |
| { |
| "completion_length": 3102.354248046875, |
| "epoch": 0.0805484147386461, |
| "grad_norm": 0.1247975304722786, |
| "kl": 0.0013914108276367188, |
| "learning_rate": 6.740368101176495e-07, |
| "loss": 0.0001, |
| "reward": -0.00813557580113411, |
| "reward_std": 0.1496633905917406, |
| "rewards/cosine_scaled_reward": -0.17180366069078445, |
| "rewards/format_reward": 0.31250000838190317, |
| "step": 47 |
| }, |
| { |
| "completion_length": 2603.8438415527344, |
| "epoch": 0.08226221079691516, |
| "grad_norm": 0.16067557036876678, |
| "kl": 0.0016384124755859375, |
| "learning_rate": 6.588648530198504e-07, |
| "loss": 0.0001, |
| "reward": 0.11067175259813666, |
| "reward_std": 0.179900873452425, |
| "rewards/cosine_scaled_reward": -0.013014900498092175, |
| "rewards/format_reward": 0.4583333507180214, |
| "step": 48 |
| }, |
| { |
| "completion_length": 3373.0313110351562, |
| "epoch": 0.08397600685518423, |
| "grad_norm": 0.10881118476390839, |
| "kl": 0.0002281665802001953, |
| "learning_rate": 6.435602608679916e-07, |
| "loss": 0.0, |
| "reward": 0.00016227364540100098, |
| "reward_std": 0.14608008973300457, |
| "rewards/cosine_scaled_reward": -0.10383528657257557, |
| "rewards/format_reward": 0.2083333358168602, |
| "step": 49 |
| }, |
| { |
| "completion_length": 3033.0208740234375, |
| "epoch": 0.0856898029134533, |
| "grad_norm": 0.13846911489963531, |
| "kl": 0.0012478828430175781, |
| "learning_rate": 6.281416799501187e-07, |
| "loss": 0.0001, |
| "reward": 0.06065037357620895, |
| "reward_std": 0.17429610714316368, |
| "rewards/cosine_scaled_reward": -0.05210728943347931, |
| "rewards/format_reward": 0.3437500074505806, |
| "step": 50 |
| }, |
| { |
| "completion_length": 2933.4793090820312, |
| "epoch": 0.08740359897172237, |
| "grad_norm": 0.13350166380405426, |
| "kl": 0.00033092498779296875, |
| "learning_rate": 6.126278954320294e-07, |
| "loss": 0.0, |
| "reward": -0.01886086305603385, |
| "reward_std": 0.13372720032930374, |
| "rewards/cosine_scaled_reward": -0.23485597968101501, |
| "rewards/format_reward": 0.3958333507180214, |
| "step": 51 |
| }, |
| { |
| "completion_length": 3236.791748046875, |
| "epoch": 0.08911739502999143, |
| "grad_norm": 0.10703922808170319, |
| "kl": 0.0003795623779296875, |
| "learning_rate": 5.97037808470444e-07, |
| "loss": 0.0, |
| "reward": 0.0947269550524652, |
| "reward_std": 0.2209848240017891, |
| "rewards/cosine_scaled_reward": 0.004282929934561253, |
| "rewards/format_reward": 0.3645833432674408, |
| "step": 52 |
| }, |
| { |
| "completion_length": 2535.1875610351562, |
| "epoch": 0.0908311910882605, |
| "grad_norm": 0.19642022252082825, |
| "kl": 0.0011463165283203125, |
| "learning_rate": 5.813904131848564e-07, |
| "loss": 0.0, |
| "reward": 0.074245510622859, |
| "reward_std": 0.20659937523305416, |
| "rewards/cosine_scaled_reward": -0.10727175627835095, |
| "rewards/format_reward": 0.5000000074505806, |
| "step": 53 |
| }, |
| { |
| "completion_length": 3126.1146240234375, |
| "epoch": 0.09254498714652956, |
| "grad_norm": 0.17827318608760834, |
| "kl": 0.001310110092163086, |
| "learning_rate": 5.657047735161255e-07, |
| "loss": 0.0001, |
| "reward": 0.0032570459879934788, |
| "reward_std": 0.15636450424790382, |
| "rewards/cosine_scaled_reward": -0.14380789827555418, |
| "rewards/format_reward": 0.30208333395421505, |
| "step": 54 |
| }, |
| { |
| "completion_length": 2929.9688720703125, |
| "epoch": 0.09425878320479864, |
| "grad_norm": 0.1380075216293335, |
| "kl": 0.001453399658203125, |
| "learning_rate": 5.5e-07, |
| "loss": 0.0001, |
| "reward": 0.045829253271222115, |
| "reward_std": 0.1648220308125019, |
| "rewards/cosine_scaled_reward": -0.06621671980246902, |
| "rewards/format_reward": 0.31250000558793545, |
| "step": 55 |
| }, |
| { |
| "completion_length": 2777.7083435058594, |
| "epoch": 0.0959725792630677, |
| "grad_norm": 0.15538543462753296, |
| "kl": 0.0006000995635986328, |
| "learning_rate": 5.342952264838747e-07, |
| "loss": 0.0, |
| "reward": 0.05388350097928196, |
| "reward_std": 0.1790449097752571, |
| "rewards/cosine_scaled_reward": -0.09807816706597805, |
| "rewards/format_reward": 0.4062500186264515, |
| "step": 56 |
| }, |
| { |
| "completion_length": 3187.697998046875, |
| "epoch": 0.09768637532133675, |
| "grad_norm": 0.10485029965639114, |
| "kl": 0.0006384849548339844, |
| "learning_rate": 5.186095868151436e-07, |
| "loss": 0.0, |
| "reward": 0.02806462941225618, |
| "reward_std": 0.1900525726377964, |
| "rewards/cosine_scaled_reward": -0.10978525504469872, |
| "rewards/format_reward": 0.3333333432674408, |
| "step": 57 |
| }, |
| { |
| "completion_length": 2514.208465576172, |
| "epoch": 0.09940017137960583, |
| "grad_norm": 0.15051576495170593, |
| "kl": 0.0010862350463867188, |
| "learning_rate": 5.02962191529556e-07, |
| "loss": 0.0, |
| "reward": 0.04878209065645933, |
| "reward_std": 0.17859612964093685, |
| "rewards/cosine_scaled_reward": -0.17480848729610443, |
| "rewards/format_reward": 0.5416666865348816, |
| "step": 58 |
| }, |
| { |
| "completion_length": 2698.4896240234375, |
| "epoch": 0.10111396743787489, |
| "grad_norm": 0.17716623842716217, |
| "kl": 0.00115966796875, |
| "learning_rate": 4.873721045679706e-07, |
| "loss": 0.0, |
| "reward": 0.18329142779111862, |
| "reward_std": 0.27025362849235535, |
| "rewards/cosine_scaled_reward": 0.0847717362921685, |
| "rewards/format_reward": 0.5520833507180214, |
| "step": 59 |
| }, |
| { |
| "completion_length": 2798.9375610351562, |
| "epoch": 0.10282776349614396, |
| "grad_norm": 0.17029182612895966, |
| "kl": 0.00177001953125, |
| "learning_rate": 4.7185832004988133e-07, |
| "loss": 0.0001, |
| "reward": 0.02042369358241558, |
| "reward_std": 0.13106655701994896, |
| "rewards/cosine_scaled_reward": -0.17489198595285416, |
| "rewards/format_reward": 0.4270833395421505, |
| "step": 60 |
| }, |
| { |
| "completion_length": 2818.84375, |
| "epoch": 0.10454155955441302, |
| "grad_norm": 0.14482490718364716, |
| "kl": 0.0011817216873168945, |
| "learning_rate": 4.5643973913200837e-07, |
| "loss": 0.0, |
| "reward": 0.06702410569414496, |
| "reward_std": 0.16219814494252205, |
| "rewards/cosine_scaled_reward": -0.07603749074041843, |
| "rewards/format_reward": 0.4166666865348816, |
| "step": 61 |
| }, |
| { |
| "completion_length": 3321.2604370117188, |
| "epoch": 0.1062553556126821, |
| "grad_norm": 0.12993496656417847, |
| "kl": 0.0014019012451171875, |
| "learning_rate": 4.4113514698014953e-07, |
| "loss": 0.0001, |
| "reward": -0.06851966120302677, |
| "reward_std": 0.1139833852648735, |
| "rewards/cosine_scaled_reward": -0.20644466951489449, |
| "rewards/format_reward": 0.14583333861082792, |
| "step": 62 |
| }, |
| { |
| "completion_length": 3189.135498046875, |
| "epoch": 0.10796915167095116, |
| "grad_norm": 0.1303822249174118, |
| "kl": 0.0011129379272460938, |
| "learning_rate": 4.2596318988235037e-07, |
| "loss": 0.0, |
| "reward": -0.018826739862561226, |
| "reward_std": 0.13178249262273312, |
| "rewards/cosine_scaled_reward": -0.13564759586006403, |
| "rewards/format_reward": 0.19791666977107525, |
| "step": 63 |
| }, |
| { |
| "completion_length": 2808.697998046875, |
| "epoch": 0.10968294772922023, |
| "grad_norm": 0.1368769407272339, |
| "kl": 0.0009212493896484375, |
| "learning_rate": 4.1094235253127374e-07, |
| "loss": 0.0, |
| "reward": 0.05554246995598078, |
| "reward_std": 0.17365428805351257, |
| "rewards/cosine_scaled_reward": -0.0875049065798521, |
| "rewards/format_reward": 0.3958333460614085, |
| "step": 64 |
| }, |
| { |
| "completion_length": 3047.6771240234375, |
| "epoch": 0.11139674378748929, |
| "grad_norm": 0.11966266483068466, |
| "kl": 0.0005755424499511719, |
| "learning_rate": 3.9609093550344907e-07, |
| "loss": 0.0, |
| "reward": 0.08035540394484997, |
| "reward_std": 0.1835373416543007, |
| "rewards/cosine_scaled_reward": -0.03351259673945606, |
| "rewards/format_reward": 0.385416679084301, |
| "step": 65 |
| }, |
| { |
| "completion_length": 2938.4166870117188, |
| "epoch": 0.11311053984575835, |
| "grad_norm": 0.15843810141086578, |
| "kl": 0.0007996559143066406, |
| "learning_rate": 3.8142703296283953e-07, |
| "loss": 0.0, |
| "reward": 0.02124944794923067, |
| "reward_std": 0.14316299185156822, |
| "rewards/cosine_scaled_reward": -0.09920864924788475, |
| "rewards/format_reward": 0.28125, |
| "step": 66 |
| }, |
| { |
| "completion_length": 2877.0834350585938, |
| "epoch": 0.11482433590402742, |
| "grad_norm": 0.16664999723434448, |
| "kl": 0.0018649101257324219, |
| "learning_rate": 3.6696851061588994e-07, |
| "loss": 0.0001, |
| "reward": 0.023960473015904427, |
| "reward_std": 0.17024591751396656, |
| "rewards/cosine_scaled_reward": -0.12505127862095833, |
| "rewards/format_reward": 0.3437500102445483, |
| "step": 67 |
| }, |
| { |
| "completion_length": 2517.4479370117188, |
| "epoch": 0.11653813196229648, |
| "grad_norm": 0.18517287075519562, |
| "kl": 0.0015734434127807617, |
| "learning_rate": 3.5273298394491515e-07, |
| "loss": 0.0001, |
| "reward": 0.12474549748003483, |
| "reward_std": 0.17121394164860249, |
| "rewards/cosine_scaled_reward": -0.03634676709771156, |
| "rewards/format_reward": 0.5625000074505806, |
| "step": 68 |
| }, |
| { |
| "completion_length": 2816.3646850585938, |
| "epoch": 0.11825192802056556, |
| "grad_norm": 0.16400621831417084, |
| "kl": 0.0013251304626464844, |
| "learning_rate": 3.387377967463493e-07, |
| "loss": 0.0001, |
| "reward": 0.009894740185700357, |
| "reward_std": 0.17024565115571022, |
| "rewards/cosine_scaled_reward": -0.16932371503207833, |
| "rewards/format_reward": 0.375, |
| "step": 69 |
| }, |
| { |
| "completion_length": 2789.4168090820312, |
| "epoch": 0.11996572407883462, |
| "grad_norm": 0.1394311487674713, |
| "kl": 0.0010738372802734375, |
| "learning_rate": 3.250000000000001e-07, |
| "loss": 0.0, |
| "reward": 0.1316700098104775, |
| "reward_std": 0.2457997277379036, |
| "rewards/cosine_scaled_reward": 0.04584646038711071, |
| "rewards/format_reward": 0.4270833395421505, |
| "step": 70 |
| }, |
| { |
| "completion_length": 2267.166717529297, |
| "epoch": 0.12167952013710369, |
| "grad_norm": 0.15189269185066223, |
| "kl": 0.001094818115234375, |
| "learning_rate": 3.115363310950578e-07, |
| "loss": 0.0, |
| "reward": 0.24627735652029514, |
| "reward_std": 0.1724311113357544, |
| "rewards/cosine_scaled_reward": 0.16811909526586533, |
| "rewards/format_reward": 0.6354166865348816, |
| "step": 71 |
| }, |
| { |
| "completion_length": 3065.7501220703125, |
| "epoch": 0.12339331619537275, |
| "grad_norm": 0.1408892422914505, |
| "kl": 0.0006256103515625, |
| "learning_rate": 2.9836319343816397e-07, |
| "loss": 0.0, |
| "reward": 0.0009416388347744942, |
| "reward_std": 0.1657393015921116, |
| "rewards/cosine_scaled_reward": -0.15875469404272735, |
| "rewards/format_reward": 0.3229166744276881, |
| "step": 72 |
| }, |
| { |
| "completion_length": 2957.0625610351562, |
| "epoch": 0.12510711225364182, |
| "grad_norm": 0.13583190739154816, |
| "kl": 0.00033473968505859375, |
| "learning_rate": 2.854966364683872e-07, |
| "loss": 0.0, |
| "reward": 0.06538257002830505, |
| "reward_std": 0.1970806121826172, |
| "rewards/cosine_scaled_reward": -0.09460783936083317, |
| "rewards/format_reward": 0.447916679084301, |
| "step": 73 |
| }, |
| { |
| "completion_length": 3106.010498046875, |
| "epoch": 0.1268209083119109, |
| "grad_norm": 0.15821737051010132, |
| "kl": 0.0009412765502929688, |
| "learning_rate": 2.729523361034538e-07, |
| "loss": 0.0, |
| "reward": 0.0028651878237724304, |
| "reward_std": 0.19838279485702515, |
| "rewards/cosine_scaled_reward": -0.13493257109075785, |
| "rewards/format_reward": 0.2812500111758709, |
| "step": 74 |
| }, |
| { |
| "completion_length": 2847.541748046875, |
| "epoch": 0.12853470437017994, |
| "grad_norm": 0.16942313313484192, |
| "kl": 0.0019121170043945312, |
| "learning_rate": 2.6074557564105724e-07, |
| "loss": 0.0001, |
| "reward": 0.0911966897547245, |
| "reward_std": 0.21281348168849945, |
| "rewards/cosine_scaled_reward": -0.028547534719109535, |
| "rewards/format_reward": 0.41666668467223644, |
| "step": 75 |
| }, |
| { |
| "completion_length": 2846.9896850585938, |
| "epoch": 0.13024850042844902, |
| "grad_norm": 0.1670897901058197, |
| "kl": 0.0010271072387695312, |
| "learning_rate": 2.488912271385139e-07, |
| "loss": 0.0, |
| "reward": 0.02618580963462591, |
| "reward_std": 0.15880979225039482, |
| "rewards/cosine_scaled_reward": -0.18800780980382115, |
| "rewards/format_reward": 0.4791666828095913, |
| "step": 76 |
| }, |
| { |
| "completion_length": 3011.2188110351562, |
| "epoch": 0.1319622964867181, |
| "grad_norm": 0.1846207082271576, |
| "kl": 0.0016541481018066406, |
| "learning_rate": 2.374037332934512e-07, |
| "loss": 0.0001, |
| "reward": 0.01657121942844242, |
| "reward_std": 0.19087375327944756, |
| "rewards/cosine_scaled_reward": -0.10442618932574987, |
| "rewards/format_reward": 0.27083334513008595, |
| "step": 77 |
| }, |
| { |
| "completion_length": 3298.625, |
| "epoch": 0.13367609254498714, |
| "grad_norm": 0.12679187953472137, |
| "kl": 0.0008182525634765625, |
| "learning_rate": 2.2629708984760706e-07, |
| "loss": 0.0, |
| "reward": -0.07863575895316899, |
| "reward_std": 0.11674198694527149, |
| "rewards/cosine_scaled_reward": -0.24837390705943108, |
| "rewards/format_reward": 0.1875000074505806, |
| "step": 78 |
| }, |
| { |
| "completion_length": 3014.0521850585938, |
| "epoch": 0.1353898886032562, |
| "grad_norm": 0.19050873816013336, |
| "kl": 0.0012989044189453125, |
| "learning_rate": 2.1558482853517253e-07, |
| "loss": 0.0001, |
| "reward": 0.14426555391401052, |
| "reward_std": 0.21018010936677456, |
| "rewards/cosine_scaled_reward": 0.08679820597171783, |
| "rewards/format_reward": 0.3958333507180214, |
| "step": 79 |
| }, |
| { |
| "completion_length": 2568.697998046875, |
| "epoch": 0.13710368466152528, |
| "grad_norm": 0.14511092007160187, |
| "kl": 0.0016317367553710938, |
| "learning_rate": 2.0528000059645995e-07, |
| "loss": 0.0001, |
| "reward": 0.15173329645767808, |
| "reward_std": 0.1817643828690052, |
| "rewards/cosine_scaled_reward": 0.00575945433229208, |
| "rewards/format_reward": 0.5833333432674408, |
| "step": 80 |
| }, |
| { |
| "completion_length": 2273.5209045410156, |
| "epoch": 0.13881748071979436, |
| "grad_norm": 0.1482679694890976, |
| "kl": 0.0021762847900390625, |
| "learning_rate": 1.9539516087697517e-07, |
| "loss": 0.0001, |
| "reward": 0.17546646296977997, |
| "reward_std": 0.18799066171050072, |
| "rewards/cosine_scaled_reward": 0.05814665276557207, |
| "rewards/format_reward": 0.5729166865348816, |
| "step": 81 |
| }, |
| { |
| "completion_length": 2761.7709350585938, |
| "epoch": 0.1405312767780634, |
| "grad_norm": 0.14379660785198212, |
| "kl": 0.000904083251953125, |
| "learning_rate": 1.8594235253127372e-07, |
| "loss": 0.0, |
| "reward": 0.09056159481406212, |
| "reward_std": 0.19218357279896736, |
| "rewards/cosine_scaled_reward": -0.07696967083029449, |
| "rewards/format_reward": 0.5104166865348816, |
| "step": 82 |
| }, |
| { |
| "completion_length": 2424.1666870117188, |
| "epoch": 0.14224507283633248, |
| "grad_norm": 0.18656377494335175, |
| "kl": 0.0023345947265625, |
| "learning_rate": 1.7693309235023127e-07, |
| "loss": 0.0001, |
| "reward": 0.17556490686547477, |
| "reward_std": 0.24106226861476898, |
| "rewards/cosine_scaled_reward": 0.06774241663515568, |
| "rewards/format_reward": 0.5520833432674408, |
| "step": 83 |
| }, |
| { |
| "completion_length": 2954.041748046875, |
| "epoch": 0.14395886889460155, |
| "grad_norm": 0.12463019788265228, |
| "kl": 0.0012934207916259766, |
| "learning_rate": 1.6837835672960831e-07, |
| "loss": 0.0001, |
| "reward": 0.07739502191543579, |
| "reward_std": 0.14365380816161633, |
| "rewards/cosine_scaled_reward": -0.024237127974629402, |
| "rewards/format_reward": 0.3541666716337204, |
| "step": 84 |
| }, |
| { |
| "completion_length": 3007.1771850585938, |
| "epoch": 0.1456726649528706, |
| "grad_norm": 0.1296854168176651, |
| "kl": 0.0006504058837890625, |
| "learning_rate": 1.6028856829700258e-07, |
| "loss": 0.0, |
| "reward": 0.06423507491126657, |
| "reward_std": 0.18408943712711334, |
| "rewards/cosine_scaled_reward": -0.08204184658825397, |
| "rewards/format_reward": 0.416666679084301, |
| "step": 85 |
| }, |
| { |
| "completion_length": 3371.8646240234375, |
| "epoch": 0.14738646101113967, |
| "grad_norm": 0.1433609277009964, |
| "kl": 0.0012350082397460938, |
| "learning_rate": 1.5267358321348285e-07, |
| "loss": 0.0, |
| "reward": -0.03188338316977024, |
| "reward_std": 0.16921599954366684, |
| "rewards/cosine_scaled_reward": -0.13588787615299225, |
| "rewards/format_reward": 0.14583333861082792, |
| "step": 86 |
| }, |
| { |
| "completion_length": 3052.2501220703125, |
| "epoch": 0.14910025706940874, |
| "grad_norm": 0.13880982995033264, |
| "kl": 0.001392364501953125, |
| "learning_rate": 1.4554267916537495e-07, |
| "loss": 0.0001, |
| "reward": -0.018908033147454262, |
| "reward_std": 0.1546711027622223, |
| "rewards/cosine_scaled_reward": -0.18234001845121384, |
| "rewards/format_reward": 0.2916666753590107, |
| "step": 87 |
| }, |
| { |
| "completion_length": 2693.0626220703125, |
| "epoch": 0.15081405312767782, |
| "grad_norm": 0.15144215524196625, |
| "kl": 0.0007927417755126953, |
| "learning_rate": 1.3890454406082956e-07, |
| "loss": 0.0, |
| "reward": 0.12248719483613968, |
| "reward_std": 0.18367315083742142, |
| "rewards/cosine_scaled_reward": 0.043038370087742805, |
| "rewards/format_reward": 0.3958333507180214, |
| "step": 88 |
| }, |
| { |
| "completion_length": 3063.604248046875, |
| "epoch": 0.15252784918594686, |
| "grad_norm": 0.14191189408302307, |
| "kl": 0.0008802413940429688, |
| "learning_rate": 1.3276726544494571e-07, |
| "loss": 0.0, |
| "reward": 0.0365169154247269, |
| "reward_std": 0.15275101736187935, |
| "rewards/cosine_scaled_reward": -0.0832907222211361, |
| "rewards/format_reward": 0.31250000558793545, |
| "step": 89 |
| }, |
| { |
| "completion_length": 2311.010467529297, |
| "epoch": 0.15424164524421594, |
| "grad_norm": 0.17067813873291016, |
| "kl": 0.0018815994262695312, |
| "learning_rate": 1.2713832064634125e-07, |
| "loss": 0.0001, |
| "reward": 0.1953146681189537, |
| "reward_std": 0.2011973224580288, |
| "rewards/cosine_scaled_reward": 0.08297107368707657, |
| "rewards/format_reward": 0.6041666716337204, |
| "step": 90 |
| }, |
| { |
| "completion_length": 2986.729248046875, |
| "epoch": 0.155955441302485, |
| "grad_norm": 0.1307849884033203, |
| "kl": 0.0007395744323730469, |
| "learning_rate": 1.220245676671809e-07, |
| "loss": 0.0, |
| "reward": 0.05273827910423279, |
| "reward_std": 0.22075917199254036, |
| "rewards/cosine_scaled_reward": -0.054260533303022385, |
| "rewards/format_reward": 0.3125000102445483, |
| "step": 91 |
| }, |
| { |
| "completion_length": 2813.3543090820312, |
| "epoch": 0.15766923736075408, |
| "grad_norm": 0.13788200914859772, |
| "kl": 0.0018596649169921875, |
| "learning_rate": 1.1743223682775649e-07, |
| "loss": 0.0001, |
| "reward": 0.02730799838900566, |
| "reward_std": 0.15018530935049057, |
| "rewards/cosine_scaled_reward": -0.13331225764704868, |
| "rewards/format_reward": 0.375, |
| "step": 92 |
| }, |
| { |
| "completion_length": 3271.2500610351562, |
| "epoch": 0.15938303341902313, |
| "grad_norm": 0.1422564685344696, |
| "kl": 0.001224517822265625, |
| "learning_rate": 1.1336692317580158e-07, |
| "loss": 0.0, |
| "reward": 0.08444089069962502, |
| "reward_std": 0.18314170092344284, |
| "rewards/cosine_scaled_reward": 0.026102278381586075, |
| "rewards/format_reward": 0.2812500037252903, |
| "step": 93 |
| }, |
| { |
| "completion_length": 3040.9688720703125, |
| "epoch": 0.1610968294772922, |
| "grad_norm": 0.1662042886018753, |
| "kl": 0.0022430419921875, |
| "learning_rate": 1.0983357966978745e-07, |
| "loss": 0.0001, |
| "reward": 0.028276806697249413, |
| "reward_std": 0.17833666875958443, |
| "rewards/cosine_scaled_reward": -0.14247922599315643, |
| "rewards/format_reward": 0.3958333544433117, |
| "step": 94 |
| }, |
| { |
| "completion_length": 3048.197998046875, |
| "epoch": 0.16281062553556128, |
| "grad_norm": 0.12611675262451172, |
| "kl": 0.0008878707885742188, |
| "learning_rate": 1.068365111445064e-07, |
| "loss": 0.0, |
| "reward": 0.07369702425785363, |
| "reward_std": 0.18385101296007633, |
| "rewards/cosine_scaled_reward": -0.07403519004583359, |
| "rewards/format_reward": 0.4375000074505806, |
| "step": 95 |
| }, |
| { |
| "completion_length": 3014.7918090820312, |
| "epoch": 0.16452442159383032, |
| "grad_norm": 0.16144989430904388, |
| "kl": 0.0016870498657226562, |
| "learning_rate": 1.0437936906629334e-07, |
| "loss": 0.0001, |
| "reward": 0.07583070383407176, |
| "reward_std": 0.21927380189299583, |
| "rewards/cosine_scaled_reward": -0.03254310414195061, |
| "rewards/format_reward": 0.3645833395421505, |
| "step": 96 |
| }, |
| { |
| "completion_length": 2385.0521240234375, |
| "epoch": 0.1662382176520994, |
| "grad_norm": 0.15538226068019867, |
| "kl": 0.0020532608032226562, |
| "learning_rate": 1.0246514708427701e-07, |
| "loss": 0.0001, |
| "reward": 0.09558990353252739, |
| "reward_std": 0.14071721211075783, |
| "rewards/cosine_scaled_reward": -0.1022979412227869, |
| "rewards/format_reward": 0.5833333507180214, |
| "step": 97 |
| }, |
| { |
| "completion_length": 2722.2500610351562, |
| "epoch": 0.16795201371036847, |
| "grad_norm": 0.16014371812343597, |
| "kl": 0.0013885498046875, |
| "learning_rate": 1.0109617738307911e-07, |
| "loss": 0.0001, |
| "reward": 0.024186712922528386, |
| "reward_std": 0.16764022037386894, |
| "rewards/cosine_scaled_reward": -0.1459345854818821, |
| "rewards/format_reward": 0.385416679084301, |
| "step": 98 |
| }, |
| { |
| "completion_length": 2599.0625610351562, |
| "epoch": 0.16966580976863754, |
| "grad_norm": 0.132161945104599, |
| "kl": 0.0023794174194335938, |
| "learning_rate": 1.002741278414069e-07, |
| "loss": 0.0001, |
| "reward": 0.11149843037128448, |
| "reward_std": 0.1786147654056549, |
| "rewards/cosine_scaled_reward": -0.06053047860041261, |
| "rewards/format_reward": 0.5625000149011612, |
| "step": 99 |
| }, |
| { |
| "completion_length": 2782.8126220703125, |
| "epoch": 0.1713796058269066, |
| "grad_norm": 0.15893962979316711, |
| "kl": 0.0020580291748046875, |
| "learning_rate": 1e-07, |
| "loss": 0.0001, |
| "reward": 0.0819746982306242, |
| "reward_std": 0.2312169447541237, |
| "rewards/cosine_scaled_reward": -0.04185225034598261, |
| "rewards/format_reward": 0.4062500074505806, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.1713796058269066, |
| "step": 100, |
| "total_flos": 0.0, |
| "train_loss": 3.568841239768084e-05, |
| "train_runtime": 22061.4799, |
| "train_samples_per_second": 0.435, |
| "train_steps_per_second": 0.005 |
| } |
| ], |
| "logging_steps": 1, |
| "max_steps": 100, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 1, |
| "save_steps": 50, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 4, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
